Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 51 Wiki Activity

Redone caching in parralel threading, added retries on timeouts on download sessions.

Browse Source
This commit is contained in:
CakeAndBanana
2025-09-15 05:41:56 +02:00
parent 3c717bf5ef
commit 7b999bfbbc
5 changed files with 107 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
LightlessSync/FileCache/CacheMonitor.cs
View File

@@ -383,7 +383,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
scanThread.Start();
while (scanThread.IsAlive)
{
await Task.Delay(250).ConfigureAwait(false);
await Task.Delay(250, token).ConfigureAwait(false);
}
TotalFiles = 0;
_currentFileProgress = 0;
@@ -619,7 +619,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
return;
}
if (entitiesToUpdate.Any() || entitiesToRemove.Any())
if (entitiesToUpdate.Count != 0 || entitiesToRemove.Count != 0)
{
foreach (var entity in entitiesToUpdate)
{

82
LightlessSync/FileCache/FileCacheManager.cs
View File

@@ -21,7 +21,7 @@ public sealed class FileCacheManager : IHostedService
private readonly string _csvPath;
private readonly ConcurrentDictionary<string, List<FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
private readonly SemaphoreSlim _getCachesByPathsSemaphore = new(1, 1);
private readonly object _fileWriteLock = new();
private readonly Lock _fileWriteLock = new();
private readonly IpcManager _ipcManager;
private readonly ILogger<FileCacheManager> _logger;
public string CacheFolder => _configService.Current.CacheFolder;
@@ -42,10 +42,7 @@ public sealed class FileCacheManager : IHostedService
FileInfo fi = new(path);
if (!fi.Exists) return null;
_logger.LogTrace("Creating cache entry for {path}", path);
var fullName = fi.FullName.ToLowerInvariant();
if (!fullName.Contains(_configService.Current.CacheFolder.ToLowerInvariant(), StringComparison.Ordinal)) return null;
string prefixedPath = fullName.Replace(_configService.Current.CacheFolder.ToLowerInvariant(), CachePrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
return CreateFileCacheEntity(fi, prefixedPath);
return CreateFileEntity(_configService.Current.CacheFolder.ToLowerInvariant(), CachePrefix, fi);
}
public FileCacheEntity? CreateFileEntry(string path)
@@ -53,9 +50,14 @@ public sealed class FileCacheManager : IHostedService
FileInfo fi = new(path);
if (!fi.Exists) return null;
_logger.LogTrace("Creating file entry for {path}", path);
return CreateFileEntity(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), PenumbraPrefix, fi);
}
private FileCacheEntity? CreateFileEntity(string directory, string prefix, FileInfo fi)
{
var fullName = fi.FullName.ToLowerInvariant();
if (!fullName.Contains(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), StringComparison.Ordinal)) return null;
string prefixedPath = fullName.Replace(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), PenumbraPrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
if (!fullName.Contains(_configService.Current.CacheFolder.ToLowerInvariant(), StringComparison.Ordinal)) return null;
string prefixedPath = fullName.Replace(directory, prefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
return CreateFileCacheEntity(fi, prefixedPath);
}
@@ -66,7 +68,7 @@ public sealed class FileCacheManager : IHostedService
List<FileCacheEntity> output = [];
if (_fileCaches.TryGetValue(hash, out var fileCacheEntities))
{
foreach (var fileCache in fileCacheEntities.Where(c => ignoreCacheEntries ? !c.IsCacheEntry : true).ToList())
foreach (var fileCache in fileCacheEntities.Where(c => !ignoreCacheEntries || !c.IsCacheEntry).ToList())
{
if (!validate) output.Add(fileCache);
else
@@ -106,7 +108,7 @@ public sealed class FileCacheManager : IHostedService
var computedHash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal))
{
_logger.LogInformation("Failed to validate {file}, got hash {hash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
_logger.LogInformation("Failed to validate {file}, got hash {computedHash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
brokenEntities.Add(fileCache);
}
}
@@ -151,7 +153,7 @@ public sealed class FileCacheManager : IHostedService
{
if (_fileCaches.TryGetValue(hash, out var hashes))
{
var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix) ? 0 : 1).FirstOrDefault();
var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix, StringComparison.Ordinal) ? 0 : 1).FirstOrDefault();
if (item != null) return GetValidatedFileCache(item);
}
return null;
@@ -180,50 +182,66 @@ public sealed class FileCacheManager : IHostedService
try
{
var cleanedPaths = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var seenCleaned = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var allEntities = _fileCaches.SelectMany(f => f.Value).ToArray();
foreach (var p in paths)
var cacheDict = new ConcurrentDictionary<string, FileCacheEntity>(
StringComparer.OrdinalIgnoreCase);
Parallel.ForEach(allEntities, entity =>
{
cacheDict[entity.PrefixedFilePath] = entity;
});
var cleanedPaths = new ConcurrentDictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var seenCleaned = new ConcurrentDictionary<string, byte>(StringComparer.OrdinalIgnoreCase);
Parallel.ForEach(paths, p =>
{
var cleaned = p.Replace("/", "\\", StringComparison.OrdinalIgnoreCase)
.Replace(_ipcManager.Penumbra.ModDirectory!, _ipcManager.Penumbra.ModDirectory!.EndsWith('\\') ? PenumbraPrefix + '\\' : PenumbraPrefix, StringComparison.OrdinalIgnoreCase)
.Replace(_configService.Current.CacheFolder, _configService.Current.CacheFolder.EndsWith('\\') ? CachePrefix + '\\' : CachePrefix, StringComparison.OrdinalIgnoreCase)
.Replace(
_ipcManager.Penumbra.ModDirectory!,
_ipcManager.Penumbra.ModDirectory!.EndsWith('\\')
? PenumbraPrefix + '\\' : PenumbraPrefix,
StringComparison.OrdinalIgnoreCase)
.Replace(
_configService.Current.CacheFolder,
_configService.Current.CacheFolder.EndsWith('\\')
? CachePrefix + '\\' : CachePrefix,
StringComparison.OrdinalIgnoreCase)
.Replace("\\\\", "\\", StringComparison.Ordinal);
if (seenCleaned.Add(cleaned))
if (seenCleaned.TryAdd(cleaned, 0))
{
_logger.LogDebug("Adding to cleanedPaths: {cleaned}", cleaned);
cleanedPaths[p] = cleaned;
} else
{
_logger.LogWarning("Duplicated found: {cleaned}", cleaned);
}
}
else
{
_logger.LogWarning("Duplicate found: {cleaned}", cleaned);
}
});
Dictionary<string, FileCacheEntity?> result = new(StringComparer.OrdinalIgnoreCase);
var result = new ConcurrentDictionary<string, FileCacheEntity?>(StringComparer.OrdinalIgnoreCase);
var dict = _fileCaches.SelectMany(f => f.Value).Distinct()
.ToDictionary(d => d.PrefixedFilePath, d => d, StringComparer.OrdinalIgnoreCase);
foreach (var entry in cleanedPaths)
Parallel.ForEach(cleanedPaths, entry =>
{
_logger.LogDebug("Checking if in cache: {path}", entry.Value);
if (dict.TryGetValue(entry.Value, out var entity))
if (cacheDict.TryGetValue(entry.Value, out var entity))
{
var validatedCache = GetValidatedFileCache(entity);
result.Add(entry.Key, validatedCache);
result[entry.Key] = validatedCache;
}
else
{
if (!entry.Value.Contains(CachePrefix, StringComparison.Ordinal))
result.Add(entry.Key, CreateFileEntry(entry.Key));
result[entry.Key] = CreateFileEntry(entry.Key);
else
result.Add(entry.Key, CreateCacheEntry(entry.Key));
result[entry.Key] = CreateCacheEntry(entry.Key);
}
}
});
return result;
return new Dictionary<string, FileCacheEntity?>(result, StringComparer.OrdinalIgnoreCase);
}
finally
{
@@ -452,7 +470,7 @@ public sealed class FileCacheManager : IHostedService
{
attempts++;
_logger.LogWarning(ex, "Could not open {file}, trying again", _csvPath);
Thread.Sleep(100);
Task.Delay(100, cancellationToken);
}
}

6
LightlessSync/Services/CharaData/CharaDataManager.cs
View File

@@ -944,9 +944,7 @@ public sealed partial class CharaDataManager : DisposableMediatorSubscriberBase
Logger.LogTrace("[{appId}] Computing local missing files", applicationId);
Dictionary<string, string> modPaths;
List<FileReplacementData> missingFiles;
_fileHandler.ComputeMissingFiles(charaDataDownloadDto, out modPaths, out missingFiles);
_fileHandler.ComputeMissingFiles(charaDataDownloadDto, out Dictionary<string, string> modPaths, out List<FileReplacementData> missingFiles);
Logger.LogTrace("[{appId}] Computing local missing files", applicationId);
@@ -990,7 +988,7 @@ public sealed partial class CharaDataManager : DisposableMediatorSubscriberBase
{
_uploadCts = _uploadCts.CancelRecreate();
var missingFiles = await _fileHandler.UploadFiles([.. missingFileList.Select(k => k.HashOrFileSwap)], UploadProgress, _uploadCts.Token).ConfigureAwait(false);
if (missingFiles.Any())
if (missingFiles.Count != 0)
{
Logger.LogInformation("Failed to upload {files}", string.Join(", ", missingFiles));
return ($"Upload failed: {missingFiles.Count} missing or forbidden to upload local files.", false);

68
LightlessSync/WebAPI/Files/FileDownloadManager.cs
View File

@@ -144,28 +144,55 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
_downloadStatus[downloadGroup].DownloadStatus = DownloadStatus.Downloading;
const int maxRetries = 3;
int retryCount = 0;
TimeSpan retryDelay = TimeSpan.FromSeconds(2);
HttpResponseMessage response = null!;
var requestUrl = LightlessFiles.CacheGetFullPath(fileTransfer[0].DownloadUri, requestId);
Logger.LogDebug("Downloading {requestUrl} for request {id}", requestUrl, requestId);
try
while (true)
{
response = await _orchestrator.SendRequestAsync(HttpMethod.Get, requestUrl, ct, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
}
catch (HttpRequestException ex)
{
Logger.LogWarning(ex, "Error during download of {requestUrl}, HttpStatusCode: {code}", requestUrl, ex.StatusCode);
if (ex.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Unauthorized)
try
{
throw new InvalidDataException($"Http error {ex.StatusCode} (cancelled: {ct.IsCancellationRequested}): {requestUrl}", ex);
Logger.LogDebug("Attempt {attempt} - Downloading {requestUrl} for request {id}", retryCount + 1, requestUrl, requestId);
response = await _orchestrator.SendRequestAsync(HttpMethod.Get, requestUrl, ct, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
break;
}
catch (HttpRequestException ex) when (ex.InnerException is TimeoutException || ex.StatusCode == null)
{
retryCount++;
Logger.LogWarning(ex, "Timeout during download of {requestUrl}. Attempt {attempt} of {maxRetries}", requestUrl, retryCount, maxRetries);
if (retryCount >= maxRetries || ct.IsCancellationRequested)
{
Logger.LogError($"Max retries reached or cancelled. Failing download for {requestUrl}");
throw;
}
await Task.Delay(retryDelay, ct).ConfigureAwait(false); // Wait before retrying
}
catch (HttpRequestException ex)
{
Logger.LogWarning(ex, "Error during download of {requestUrl}, HttpStatusCode: {code}", requestUrl, ex.StatusCode);
if (ex.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Unauthorized)
{
throw new InvalidDataException($"Http error {ex.StatusCode} (cancelled: {ct.IsCancellationRequested}): {requestUrl}", ex);
}
throw;
}
}
ThrottledStream? stream = null;
FileStream? fileStream = null;
try
{
var fileStream = File.Create(tempPath);
fileStream = File.Create(tempPath);
await using (fileStream.ConfigureAwait(false))
{
var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 65536 : 8196;
@@ -174,8 +201,11 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
var bytesRead = 0;
var limit = _orchestrator.DownloadLimitPerSlot();
Logger.LogTrace("Starting Download of {id} with a speed limit of {limit} to {tempPath}", requestId, limit, tempPath);
stream = new ThrottledStream(await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false), limit);
stream = new(await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false), limit);
_activeDownloadStreams.Add(stream);
while ((bytesRead = await stream.ReadAsync(buffer, ct).ConfigureAwait(false)) > 0)
{
ct.ThrowIfCancellationRequested();
@@ -194,18 +224,22 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
{
throw;
}
catch (Exception ex)
catch (Exception)
{
try
{
if (!tempPath.IsNullOrEmpty())
fileStream?.Close();
if (!string.IsNullOrEmpty(tempPath) && File.Exists(tempPath))
{
File.Delete(tempPath);
}
}
catch
{
// ignore if file deletion fails
// Ignore errors during cleanup
}
throw;
throw;
}
finally
{

4
LightlessSync/WebAPI/Files/FileUploadManager.cs
View File

@@ -69,7 +69,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
Logger.LogDebug("Trying to upload files");
var filesPresentLocally = hashesToUpload.Where(h => _fileDbManager.GetFileCacheByHash(h) != null).ToHashSet(StringComparer.Ordinal);
var locallyMissingFiles = hashesToUpload.Except(filesPresentLocally, StringComparer.Ordinal).ToList();
if (locallyMissingFiles.Any())
if (locallyMissingFiles.Count != 0)
{
return locallyMissingFiles;
}
@@ -92,7 +92,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
var data = await _fileDbManager.GetCompressedFileData(file.Hash, ct ?? CancellationToken.None).ConfigureAwait(false);
Logger.LogDebug("[{hash}] Starting upload for {filePath}", data.Item1, _fileDbManager.GetFileCacheByHash(data.Item1)!.ResolvedFilepath);
await uploadTask.ConfigureAwait(false);
uploadTask = UploadFile(data.Item2, file.Hash, false, ct ?? CancellationToken.None);
uploadTask = UploadFile(data.Item2, file.Hash, postProgress: false, ct ?? CancellationToken.None);
(ct ?? CancellationToken.None).ThrowIfCancellationRequested();
}