Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 56 Wiki Activity

Redone caching in parralel threading, added retries on timeouts on download sessions.

Browse Source
This commit is contained in:
CakeAndBanana
2025-09-15 05:41:56 +02:00
parent 3c717bf5ef
commit 7b999bfbbc
5 changed files with 107 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
LightlessSync/FileCache/CacheMonitor.cs
View File

@@ -383,7 +383,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
scanThread.Start();
while (scanThread.IsAlive)
{
await Task.Delay(250).ConfigureAwait(false);
await Task.Delay(250, token).ConfigureAwait(false);
}
TotalFiles = 0;
_currentFileProgress = 0;
@@ -619,7 +619,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
return;
}
if (entitiesToUpdate.Any() || entitiesToRemove.Any())
if (entitiesToUpdate.Count != 0 || entitiesToRemove.Count != 0)
{
foreach (var entity in entitiesToUpdate)
{

82
LightlessSync/FileCache/FileCacheManager.cs
View File

@@ -21,7 +21,7 @@ public sealed class FileCacheManager : IHostedService
private readonly string _csvPath;
private readonly ConcurrentDictionary<string, List<FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
private readonly SemaphoreSlim _getCachesByPathsSemaphore = new(1, 1);
private readonly object _fileWriteLock = new();
private readonly Lock _fileWriteLock = new();
private readonly IpcManager _ipcManager;
private readonly ILogger<FileCacheManager> _logger;
public string CacheFolder => _configService.Current.CacheFolder;
@@ -42,10 +42,7 @@ public sealed class FileCacheManager : IHostedService
FileInfo fi = new(path);
if (!fi.Exists) return null;
_logger.LogTrace("Creating cache entry for {path}", path);
var fullName = fi.FullName.ToLowerInvariant();
if (!fullName.Contains(_configService.Current.CacheFolder.ToLowerInvariant(), StringComparison.Ordinal)) return null;
string prefixedPath = fullName.Replace(_configService.Current.CacheFolder.ToLowerInvariant(), CachePrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
return CreateFileCacheEntity(fi, prefixedPath);
return CreateFileEntity(_configService.Current.CacheFolder.ToLowerInvariant(), CachePrefix, fi);
}
public FileCacheEntity? CreateFileEntry(string path)
@@ -53,9 +50,14 @@ public sealed class FileCacheManager : IHostedService
FileInfo fi = new(path);
if (!fi.Exists) return null;
_logger.LogTrace("Creating file entry for {path}", path);
return CreateFileEntity(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), PenumbraPrefix, fi);
}
private FileCacheEntity? CreateFileEntity(string directory, string prefix, FileInfo fi)
{
var fullName = fi.FullName.ToLowerInvariant();
if (!fullName.Contains(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), StringComparison.Ordinal)) return null;
string prefixedPath = fullName.Replace(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), PenumbraPrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
if (!fullName.Contains(_configService.Current.CacheFolder.ToLowerInvariant(), StringComparison.Ordinal)) return null;
string prefixedPath = fullName.Replace(directory, prefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
return CreateFileCacheEntity(fi, prefixedPath);
}
@@ -66,7 +68,7 @@ public sealed class FileCacheManager : IHostedService
List<FileCacheEntity> output = [];
if (_fileCaches.TryGetValue(hash, out var fileCacheEntities))
{
foreach (var fileCache in fileCacheEntities.Where(c => ignoreCacheEntries ? !c.IsCacheEntry : true).ToList())
foreach (var fileCache in fileCacheEntities.Where(c => !ignoreCacheEntries || !c.IsCacheEntry).ToList())
{
if (!validate) output.Add(fileCache);
else
@@ -106,7 +108,7 @@ public sealed class FileCacheManager : IHostedService
var computedHash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal))
{
_logger.LogInformation("Failed to validate {file}, got hash {hash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
_logger.LogInformation("Failed to validate {file}, got hash {computedHash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
brokenEntities.Add(fileCache);
}
}
@@ -151,7 +153,7 @@ public sealed class FileCacheManager : IHostedService
{
if (_fileCaches.TryGetValue(hash, out var hashes))
{
var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix) ? 0 : 1).FirstOrDefault();
var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix, StringComparison.Ordinal) ? 0 : 1).FirstOrDefault();
if (item != null) return GetValidatedFileCache(item);
}
return null;
@@ -180,50 +182,66 @@ public sealed class FileCacheManager : IHostedService
try
{
var cleanedPaths = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var seenCleaned = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var allEntities = _fileCaches.SelectMany(f => f.Value).ToArray();
foreach (var p in paths)
var cacheDict = new ConcurrentDictionary<string, FileCacheEntity>(
StringComparer.OrdinalIgnoreCase);
Parallel.ForEach(allEntities, entity =>
{
cacheDict[entity.PrefixedFilePath] = entity;
});
var cleanedPaths = new ConcurrentDictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var seenCleaned = new ConcurrentDictionary<string, byte>(StringComparer.OrdinalIgnoreCase);
Parallel.ForEach(paths, p =>
{
var cleaned = p.Replace("/", "\\", StringComparison.OrdinalIgnoreCase)
.Replace(_ipcManager.Penumbra.ModDirectory!, _ipcManager.Penumbra.ModDirectory!.EndsWith('\\') ? PenumbraPrefix + '\\' : PenumbraPrefix, StringComparison.OrdinalIgnoreCase)
.Replace(_configService.Current.CacheFolder, _configService.Current.CacheFolder.EndsWith('\\') ? CachePrefix + '\\' : CachePrefix, StringComparison.OrdinalIgnoreCase)
.Replace(
_ipcManager.Penumbra.ModDirectory!,
_ipcManager.Penumbra.ModDirectory!.EndsWith('\\')
? PenumbraPrefix + '\\' : PenumbraPrefix,
StringComparison.OrdinalIgnoreCase)
.Replace(
_configService.Current.CacheFolder,
_configService.Current.CacheFolder.EndsWith('\\')
? CachePrefix + '\\' : CachePrefix,
StringComparison.OrdinalIgnoreCase)
.Replace("\\\\", "\\", StringComparison.Ordinal);
if (seenCleaned.Add(cleaned))
if (seenCleaned.TryAdd(cleaned, 0))
{
_logger.LogDebug("Adding to cleanedPaths: {cleaned}", cleaned);
cleanedPaths[p] = cleaned;
} else
{
_logger.LogWarning("Duplicated found: {cleaned}", cleaned);
}
}
else
{
_logger.LogWarning("Duplicate found: {cleaned}", cleaned);
}
});
Dictionary<string, FileCacheEntity?> result = new(StringComparer.OrdinalIgnoreCase);
var result = new ConcurrentDictionary<string, FileCacheEntity?>(StringComparer.OrdinalIgnoreCase);
var dict = _fileCaches.SelectMany(f => f.Value).Distinct()
.ToDictionary(d => d.PrefixedFilePath, d => d, StringComparer.OrdinalIgnoreCase);
foreach (var entry in cleanedPaths)
Parallel.ForEach(cleanedPaths, entry =>
{
_logger.LogDebug("Checking if in cache: {path}", entry.Value);
if (dict.TryGetValue(entry.Value, out var entity))
if (cacheDict.TryGetValue(entry.Value, out var entity))
{
var validatedCache = GetValidatedFileCache(entity);
result.Add(entry.Key, validatedCache);
result[entry.Key] = validatedCache;
}
else
{
if (!entry.Value.Contains(CachePrefix, StringComparison.Ordinal))
result.Add(entry.Key, CreateFileEntry(entry.Key));
result[entry.Key] = CreateFileEntry(entry.Key);
else
result.Add(entry.Key, CreateCacheEntry(entry.Key));
result[entry.Key] = CreateCacheEntry(entry.Key);
}
}
});
return result;
return new Dictionary<string, FileCacheEntity?>(result, StringComparer.OrdinalIgnoreCase);
}
finally
{
@@ -452,7 +470,7 @@ public sealed class FileCacheManager : IHostedService
{
attempts++;
_logger.LogWarning(ex, "Could not open {file}, trying again", _csvPath);
Thread.Sleep(100);
Task.Delay(100, cancellationToken);
}
}