Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 53 Wiki Activity

changes

Browse Source
This commit is contained in:
Essie
2025-10-06 03:47:24 +09:00
parent 39784a1fea
commit c35650438c
22 changed files with 1480 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

235
LightlessSync/FileCache/FileCacheManager.cs
View File

@@ -1,4 +1,4 @@
using K4os.Compression.LZ4.Legacy;
using K4os.Compression.LZ4.Legacy;
using LightlessSync.Interop.Ipc;
using LightlessSync.LightlessConfiguration;
using LightlessSync.Services.Mediator;
@@ -19,7 +19,8 @@ public sealed class FileCacheManager : IHostedService
private readonly LightlessConfigService _configService;
private readonly LightlessMediator _lightlessMediator;
private readonly string _csvPath;
private readonly ConcurrentDictionary<string, List<FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, FileCacheEntity> _fileCachesByPrefixedPath = new(StringComparer.OrdinalIgnoreCase);
private readonly SemaphoreSlim _getCachesByPathsSemaphore = new(1, 1);
private readonly Lock _fileWriteLock = new();
private readonly IpcManager _ipcManager;
@@ -37,6 +38,57 @@ public sealed class FileCacheManager : IHostedService
private string CsvBakPath => _csvPath + ".bak";
private static string NormalizeSeparators(string path)
{
return path.Replace("/", "\\", StringComparison.Ordinal)
.Replace("\\\\", "\\", StringComparison.Ordinal);
}
private static string NormalizePrefixedPathKey(string prefixedPath)
{
if (string.IsNullOrEmpty(prefixedPath))
{
return string.Empty;
}
return NormalizeSeparators(prefixedPath).ToLowerInvariant();
}
private string NormalizeToPrefixedPath(string path)
{
if (string.IsNullOrEmpty(path)) return string.Empty;
var normalized = NormalizeSeparators(path);
if (normalized.StartsWith(CachePrefix, StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith(PenumbraPrefix, StringComparison.OrdinalIgnoreCase))
{
return NormalizePrefixedPathKey(normalized);
}
var penumbraDir = _ipcManager.Penumbra.ModDirectory;
if (!string.IsNullOrEmpty(penumbraDir))
{
var normalizedPenumbra = NormalizeSeparators(penumbraDir);
var replacement = normalizedPenumbra.EndsWith("\\", StringComparison.Ordinal)
? PenumbraPrefix + "\\"
: PenumbraPrefix;
normalized = normalized.Replace(normalizedPenumbra, replacement, StringComparison.OrdinalIgnoreCase);
}
var cacheFolder = _configService.Current.CacheFolder;
if (!string.IsNullOrEmpty(cacheFolder))
{
var normalizedCache = NormalizeSeparators(cacheFolder);
var replacement = normalizedCache.EndsWith("\\", StringComparison.Ordinal)
? CachePrefix + "\\"
: CachePrefix;
normalized = normalized.Replace(normalizedCache, replacement, StringComparison.OrdinalIgnoreCase);
}
return NormalizePrefixedPathKey(normalized);
}
public FileCacheEntity? CreateCacheEntry(string path)
{
FileInfo fi = new(path);
@@ -61,20 +113,26 @@ public sealed class FileCacheManager : IHostedService
return CreateFileCacheEntity(fi, prefixedPath);
}
public List<FileCacheEntity> GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v).ToList();
public List<FileCacheEntity> GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v.Values.Where(e => e != null)).ToList();
public List<FileCacheEntity> GetAllFileCachesByHash(string hash, bool ignoreCacheEntries = false, bool validate = true)
{
List<FileCacheEntity> output = [];
if (_fileCaches.TryGetValue(hash, out var fileCacheEntities))
{
foreach (var fileCache in fileCacheEntities.Where(c => !ignoreCacheEntries || !c.IsCacheEntry).ToList())
foreach (var fileCache in fileCacheEntities.Values.Where(c => !ignoreCacheEntries || !c.IsCacheEntry).ToList())
{
if (!validate) output.Add(fileCache);
if (!validate)
{
output.Add(fileCache);
}
else
{
var validated = GetValidatedFileCache(fileCache);
if (validated != null) output.Add(validated);
if (validated != null)
{
output.Add(validated);
}
}
}
}
@@ -86,7 +144,7 @@ public sealed class FileCacheManager : IHostedService
{
_lightlessMediator.Publish(new HaltScanMessage(nameof(ValidateLocalIntegrity)));
_logger.LogInformation("Validating local storage");
var cacheEntries = _fileCaches.SelectMany(v => v.Value).Where(v => v.IsCacheEntry).ToList();
var cacheEntries = _fileCaches.Values.SelectMany(v => v.Values.Where(e => e != null)).Where(v => v.IsCacheEntry).ToList();
List<FileCacheEntity> brokenEntities = [];
int i = 0;
foreach (var fileCache in cacheEntries)
@@ -151,29 +209,40 @@ public sealed class FileCacheManager : IHostedService
public FileCacheEntity? GetFileCacheByHash(string hash)
{
if (_fileCaches.TryGetValue(hash, out var hashes))
if (_fileCaches.TryGetValue(hash, out var entries))
{
var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix, StringComparison.Ordinal) ? 0 : 1).FirstOrDefault();
if (item != null) return GetValidatedFileCache(item);
var item = entries.Values
.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix, StringComparison.Ordinal) ? 0 : 1)
.FirstOrDefault();
if (item != null)
{
return GetValidatedFileCache(item);
}
}
return null;
}
private FileCacheEntity? GetFileCacheByPath(string path)
{
var cleanedPath = path.Replace("/", "\\", StringComparison.OrdinalIgnoreCase).ToLowerInvariant()
.Replace(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), "", StringComparison.OrdinalIgnoreCase);
var entry = _fileCaches.SelectMany(v => v.Value).FirstOrDefault(f => f.ResolvedFilepath.EndsWith(cleanedPath, StringComparison.OrdinalIgnoreCase));
if (entry == null)
var normalizedPrefixedPath = NormalizeToPrefixedPath(path);
if (string.IsNullOrEmpty(normalizedPrefixedPath))
{
_logger.LogDebug("Found no entries for {path}", cleanedPath);
return CreateFileEntry(path);
return null;
}
var validatedCacheEntry = GetValidatedFileCache(entry);
if (_fileCachesByPrefixedPath.TryGetValue(normalizedPrefixedPath, out var entry))
{
return GetValidatedFileCache(entry);
}
return validatedCacheEntry;
_logger.LogDebug("Found no entries for {path}", normalizedPrefixedPath);
if (normalizedPrefixedPath.Contains(CachePrefix, StringComparison.Ordinal))
{
return CreateCacheEntry(path);
}
return CreateFileEntry(path) ?? CreateCacheEntry(path);
}
public Dictionary<string, FileCacheEntity?> GetFileCachesByPaths(string[] paths)
@@ -182,73 +251,55 @@ public sealed class FileCacheManager : IHostedService
try
{
var allEntities = _fileCaches.SelectMany(f => f.Value).ToArray();
var result = new Dictionary<string, FileCacheEntity?>(StringComparer.OrdinalIgnoreCase);
var seenNormalized = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var cacheDict = new ConcurrentDictionary<string, FileCacheEntity>(
StringComparer.OrdinalIgnoreCase);
Parallel.ForEach(allEntities, entity =>
foreach (var originalPath in paths)
{
if (entity != null && entity.PrefixedFilePath != null)
if (string.IsNullOrEmpty(originalPath))
{
cacheDict[entity.PrefixedFilePath] = entity;
result[originalPath] = null;
continue;
}
var normalized = NormalizeToPrefixedPath(originalPath);
if (seenNormalized.Add(normalized))
{
if (!string.IsNullOrEmpty(normalized))
{
_logger.LogDebug("Normalized path {cleaned}", normalized);
}
}
else if (!string.IsNullOrEmpty(normalized))
{
_logger.LogWarning("Duplicate normalized path detected: {cleaned}", normalized);
}
if (_fileCachesByPrefixedPath.TryGetValue(normalized, out var entity))
{
result[originalPath] = GetValidatedFileCache(entity);
continue;
}
FileCacheEntity? created = null;
if (normalized.Contains(CachePrefix, StringComparison.Ordinal))
{
created = CreateCacheEntry(originalPath);
}
else if (normalized.Contains(PenumbraPrefix, StringComparison.Ordinal))
{
created = CreateFileEntry(originalPath);
}
else
{
_logger.LogWarning("Null FileCacheEntity or PrefixedFilePath encountered in cache population: {entity}", entity);
created = CreateFileEntry(originalPath) ?? CreateCacheEntry(originalPath);
}
});
var cleanedPaths = new ConcurrentDictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var seenCleaned = new ConcurrentDictionary<string, byte>(StringComparer.OrdinalIgnoreCase);
result[originalPath] = created;
}
Parallel.ForEach(paths, p =>
{
var cleaned = p.Replace("/", "\\", StringComparison.OrdinalIgnoreCase)
.Replace(
_ipcManager.Penumbra.ModDirectory!,
_ipcManager.Penumbra.ModDirectory!.EndsWith('\\')
? PenumbraPrefix + '\\' : PenumbraPrefix,
StringComparison.OrdinalIgnoreCase)
.Replace(
_configService.Current.CacheFolder,
_configService.Current.CacheFolder.EndsWith('\\')
? CachePrefix + '\\' : CachePrefix,
StringComparison.OrdinalIgnoreCase)
.Replace("\\\\", "\\", StringComparison.Ordinal);
if (seenCleaned.TryAdd(cleaned, 0))
{
_logger.LogDebug("Adding to cleanedPaths: {cleaned}", cleaned);
cleanedPaths[p] = cleaned;
}
else
{
_logger.LogWarning("Duplicate found: {cleaned}", cleaned);
}
});
var result = new ConcurrentDictionary<string, FileCacheEntity?>(StringComparer.OrdinalIgnoreCase);
Parallel.ForEach(cleanedPaths, entry =>
{
_logger.LogDebug("Checking if in cache: {path}", entry.Value);
if (cacheDict.TryGetValue(entry.Value, out var entity))
{
var validatedCache = GetValidatedFileCache(entity);
result[entry.Key] = validatedCache;
}
else
{
if (!entry.Value.Contains(CachePrefix, StringComparison.Ordinal))
result[entry.Key] = CreateFileEntry(entry.Key);
else
result[entry.Key] = CreateCacheEntry(entry.Key);
}
});
return new Dictionary<string, FileCacheEntity?>(result, StringComparer.OrdinalIgnoreCase);
return result;
}
finally
{
@@ -258,17 +309,24 @@ public sealed class FileCacheManager : IHostedService
public void RemoveHashedFile(string hash, string prefixedFilePath)
{
var normalizedPath = NormalizePrefixedPathKey(prefixedFilePath);
if (_fileCaches.TryGetValue(hash, out var caches))
{
_logger.LogTrace("Removing from DB: {hash} => {path}", hash, prefixedFilePath);
var removedCount = caches?.RemoveAll(c => string.Equals(c.PrefixedFilePath, prefixedFilePath, StringComparison.Ordinal));
_logger.LogTrace("Removed from DB: {count} file(s) with hash {hash} and file cache {path}", removedCount, hash, prefixedFilePath);
if (caches?.Count == 0)
if (caches.TryRemove(normalizedPath, out var removedEntity))
{
_fileCaches.Remove(hash, out var entity);
_logger.LogTrace("Removed from DB: {hash} => {path}", hash, removedEntity.PrefixedFilePath);
}
if (caches.IsEmpty)
{
_fileCaches.TryRemove(hash, out _);
}
}
_fileCachesByPrefixedPath.TryRemove(normalizedPath, out _);
}
public void UpdateHashedFile(FileCacheEntity fileCache, bool computeProperties = true)
@@ -309,7 +367,7 @@ public sealed class FileCacheManager : IHostedService
lock (_fileWriteLock)
{
StringBuilder sb = new();
foreach (var entry in _fileCaches.SelectMany(k => k.Value).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase))
foreach (var entry in _fileCaches.Values.SelectMany(k => k.Values).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase))
{
sb.AppendLine(entry.CsvEntry);
}
@@ -354,16 +412,11 @@ public sealed class FileCacheManager : IHostedService
private void AddHashedFile(FileCacheEntity fileCache)
{
if (!_fileCaches.TryGetValue(fileCache.Hash, out var entries) || entries is null)
{
_fileCaches[fileCache.Hash] = entries = [];
}
var normalizedPath = NormalizePrefixedPathKey(fileCache.PrefixedFilePath);
var entries = _fileCaches.GetOrAdd(fileCache.Hash, _ => new ConcurrentDictionary<string, FileCacheEntity>(StringComparer.OrdinalIgnoreCase));
if (!entries.Exists(u => string.Equals(u.PrefixedFilePath, fileCache.PrefixedFilePath, StringComparison.OrdinalIgnoreCase)))
{
//_logger.LogTrace("Adding to DB: {hash} => {path}", fileCache.Hash, fileCache.PrefixedFilePath);
entries.Add(fileCache);
}
entries[normalizedPath] = fileCache;
_fileCachesByPrefixedPath[normalizedPath] = fileCache;
}
private FileCacheEntity? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath, string? hash = null)