Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 53 Wiki Activity

Merged Cake and Abel branched into 2.0.3 (#131)

Browse Source 
Co-authored-by: azyges <aaaaaa@aaa.aaa>
Co-authored-by: cake <admin@cakeandbanana.nl>
Co-authored-by: defnotken <itsdefnotken@gmail.com>
Reviewed-on: #131
This commit was merged in pull request #131.
This commit is contained in:
cake
2026-01-05 00:45:14 +00:00
parent e0b8070aa8
commit 30717ba200
67 changed files with 13247 additions and 802 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
LightlessSync/FileCache/FileCacheManager.cs
View File

@@ -27,6 +27,7 @@ public sealed class FileCacheManager : IHostedService
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, FileCacheEntity> _fileCachesByPrefixedPath = new(StringComparer.OrdinalIgnoreCase);
private readonly SemaphoreSlim _getCachesByPathsSemaphore = new(1, 1);
private readonly SemaphoreSlim _evictSemaphore = new(1, 1);
private readonly Lock _fileWriteLock = new();
private readonly IpcManager _ipcManager;
private readonly ILogger<FileCacheManager> _logger;
@@ -226,13 +227,23 @@ public sealed class FileCacheManager : IHostedService
var compressed = LZ4Wrapper.WrapHC(raw, 0, raw.Length);
var tmpPath = compressedPath + ".tmp";
await File.WriteAllBytesAsync(tmpPath, compressed, token).ConfigureAwait(false);
File.Move(tmpPath, compressedPath, overwrite: true);
try
{
await File.WriteAllBytesAsync(tmpPath, compressed, token).ConfigureAwait(false);
File.Move(tmpPath, compressedPath, overwrite: true);
}
finally
{
try { if (File.Exists(tmpPath)) File.Delete(tmpPath); } catch { /* ignore */ }
}
var compressedSize = compressed.LongLength;
var compressedSize = new FileInfo(compressedPath).Length;
SetSizeInfo(hash, originalSize, compressedSize);
UpdateEntitiesSizes(hash, originalSize, compressedSize);
var maxBytes = GiBToBytes(_configService.Current.MaxLocalCacheInGiB);
await EnforceCacheLimitAsync(maxBytes, token).ConfigureAwait(false);
return compressed;
}
finally
@@ -280,6 +291,26 @@ public sealed class FileCacheManager : IHostedService
return CreateFileEntity(cacheFolder, CachePrefix, fi);
}
public FileCacheEntity? CreateCacheEntryWithKnownHash(string path, string hash)
{
if (string.IsNullOrWhiteSpace(hash))
{
return CreateCacheEntry(path);
}
FileInfo fi = new(path);
if (!fi.Exists) return null;
_logger.LogTrace("Creating cache entry for {path} using provided hash", path);
var cacheFolder = _configService.Current.CacheFolder;
if (string.IsNullOrEmpty(cacheFolder)) return null;
if (!TryBuildPrefixedPath(fi.FullName, cacheFolder, CachePrefix, out var prefixedPath, out _))
{
return null;
}
return CreateFileCacheEntity(fi, prefixedPath, hash);
}
public FileCacheEntity? CreateFileEntry(string path)
{
FileInfo fi = new(path);
@@ -562,9 +593,10 @@ public sealed class FileCacheManager : IHostedService
}
}
public void RemoveHashedFile(string hash, string prefixedFilePath)
public void RemoveHashedFile(string hash, string prefixedFilePath, bool removeDerivedFiles = true)
{
var normalizedPath = NormalizePrefixedPathKey(prefixedFilePath);
var removedHash = false;
if (_fileCaches.TryGetValue(hash, out var caches))
{
@@ -577,11 +609,16 @@ public sealed class FileCacheManager : IHostedService
if (caches.IsEmpty)
{
_fileCaches.TryRemove(hash, out _);
removedHash = _fileCaches.TryRemove(hash, out _);
}
}
_fileCachesByPrefixedPath.TryRemove(normalizedPath, out _);
if (removeDerivedFiles && removedHash)
{
RemoveDerivedCacheFiles(hash);
}
}
public void UpdateHashedFile(FileCacheEntity fileCache, bool computeProperties = true)
@@ -597,7 +634,8 @@ public sealed class FileCacheManager : IHostedService
fileCache.Hash = Crypto.ComputeFileHash(fileCache.ResolvedFilepath, Crypto.HashAlgo.Sha1);
fileCache.LastModifiedDateTicks = fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
}
RemoveHashedFile(oldHash, prefixedPath);
var removeDerivedFiles = !string.Equals(oldHash, fileCache.Hash, StringComparison.OrdinalIgnoreCase);
RemoveHashedFile(oldHash, prefixedPath, removeDerivedFiles);
AddHashedFile(fileCache);
}
@@ -747,7 +785,7 @@ public sealed class FileCacheManager : IHostedService
{
try
{
RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath);
RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath, removeDerivedFiles: false);
var extensionPath = fileCache.ResolvedFilepath.ToUpper(CultureInfo.InvariantCulture) + "." + ext;
File.Move(fileCache.ResolvedFilepath, extensionPath, overwrite: true);
var newHashedEntity = new FileCacheEntity(fileCache.Hash, fileCache.PrefixedFilePath + "." + ext, DateTime.UtcNow.Ticks.ToString(CultureInfo.InvariantCulture));
@@ -764,6 +802,33 @@ public sealed class FileCacheManager : IHostedService
}
}
private void RemoveDerivedCacheFiles(string hash)
{
var cacheFolder = _configService.Current.CacheFolder;
if (string.IsNullOrWhiteSpace(cacheFolder))
{
return;
}
TryDeleteDerivedCacheFile(Path.Combine(cacheFolder, "downscaled", $"{hash}.tex"));
TryDeleteDerivedCacheFile(Path.Combine(cacheFolder, "decimated", $"{hash}.mdl"));
}
private void TryDeleteDerivedCacheFile(string path)
{
try
{
if (File.Exists(path))
{
File.Delete(path);
}
}
catch (Exception ex)
{
_logger.LogTrace(ex, "Failed to delete derived cache file {path}", path);
}
}
private void AddHashedFile(FileCacheEntity fileCache)
{
var normalizedPath = NormalizePrefixedPathKey(fileCache.PrefixedFilePath);
@@ -877,6 +942,83 @@ public sealed class FileCacheManager : IHostedService
}, token).ConfigureAwait(false);
}
private async Task EnforceCacheLimitAsync(long maxBytes, CancellationToken token)
{
if (string.IsNullOrWhiteSpace(CacheFolder) || maxBytes <= 0) return;
await _evictSemaphore.WaitAsync(token).ConfigureAwait(false);
try
{
Directory.CreateDirectory(CacheFolder);
foreach (var tmp in Directory.EnumerateFiles(CacheFolder, "*" + _compressedCacheExtension + ".tmp"))
{
try { File.Delete(tmp); } catch { /* ignore */ }
}
var files = Directory.EnumerateFiles(CacheFolder, "*" + _compressedCacheExtension, SearchOption.TopDirectoryOnly)
.Select(p => new FileInfo(p))
.Where(fi => fi.Exists)
.OrderBy(fi => fi.LastWriteTimeUtc)
.ToList();
long total = files.Sum(f => f.Length);
if (total <= maxBytes) return;
foreach (var fi in files)
{
token.ThrowIfCancellationRequested();
if (total <= maxBytes) break;
var hash = Path.GetFileNameWithoutExtension(fi.Name);
try
{
var len = fi.Length;
fi.Delete();
total -= len;
_sizeCache.TryRemove(hash, out _);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to evict cache file {file}", fi.FullName);
}
}
}
finally
{
_evictSemaphore.Release();
}
}
private static long GiBToBytes(double gib)
{
if (double.IsNaN(gib) || double.IsInfinity(gib) || gib <= 0)
return 0;
var bytes = gib * 1024d * 1024d * 1024d;
if (bytes >= long.MaxValue) return long.MaxValue;
return (long)Math.Round(bytes, MidpointRounding.AwayFromZero);
}
private void CleanupOrphanCompressedCache()
{
if (string.IsNullOrWhiteSpace(CacheFolder) || !Directory.Exists(CacheFolder))
return;
foreach (var path in Directory.EnumerateFiles(CacheFolder, "*" + _compressedCacheExtension))
{
var hash = Path.GetFileNameWithoutExtension(path);
if (!_fileCaches.ContainsKey(hash))
{
try { File.Delete(path); }
catch (Exception ex) { _logger.LogWarning(ex, "Failed deleting orphan {file}", path); }
}
}
}
public async Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Starting FileCacheManager");
@@ -1060,6 +1202,8 @@ public sealed class FileCacheManager : IHostedService
{
await WriteOutFullCsvAsync(cancellationToken).ConfigureAwait(false);
}
CleanupOrphanCompressedCache();
}
_logger.LogInformation("Started FileCacheManager");