Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 56 Wiki Activity

test-abel-cake-changes

Browse Source
This commit is contained in:
cake
2026-01-03 22:45:55 +01:00
parent 6af61451dc 979443d9bb
commit 57a076ae77
57 changed files with 11420 additions and 357 deletions
Download Patch File Download Diff File Expand all files Collapse all files

215
LightlessSync/FileCache/CacheMonitor.cs
View File

@@ -103,6 +103,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
}
record WatcherChange(WatcherChangeTypes ChangeType, string? OldPath = null);
private readonly record struct CacheEvictionCandidate(string FullPath, long Size, DateTime LastAccessTime);
private readonly Dictionary<string, WatcherChange> _watcherChanges = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, WatcherChange> _lightlessChanges = new(StringComparer.OrdinalIgnoreCase);
@@ -441,116 +442,40 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
Logger.LogWarning(ex, "Could not determine drive size for storage folder {folder}", _configService.Current.CacheFolder);
}
var files = Directory.EnumerateFiles(_configService.Current.CacheFolder)
.Select(f => new FileInfo(f))
.OrderBy(f => f.LastAccessTime)
.ToList();
var cacheFolder = _configService.Current.CacheFolder;
var candidates = new List<CacheEvictionCandidate>();
long totalSize = 0;
foreach (var f in files)
{
token.ThrowIfCancellationRequested();
try
{
long size = 0;
if (!isWine)
{
try
{
size = _fileCompactor.GetFileSizeOnDisk(f);
}
catch (Exception ex)
{
Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName);
size = f.Length;
}
}
else
{
size = f.Length;
}
totalSize += size;
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Error getting size for {file}", f.FullName);
}
}
totalSize += AddFolderCandidates(cacheFolder, candidates, token, isWine);
totalSize += AddFolderCandidates(Path.Combine(cacheFolder, "downscaled"), candidates, token, isWine);
totalSize += AddFolderCandidates(Path.Combine(cacheFolder, "decimated"), candidates, token, isWine);
FileCacheSize = totalSize;
if (Directory.Exists(_configService.Current.CacheFolder + "/downscaled"))
{
var filesDownscaled = Directory.EnumerateFiles(_configService.Current.CacheFolder + "/downscaled").Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList();
long totalSizeDownscaled = 0;
foreach (var f in filesDownscaled)
{
token.ThrowIfCancellationRequested();
try
{
long size = 0;
if (!isWine)
{
try
{
size = _fileCompactor.GetFileSizeOnDisk(f);
}
catch (Exception ex)
{
Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName);
size = f.Length;
}
}
else
{
size = f.Length;
}
totalSizeDownscaled += size;
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Error getting size for {file}", f.FullName);
}
}
FileCacheSize = (totalSize + totalSizeDownscaled);
}
else
{
FileCacheSize = totalSize;
}
var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d);
if (FileCacheSize < maxCacheInBytes)
return;
var maxCacheBuffer = maxCacheInBytes * 0.05d;
while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer && files.Count > 0)
candidates.Sort(static (a, b) => a.LastAccessTime.CompareTo(b.LastAccessTime));
var evictionTarget = maxCacheInBytes - (long)maxCacheBuffer;
var index = 0;
while (FileCacheSize > evictionTarget && index < candidates.Count)
{
var oldestFile = files[0];
var oldestFile = candidates[index];
try
{
long fileSize = oldestFile.Length;
File.Delete(oldestFile.FullName);
FileCacheSize -= fileSize;
EvictCacheCandidate(oldestFile, cacheFolder);
FileCacheSize -= oldestFile.Size;
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullName);
Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullPath);
}
files.RemoveAt(0);
index++;
}
}
@@ -559,6 +484,114 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
HaltScanLocks.Clear();
}
private long AddFolderCandidates(string directory, List<CacheEvictionCandidate> candidates, CancellationToken token, bool isWine)
{
if (!Directory.Exists(directory))
{
return 0;
}
long totalSize = 0;
foreach (var path in Directory.EnumerateFiles(directory))
{
token.ThrowIfCancellationRequested();
try
{
var file = new FileInfo(path);
var size = GetFileSizeOnDisk(file, isWine);
totalSize += size;
candidates.Add(new CacheEvictionCandidate(file.FullName, size, file.LastAccessTime));
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Error getting size for {file}", path);
}
}
return totalSize;
}
private long GetFileSizeOnDisk(FileInfo file, bool isWine)
{
if (isWine)
{
return file.Length;
}
try
{
return _fileCompactor.GetFileSizeOnDisk(file);
}
catch (Exception ex)
{
Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", file.FullName);
return file.Length;
}
}
private void EvictCacheCandidate(CacheEvictionCandidate candidate, string cacheFolder)
{
if (TryGetCacheHashAndPrefixedPath(candidate.FullPath, cacheFolder, out var hash, out var prefixedPath))
{
_fileDbManager.RemoveHashedFile(hash, prefixedPath);
}
try
{
if (File.Exists(candidate.FullPath))
{
File.Delete(candidate.FullPath);
}
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Failed to delete old file {file}", candidate.FullPath);
}
}
private static bool TryGetCacheHashAndPrefixedPath(string filePath, string cacheFolder, out string hash, out string prefixedPath)
{
hash = string.Empty;
prefixedPath = string.Empty;
if (string.IsNullOrEmpty(cacheFolder))
{
return false;
}
var fileName = Path.GetFileNameWithoutExtension(filePath);
if (string.IsNullOrEmpty(fileName) || !IsSha1Hash(fileName))
{
return false;
}
var relative = Path.GetRelativePath(cacheFolder, filePath)
.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
var sanitizedRelative = relative.TrimStart(Path.DirectorySeparatorChar);
prefixedPath = Path.Combine(FileCacheManager.CachePrefix, sanitizedRelative);
hash = fileName;
return true;
}
private static bool IsSha1Hash(string value)
{
if (value.Length != 40)
{
return false;
}
foreach (var ch in value)
{
if (!Uri.IsHexDigit(ch))
{
return false;
}
}
return true;
}
public void ResumeScan(string source)
{
if (!HaltScanLocks.ContainsKey(source)) HaltScanLocks[source] = 0;

62
LightlessSync/FileCache/FileCacheManager.cs
View File

@@ -291,6 +291,26 @@ public sealed class FileCacheManager : IHostedService
return CreateFileEntity(cacheFolder, CachePrefix, fi);
}
public FileCacheEntity? CreateCacheEntryWithKnownHash(string path, string hash)
{
if (string.IsNullOrWhiteSpace(hash))
{
return CreateCacheEntry(path);
}
FileInfo fi = new(path);
if (!fi.Exists) return null;
_logger.LogTrace("Creating cache entry for {path} using provided hash", path);
var cacheFolder = _configService.Current.CacheFolder;
if (string.IsNullOrEmpty(cacheFolder)) return null;
if (!TryBuildPrefixedPath(fi.FullName, cacheFolder, CachePrefix, out var prefixedPath, out _))
{
return null;
}
return CreateFileCacheEntity(fi, prefixedPath, hash);
}
public FileCacheEntity? CreateFileEntry(string path)
{
FileInfo fi = new(path);
@@ -573,9 +593,10 @@ public sealed class FileCacheManager : IHostedService
}
}
public void RemoveHashedFile(string hash, string prefixedFilePath)
public void RemoveHashedFile(string hash, string prefixedFilePath, bool removeDerivedFiles = true)
{
var normalizedPath = NormalizePrefixedPathKey(prefixedFilePath);
var removedHash = false;
if (_fileCaches.TryGetValue(hash, out var caches))
{
@@ -588,11 +609,16 @@ public sealed class FileCacheManager : IHostedService
if (caches.IsEmpty)
{
_fileCaches.TryRemove(hash, out _);
removedHash = _fileCaches.TryRemove(hash, out _);
}
}
_fileCachesByPrefixedPath.TryRemove(normalizedPath, out _);
if (removeDerivedFiles && removedHash)
{
RemoveDerivedCacheFiles(hash);
}
}
public void UpdateHashedFile(FileCacheEntity fileCache, bool computeProperties = true)
@@ -608,7 +634,8 @@ public sealed class FileCacheManager : IHostedService
fileCache.Hash = Crypto.ComputeFileHash(fileCache.ResolvedFilepath, Crypto.HashAlgo.Sha1);
fileCache.LastModifiedDateTicks = fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
}
RemoveHashedFile(oldHash, prefixedPath);
var removeDerivedFiles = !string.Equals(oldHash, fileCache.Hash, StringComparison.OrdinalIgnoreCase);
RemoveHashedFile(oldHash, prefixedPath, removeDerivedFiles);
AddHashedFile(fileCache);
}
@@ -758,7 +785,7 @@ public sealed class FileCacheManager : IHostedService
{
try
{
RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath);
RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath, removeDerivedFiles: false);
var extensionPath = fileCache.ResolvedFilepath.ToUpper(CultureInfo.InvariantCulture) + "." + ext;
File.Move(fileCache.ResolvedFilepath, extensionPath, overwrite: true);
var newHashedEntity = new FileCacheEntity(fileCache.Hash, fileCache.PrefixedFilePath + "." + ext, DateTime.UtcNow.Ticks.ToString(CultureInfo.InvariantCulture));
@@ -775,6 +802,33 @@ public sealed class FileCacheManager : IHostedService
}
}
private void RemoveDerivedCacheFiles(string hash)
{
var cacheFolder = _configService.Current.CacheFolder;
if (string.IsNullOrWhiteSpace(cacheFolder))
{
return;
}
TryDeleteDerivedCacheFile(Path.Combine(cacheFolder, "downscaled", $"{hash}.tex"));
TryDeleteDerivedCacheFile(Path.Combine(cacheFolder, "decimated", $"{hash}.mdl"));
}
private void TryDeleteDerivedCacheFile(string path)
{
try
{
if (File.Exists(path))
{
File.Delete(path);
}
}
catch (Exception ex)
{
_logger.LogTrace(ex, "Failed to delete derived cache file {path}", path);
}
}
private void AddHashedFile(FileCacheEntity fileCache)
{
var normalizedPath = NormalizePrefixedPathKey(fileCache.PrefixedFilePath);