readjust cache clean up and add keep originals setting for models

LightlessSync/FileCache/CacheMonitor.cs

@@ -103,6 +103,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
     }
 
     record WatcherChange(WatcherChangeTypes ChangeType, string? OldPath = null);
+    private readonly record struct CacheEvictionCandidate(string FullPath, long Size, DateTime LastAccessTime);
     private readonly Dictionary<string, WatcherChange> _watcherChanges = new(StringComparer.OrdinalIgnoreCase);
     private readonly Dictionary<string, WatcherChange> _lightlessChanges = new(StringComparer.OrdinalIgnoreCase);
 
@@ -441,76 +442,40 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
             Logger.LogWarning(ex, "Could not determine drive size for storage folder {folder}", _configService.Current.CacheFolder);
         }
 
-        var files = Directory.EnumerateFiles(_configService.Current.CacheFolder)
-            .Select(f => new FileInfo(f))
-            .OrderBy(f => f.LastAccessTime)
-            .ToList();
-
+        var cacheFolder = _configService.Current.CacheFolder;
+        var candidates = new List<CacheEvictionCandidate>();
         long totalSize = 0;
+        totalSize += AddFolderCandidates(cacheFolder, candidates, token, isWine);
+        totalSize += AddFolderCandidates(Path.Combine(cacheFolder, "downscaled"), candidates, token, isWine);
+        totalSize += AddFolderCandidates(Path.Combine(cacheFolder, "decimated"), candidates, token, isWine);
 
-        foreach (var f in files)
-        {
-            token.ThrowIfCancellationRequested();
-
-            try
-            {
-                long size = 0;
-
-                if (!isWine)
-                {
-                    try
-                    {
-                        size = _fileCompactor.GetFileSizeOnDisk(f);
-                    }
-                    catch (Exception ex)
-                    {
-                        Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName);
-                        size = f.Length;
-                    }
-                }
-                else
-                {
-                    size = f.Length;
-                }
-
-                totalSize += size;
-            }
-            catch (Exception ex)
-            {
-                Logger.LogTrace(ex, "Error getting size for {file}", f.FullName);
-            }
-        }
-
-        long totalSizeExtras = 0;
         FileCacheSize = totalSize;
 
-        totalSizeExtras += GetFolderSize(Path.Combine(_configService.Current.CacheFolder, "downscaled"), token, isWine);
-        totalSizeExtras += GetFolderSize(Path.Combine(_configService.Current.CacheFolder, "decimated"), token, isWine);
-
-        FileCacheSize = totalSize + totalSizeExtras;
-
         var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d);
         if (FileCacheSize < maxCacheInBytes)
             return;
 
         var maxCacheBuffer = maxCacheInBytes * 0.05d;
 
-        while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer && files.Count > 0)
+        candidates.Sort(static (a, b) => a.LastAccessTime.CompareTo(b.LastAccessTime));
+
+        var evictionTarget = maxCacheInBytes - (long)maxCacheBuffer;
+        var index = 0;
+        while (FileCacheSize > evictionTarget && index < candidates.Count)
         {
-            var oldestFile = files[0];
+            var oldestFile = candidates[index];
 
             try
             {
-                long fileSize = oldestFile.Length;
-                File.Delete(oldestFile.FullName);
-                FileCacheSize -= fileSize;
+                EvictCacheCandidate(oldestFile, cacheFolder);
+                FileCacheSize -= oldestFile.Size;
             }
             catch (Exception ex)
             {
-                Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullName);
+                Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullPath);
             }
 
-            files.RemoveAt(0);
+            index++;
         }
     }
 
@@ -519,54 +484,114 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         HaltScanLocks.Clear();
     }
 
-    private long GetFolderSize(string directory, CancellationToken token, bool isWine)
+    private long AddFolderCandidates(string directory, List<CacheEvictionCandidate> candidates, CancellationToken token, bool isWine)
     {
         if (!Directory.Exists(directory))
         {
             return 0;
         }
 
-        var files = Directory.EnumerateFiles(directory)
-            .Select(f => new FileInfo(f))
-            .OrderBy(f => f.LastAccessTime)
-            .ToList();
-
         long totalSize = 0;
-        foreach (var file in files)
+        foreach (var path in Directory.EnumerateFiles(directory))
         {
             token.ThrowIfCancellationRequested();
 
             try
             {
-                long size;
-                if (!isWine)
-                {
-                    try
-                    {
-                        size = _fileCompactor.GetFileSizeOnDisk(file);
-                    }
-                    catch (Exception ex)
-                    {
-                        Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", file.FullName);
-                        size = file.Length;
-                    }
-                }
-                else
-                {
-                    size = file.Length;
-                }
-
+                var file = new FileInfo(path);
+                var size = GetFileSizeOnDisk(file, isWine);
                 totalSize += size;
+                candidates.Add(new CacheEvictionCandidate(file.FullName, size, file.LastAccessTime));
             }
             catch (Exception ex)
             {
-                Logger.LogTrace(ex, "Error getting size for {file}", file.FullName);
+                Logger.LogTrace(ex, "Error getting size for {file}", path);
             }
         }
 
         return totalSize;
     }
 
+    private long GetFileSizeOnDisk(FileInfo file, bool isWine)
+    {
+        if (isWine)
+        {
+            return file.Length;
+        }
+
+        try
+        {
+            return _fileCompactor.GetFileSizeOnDisk(file);
+        }
+        catch (Exception ex)
+        {
+            Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", file.FullName);
+            return file.Length;
+        }
+    }
+
+    private void EvictCacheCandidate(CacheEvictionCandidate candidate, string cacheFolder)
+    {
+        if (TryGetCacheHashAndPrefixedPath(candidate.FullPath, cacheFolder, out var hash, out var prefixedPath))
+        {
+            _fileDbManager.RemoveHashedFile(hash, prefixedPath);
+        }
+
+        try
+        {
+            if (File.Exists(candidate.FullPath))
+            {
+                File.Delete(candidate.FullPath);
+            }
+        }
+        catch (Exception ex)
+        {
+            Logger.LogTrace(ex, "Failed to delete old file {file}", candidate.FullPath);
+        }
+    }
+
+    private static bool TryGetCacheHashAndPrefixedPath(string filePath, string cacheFolder, out string hash, out string prefixedPath)
+    {
+        hash = string.Empty;
+        prefixedPath = string.Empty;
+
+        if (string.IsNullOrEmpty(cacheFolder))
+        {
+            return false;
+        }
+
+        var fileName = Path.GetFileNameWithoutExtension(filePath);
+        if (string.IsNullOrEmpty(fileName) || !IsSha1Hash(fileName))
+        {
+            return false;
+        }
+
+        var relative = Path.GetRelativePath(cacheFolder, filePath)
+            .Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
+        var sanitizedRelative = relative.TrimStart(Path.DirectorySeparatorChar);
+        prefixedPath = Path.Combine(FileCacheManager.CachePrefix, sanitizedRelative);
+        hash = fileName;
+        return true;
+    }
+
+    private static bool IsSha1Hash(string value)
+    {
+        if (value.Length != 40)
+        {
+            return false;
+        }
+
+        foreach (var ch in value)
+        {
+            if (!Uri.IsHexDigit(ch))
+            {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
     public void ResumeScan(string source)
     {
         if (!HaltScanLocks.ContainsKey(source)) HaltScanLocks[source] = 0;