1.12.4

Co-authored-by: cake <cake@noreply.git.lightless-sync.org>
Co-authored-by: cake <admin@cakeandbanana.nl>
Co-authored-by: azyges <229218900+azyges@users.noreply.github.com>
Co-authored-by: choco <choco@patat.nl>
Co-authored-by: choco <choco@noreply.git.lightless-sync.org>
Co-authored-by: defnotken <itsdefnotken@gmail.com>
Reviewed-on: #73

LightlessSync/FileCache/CacheMonitor.cs

@@ -115,6 +115,8 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
 
     public bool StorageisNTFS { get; private set; } = false;
 
+    public bool StorageIsBtrfs { get ; private set; } = false;
+
     public void StartLightlessWatcher(string? lightlessPath)
     {
         LightlessWatcher?.Dispose();
@@ -124,10 +126,19 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
             Logger.LogWarning("Lightless file path is not set, cannot start the FSW for Lightless.");
             return;
         }
+        var fsType = FileSystemHelper.GetFilesystemType(_configService.Current.CacheFolder, _dalamudUtil.IsWine);
 
-        DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
-        StorageisNTFS = string.Equals("NTFS", di.DriveFormat, StringComparison.OrdinalIgnoreCase);
-        Logger.LogInformation("Lightless Storage is on NTFS drive: {isNtfs}", StorageisNTFS);
+        if (fsType == FileSystemHelper.FilesystemType.NTFS)
+        {
+            StorageisNTFS = true;
+            Logger.LogInformation("Lightless Storage is on NTFS drive: {isNtfs}", StorageisNTFS);
+        }
+
+        if (fsType == FileSystemHelper.FilesystemType.Btrfs)
+        {
+            StorageIsBtrfs = true;
+            Logger.LogInformation("Lightless Storage is on BTRFS drive: {isBtrfs}", StorageIsBtrfs);
+        }
 
         Logger.LogDebug("Initializing Lightless FSW on {path}", lightlessPath);
         LightlessWatcher = new()
@@ -392,51 +403,94 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
 
     public void RecalculateFileCacheSize(CancellationToken token)
     {
-        if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder))
+        if (string.IsNullOrEmpty(_configService.Current.CacheFolder) ||
+            !Directory.Exists(_configService.Current.CacheFolder))
         {
             FileCacheSize = 0;
             return;
         }
 
         FileCacheSize = -1;
-        DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
+        bool isWine = _dalamudUtil?.IsWine ?? false;
+
         try
         {
-            FileCacheDriveFree = di.AvailableFreeSpace;
+            var drive = DriveInfo.GetDrives()
+                .FirstOrDefault(d => _configService.Current.CacheFolder
+                    .StartsWith(d.Name, StringComparison.OrdinalIgnoreCase));
+
+            if (drive != null)
+                FileCacheDriveFree = drive.AvailableFreeSpace;
         }
         catch (Exception ex)
         {
-            Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder);
+            Logger.LogWarning(ex, "Could not determine drive size for storage folder {folder}", _configService.Current.CacheFolder);
         }
 
-        var files = Directory.EnumerateFiles(_configService.Current.CacheFolder).Select(f => new FileInfo(f))
-            .OrderBy(f => f.LastAccessTime).ToList();
-        FileCacheSize = files
-            .Sum(f =>
-            {
-                token.ThrowIfCancellationRequested();
+        var files = Directory.EnumerateFiles(_configService.Current.CacheFolder)
+            .Select(f => new FileInfo(f))
+            .OrderBy(f => f.LastAccessTime)
+            .ToList();
 
-                try
+        long totalSize = 0;
+
+        foreach (var f in files)
+        {
+            token.ThrowIfCancellationRequested();
+
+            try
+            {
+                long size = 0;
+
+                if (!isWine)
                 {
-                    return _fileCompactor.GetFileSizeOnDisk(f, StorageisNTFS);
+                    try
+                    {
+                        size = _fileCompactor.GetFileSizeOnDisk(f);
+                    }
+                    catch (Exception ex)
+                    {
+                        Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName);
+                        size = f.Length;
+                    }
                 }
-                catch
+                else
                 {
-                    return 0;
+                    size = f.Length;
                 }
-            });
+
+                totalSize += size;
+            }
+            catch (Exception ex)
+            {
+                Logger.LogTrace(ex, "Error getting size for {file}", f.FullName);
+            }
+        }
+
+        FileCacheSize = totalSize;
 
         var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d);
-
-        if (FileCacheSize < maxCacheInBytes) return;
+        if (FileCacheSize < maxCacheInBytes)
+            return;
 
         var maxCacheBuffer = maxCacheInBytes * 0.05d;
-        while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer)
+
+        while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer && files.Count > 0)
         {
             var oldestFile = files[0];
-            FileCacheSize -= _fileCompactor.GetFileSizeOnDisk(oldestFile);
-            File.Delete(oldestFile.FullName);
-            files.Remove(oldestFile);
+
+            try
+            {
+                long fileSize = oldestFile.Length;
+                File.Delete(oldestFile.FullName);
+                FileCacheSize -= fileSize;
+            }
+            catch (Exception ex)
+            {
+                Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullName);
+            }
+
+            files.RemoveAt(0);
         }
     }
 
@@ -644,44 +698,44 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
 
         if (ct.IsCancellationRequested) return;
 
-        // scan new files
-        if (allScannedFiles.Any(c => !c.Value))
+        var newFiles = allScannedFiles.Where(c => !c.Value).Select(c => c.Key).ToList();
+        foreach (var cachePath in newFiles)
         {
-            Parallel.ForEach(allScannedFiles.Where(c => !c.Value).Select(c => c.Key),
-                new ParallelOptions()
-                {
-                    MaxDegreeOfParallelism = threadCount,
-                    CancellationToken = ct
-                }, (cachePath) =>
-                {
-                    if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null)
-                    {
-                        Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}", _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null);
-                        return;
-                    }
+            if (ct.IsCancellationRequested) break;
+            ProcessOne(cachePath);
+            Interlocked.Increment(ref _currentFileProgress);
+        }
 
-                    if (ct.IsCancellationRequested) return;
+        Logger.LogTrace("Scanner added {count} new files to db", newFiles.Count);
 
-                    if (!_ipcManager.Penumbra.APIAvailable)
-                    {
-                        Logger.LogWarning("Penumbra not available");
-                        return;
-                    }
+        void ProcessOne(string? cachePath)
+        {
+            if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null)
+            {
+                Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}",
+                    _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null);
+                return;
+            }
 
-                    try
-                    {
-                        var entry = _fileDbManager.CreateFileEntry(cachePath);
-                        if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath);
-                    }
-                    catch (Exception ex)
-                    {
-                        Logger.LogWarning(ex, "Failed adding {file}", cachePath);
-                    }
+            if (!_ipcManager.Penumbra.APIAvailable)
+            {
+                Logger.LogWarning("Penumbra not available");
+                return;
+            }
 
-                    Interlocked.Increment(ref _currentFileProgress);
-                });
-
-            Logger.LogTrace("Scanner added {notScanned} new files to db", allScannedFiles.Count(c => !c.Value));
+            try
+            {
+                var entry = _fileDbManager.CreateFileEntry(cachePath);
+                if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath);
+            }
+            catch (IOException ioex)
+            {
+                Logger.LogDebug(ioex, "File busy or locked: {file}", cachePath);
+            }
+            catch (Exception ex)
+            {
+                Logger.LogWarning(ex, "Failed adding {file}", cachePath);
+            }
         }
 
         Logger.LogDebug("Scan complete");

LightlessSync/FileCache/FileCacheManager.cs

@@ -203,42 +203,72 @@ public sealed class FileCacheManager : IHostedService
         return output;
     }
 
-    public Task<List<FileCacheEntity>> ValidateLocalIntegrity(IProgress<(int, int, FileCacheEntity)> progress, CancellationToken cancellationToken)
+    public async Task<List<FileCacheEntity>> ValidateLocalIntegrity(IProgress<(int completed, int total, FileCacheEntity current)> progress, CancellationToken cancellationToken)
     {
         _lightlessMediator.Publish(new HaltScanMessage(nameof(ValidateLocalIntegrity)));
         _logger.LogInformation("Validating local storage");
-        var cacheEntries = _fileCaches.Values.SelectMany(v => v.Values.Where(e => e != null)).Where(v => v.IsCacheEntry).ToList();
-        List<FileCacheEntity> brokenEntities = [];
-        int i = 0;
-        foreach (var fileCache in cacheEntries)
+
+        var cacheEntries = _fileCaches.Values
+            .SelectMany(v => v.Values)
+            .Where(v => v.IsCacheEntry)
+            .ToList();
+
+        int total = cacheEntries.Count;
+        int processed = 0;
+        var brokenEntities = new ConcurrentBag<FileCacheEntity>();
+
+        _logger.LogInformation("Checking {count} cache entries...", total);
+
+        await Parallel.ForEachAsync(cacheEntries, new ParallelOptions
+        {
+            MaxDegreeOfParallelism = Environment.ProcessorCount,
+            CancellationToken = cancellationToken
+        },
+        async (fileCache, token) =>
         {
-            if (cancellationToken.IsCancellationRequested) break;
-
-            _logger.LogInformation("Validating {file}", fileCache.ResolvedFilepath);
-
-            progress.Report((i, cacheEntries.Count, fileCache));
-            i++;
-            if (!File.Exists(fileCache.ResolvedFilepath))
-            {
-                brokenEntities.Add(fileCache);
-                continue;
-            }
-
             try
             {
-                var computedHash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
+                int current = Interlocked.Increment(ref processed);
+                if (current % 10 == 0)
+                    progress.Report((current, total, fileCache));
+
+                if (!File.Exists(fileCache.ResolvedFilepath))
+                {
+                    brokenEntities.Add(fileCache);
+                    return;
+                }
+
+                string computedHash;
+                try
+                {
+                    computedHash = await Crypto.GetFileHashAsync(fileCache.ResolvedFilepath, token).ConfigureAwait(false);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogWarning(ex, "Error hashing {file}", fileCache.ResolvedFilepath);
+                    brokenEntities.Add(fileCache);
+                    return;
+                }
+
                 if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal))
                 {
-                    _logger.LogInformation("Failed to validate {file}, got hash {computedHash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
+                    _logger.LogInformation(
+                        "Hash mismatch: {file} (got {computedHash}, expected {expected})",
+                        fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
+
                     brokenEntities.Add(fileCache);
                 }
             }
-            catch (Exception e)
+            catch (OperationCanceledException)
             {
-                _logger.LogWarning(e, "Error during validation of {file}", fileCache.ResolvedFilepath);
+                _logger.LogError("Validation got cancelled for {file}", fileCache.ResolvedFilepath);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Unexpected error validating {file}", fileCache.ResolvedFilepath);
                 brokenEntities.Add(fileCache);
             }
-        }
+        }).ConfigureAwait(false);
 
         foreach (var brokenEntity in brokenEntities)
         {
@@ -250,12 +280,14 @@ public sealed class FileCacheManager : IHostedService
             }
             catch (Exception ex)
             {
-                _logger.LogWarning(ex, "Could not delete {file}", brokenEntity.ResolvedFilepath);
+                _logger.LogWarning(ex, "Failed to delete invalid cache file {file}", brokenEntity.ResolvedFilepath);
             }
         }
 
         _lightlessMediator.Publish(new ResumeScanMessage(nameof(ValidateLocalIntegrity)));
-        return Task.FromResult(brokenEntities);
+        _logger.LogInformation("Validation complete. Found {count} invalid entries.", brokenEntities.Count);
+
+        return [.. brokenEntities];
     }
 
     public string GetCacheFilePath(string hash, string extension)