Added seperate collections for other states, moved clean up of penumbra collection out of config. Safe read of ptr on process, fixed notfications on popup and notifications with flags.

LightlessSync/FileCache/CacheMonitor.cs

@@ -21,6 +21,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
     private long _currentFileProgress = 0;
     private CancellationTokenSource _scanCancellationTokenSource = new();
     private readonly CancellationTokenSource _periodicCalculationTokenSource = new();
+    private readonly SemaphoreSlim _dbGate = new(1, 1);
     public static readonly IImmutableList<string> AllowedFileExtensions = [".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".pbd", ".scd", ".skp", ".shpk", ".kdb"];
     private static readonly HashSet<string> AllowedFileExtensionSet = new(AllowedFileExtensions, StringComparer.OrdinalIgnoreCase);
 
@@ -68,6 +69,9 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         {
             Logger.LogInformation("Starting Periodic Storage Directory Calculation Task");
             var token = _periodicCalculationTokenSource.Token;
+            while (IsHalted() && !token.IsCancellationRequested)
+                await Task.Delay(TimeSpan.FromSeconds(5), token).ConfigureAwait(false);
+
             while (!token.IsCancellationRequested)
             {
                 try
@@ -91,6 +95,9 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
     public long CurrentFileProgress => _currentFileProgress;
     public long FileCacheSize { get; set; }
     public long FileCacheDriveFree { get; set; }
+
+    private int _haltCount;
+    private bool IsHalted() => Volatile.Read(ref _haltCount) > 0;
     public ConcurrentDictionary<string, int> HaltScanLocks { get; set; } = new(StringComparer.Ordinal);
     public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
     public long TotalFiles { get; private set; }
@@ -98,14 +105,36 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
 
     public void HaltScan(string source)
     {
-        if (!HaltScanLocks.ContainsKey(source)) HaltScanLocks[source] = 0;
-        HaltScanLocks[source]++;
+        HaltScanLocks.AddOrUpdate(source, 1, (_, v) => v + 1);
+        Interlocked.Increment(ref _haltCount);
     }
 
     record WatcherChange(WatcherChangeTypes ChangeType, string? OldPath = null);
     private readonly record struct CacheEvictionCandidate(string FullPath, long Size, DateTime LastAccessTime);
-    private readonly Dictionary<string, WatcherChange> _watcherChanges = new(StringComparer.OrdinalIgnoreCase);
-    private readonly Dictionary<string, WatcherChange> _lightlessChanges = new(StringComparer.OrdinalIgnoreCase);
+    private readonly object _penumbraGate = new();
+    private Dictionary<string, WatcherChange> _watcherChanges = new(StringComparer.OrdinalIgnoreCase);
+
+    private readonly object _lightlessGate = new();
+    private Dictionary<string, WatcherChange> _lightlessChanges = new(StringComparer.OrdinalIgnoreCase);
+    private Dictionary<string, WatcherChange> DrainPenumbraChanges()
+    {
+        lock (_penumbraGate)
+        {
+            var snapshot = _watcherChanges;
+            _watcherChanges = new(StringComparer.OrdinalIgnoreCase);
+            return snapshot;
+        }
+    }
+
+    private Dictionary<string, WatcherChange> DrainLightlessChanges()
+    {
+        lock (_lightlessGate)
+        {
+            var snapshot = _lightlessChanges;
+            _lightlessChanges = new(StringComparer.OrdinalIgnoreCase);
+            return snapshot;
+        }
+    }
 
     public void StopMonitoring()
     {
@@ -168,7 +197,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
 
         if (!HasAllowedExtension(e.FullPath)) return;
 
-        lock (_watcherChanges)
+        lock (_lightlessChanges)
         {
             _lightlessChanges[e.FullPath] = new(e.ChangeType);
         }
@@ -279,67 +308,58 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
 
     private async Task LightlessWatcherExecution()
     {
-        _lightlessFswCts = _lightlessFswCts.CancelRecreate();
+       _lightlessFswCts = _lightlessFswCts.CancelRecreate();
         var token = _lightlessFswCts.Token;
-        var delay = TimeSpan.FromSeconds(5);
-        Dictionary<string, WatcherChange> changes;
-        lock (_lightlessChanges)
-            changes = _lightlessChanges.ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal);
+
         try
         {
-            do
-            {
-                await Task.Delay(delay, token).ConfigureAwait(false);
-            } while (HaltScanLocks.Any(f => f.Value > 0));
-        }
-        catch (TaskCanceledException)
-        {
-            return;
+            await Task.Delay(TimeSpan.FromSeconds(10), token).ConfigureAwait(false);
+            while (IsHalted() && !token.IsCancellationRequested)
+                await Task.Delay(250, token).ConfigureAwait(false);
         }
+        catch (TaskCanceledException) { return; }
 
-        lock (_lightlessChanges)
-        {
-            foreach (var key in changes.Keys)
-            {
-                _lightlessChanges.Remove(key);
-            }
-        }
-
-        HandleChanges(changes);
+        var changes = DrainLightlessChanges();
+        if (changes.Count > 0)
+            _ = HandleChangesAsync(changes, token);
     }
-
-    private void HandleChanges(Dictionary<string, WatcherChange> changes)
+    private async Task HandleChangesAsync(Dictionary<string, WatcherChange> changes, CancellationToken token)
     {
-        lock (_fileDbManager)
+        await _dbGate.WaitAsync(token).ConfigureAwait(false);
+        try
         {
-            var deletedEntries = changes.Where(c => c.Value.ChangeType == WatcherChangeTypes.Deleted).Select(c => c.Key);
-            var renamedEntries = changes.Where(c => c.Value.ChangeType == WatcherChangeTypes.Renamed);
-            var remainingEntries = changes.Where(c => c.Value.ChangeType != WatcherChangeTypes.Deleted).Select(c => c.Key);
+            var deleted = changes.Where(c => c.Value.ChangeType == WatcherChangeTypes.Deleted).Select(c => c.Key);
+            var renamed = changes.Where(c => c.Value.ChangeType == WatcherChangeTypes.Renamed);
+            var remaining = changes.Where(c => c.Value.ChangeType != WatcherChangeTypes.Deleted).Select(c => c.Key);
 
-            foreach (var entry in deletedEntries)
+            foreach (var entry in deleted)
             {
                 Logger.LogDebug("FSW Change: Deletion - {val}", entry);
             }
 
-            foreach (var entry in renamedEntries)
+            foreach (var entry in renamed)
             {
                 Logger.LogDebug("FSW Change: Renamed - {oldVal} => {val}", entry.Value.OldPath, entry.Key);
             }
 
-            foreach (var entry in remainingEntries)
+            foreach (var entry in remaining)
             {
                 Logger.LogDebug("FSW Change: Creation or Change - {val}", entry);
             }
 
-            var allChanges = deletedEntries
-                .Concat(renamedEntries.Select(c => c.Value.OldPath!))
-                .Concat(renamedEntries.Select(c => c.Key))
-                .Concat(remainingEntries)
+            var allChanges = deleted
+                .Concat(renamed.Select(c => c.Value.OldPath!))
+                .Concat(renamed.Select(c => c.Key))
+                .Concat(remaining)
+                .Distinct(StringComparer.OrdinalIgnoreCase)
                 .ToArray();
 
             _ = _fileDbManager.GetFileCachesByPaths(allChanges);
-
-            _fileDbManager.WriteOutFullCsv();
+            await _fileDbManager.WriteOutFullCsvAsync(token).ConfigureAwait(false);
+        }
+        finally
+        {
+            _dbGate.Release();
         }
     }
 
@@ -347,77 +367,97 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
     {
         _penumbraFswCts = _penumbraFswCts.CancelRecreate();
         var token = _penumbraFswCts.Token;
-        Dictionary<string, WatcherChange> changes;
-        lock (_watcherChanges)
-            changes = _watcherChanges.ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal);
-        var delay = TimeSpan.FromSeconds(10);
+
         try
         {
-            do
-            {
-                await Task.Delay(delay, token).ConfigureAwait(false);
-            } while (HaltScanLocks.Any(f => f.Value > 0));
-        }
-        catch (TaskCanceledException)
-        {
-            return;
+            await Task.Delay(TimeSpan.FromSeconds(10), token).ConfigureAwait(false);
+            while (IsHalted() && !token.IsCancellationRequested)
+                await Task.Delay(250, token).ConfigureAwait(false);
         }
+        catch (TaskCanceledException) { return; }
 
-        lock (_watcherChanges)
-        {
-            foreach (var key in changes.Keys)
-            {
-                _watcherChanges.Remove(key);
-            }
-        }
-
-        HandleChanges(changes);
+        var changes = DrainPenumbraChanges();
+        if (changes.Count > 0)
+            _ = HandleChangesAsync(changes, token);
     }
 
     public void InvokeScan()
     {
         TotalFiles = 0;
-        _currentFileProgress = 0;
+        TotalFilesStorage = 0;
+        Interlocked.Exchange(ref _currentFileProgress, 0);
+
         _scanCancellationTokenSource = _scanCancellationTokenSource?.CancelRecreate() ?? new CancellationTokenSource();
         var token = _scanCancellationTokenSource.Token;
+
         _ = Task.Run(async () =>
         {
-            Logger.LogDebug("Starting Full File Scan");
-            TotalFiles = 0;
-            _currentFileProgress = 0;
-            while (_dalamudUtil.IsOnFrameworkThread)
-            {
-                Logger.LogWarning("Scanner is on framework, waiting for leaving thread before continuing");
-                await Task.Delay(250, token).ConfigureAwait(false);
-            }
+            TaskCompletionSource scanTcs = new(TaskCreationOptions.RunContinuationsAsynchronously);
 
-            Thread scanThread = new(() =>
+            try
             {
-                try
+                Logger.LogDebug("Starting Full File Scan");
+
+                while (IsHalted() && !token.IsCancellationRequested)
                 {
-                    _performanceCollector.LogPerformance(this, $"FullFileScan", () => FullFileScan(token));
+                    Logger.LogDebug("Scan is halted, waiting...");
+                    await Task.Delay(250, token).ConfigureAwait(false);
                 }
-                catch (Exception ex)
+
+                var scanThread = new Thread(() =>
                 {
-                    Logger.LogError(ex, "Error during Full File Scan");
-                }
-            })
-            {
-                Priority = ThreadPriority.Lowest,
-                IsBackground = true
-            };
-            scanThread.Start();
-            while (scanThread.IsAlive)
-            {
-                await Task.Delay(250, token).ConfigureAwait(false);
+                    try
+                    {
+                        token.ThrowIfCancellationRequested();
+
+                        _performanceCollector.LogPerformance(this, $"FullFileScan",
+                            () => FullFileScan(token));
+
+                        scanTcs.TrySetResult();
+                    }
+                    catch (OperationCanceledException)
+                    {
+                        scanTcs.TrySetCanceled(token);
+                    }
+                    catch (Exception ex)
+                    {
+                        Logger.LogError(ex, "Error during Full File Scan");
+                        scanTcs.TrySetException(ex);
+                    }
+                })
+                {
+                    Priority = ThreadPriority.Lowest,
+                    IsBackground = true,
+                    Name = "LightlessSync.FullFileScan"
+                };
+
+                scanThread.Start();
+
+                using var _ = token.Register(() => scanTcs.TrySetCanceled(token));
+
+                await scanTcs.Task.ConfigureAwait(false);
+            }
+            catch (TaskCanceledException)
+            {
+                Logger.LogInformation("Full File Scan was canceled.");
+            }
+            catch (Exception ex)
+            {
+                Logger.LogError(ex, "Unexpected error in InvokeScan task");
+            }
+            finally
+            {
+                TotalFiles = 0;
+                TotalFilesStorage = 0;
+                Interlocked.Exchange(ref _currentFileProgress, 0);
             }
-            TotalFiles = 0;
-            _currentFileProgress = 0;
         }, token);
     }
 
     public void RecalculateFileCacheSize(CancellationToken token)
     {
+        if (IsHalted()) return;
+
         if (string.IsNullOrEmpty(_configService.Current.CacheFolder) ||
             !Directory.Exists(_configService.Current.CacheFolder))
         {
@@ -594,10 +634,20 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
 
     public void ResumeScan(string source)
     {
-        if (!HaltScanLocks.ContainsKey(source)) HaltScanLocks[source] = 0;
+        int delta = 0;
 
-        HaltScanLocks[source]--;
-        if (HaltScanLocks[source] < 0) HaltScanLocks[source] = 0;
+        HaltScanLocks.AddOrUpdate(source,
+            addValueFactory: _ => 0,
+            updateValueFactory: (_, v) =>
+            {
+                ArgumentException.ThrowIfNullOrEmpty(_);
+                if (v <= 0) return 0;
+                delta = 1;
+                return v - 1;
+            });
+
+        if (delta == 1)
+            Interlocked.Decrement(ref _haltCount);
     }
 
     protected override void Dispose(bool disposing)
@@ -621,201 +671,81 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
     private void FullFileScan(CancellationToken ct)
     {
         TotalFiles = 1;
+        _currentFileProgress = 0;
+
         var penumbraDir = _ipcManager.Penumbra.ModDirectory;
-        bool penDirExists = true;
-        bool cacheDirExists = true;
+        var cacheFolder = _configService.Current.CacheFolder;
+
         if (string.IsNullOrEmpty(penumbraDir) || !Directory.Exists(penumbraDir))
         {
-            penDirExists = false;
             Logger.LogWarning("Penumbra directory is not set or does not exist.");
+            return;
         }
-        if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder))
+
+        if (string.IsNullOrEmpty(cacheFolder) || !Directory.Exists(cacheFolder))
         {
-            cacheDirExists = false;
             Logger.LogWarning("Lightless Cache directory is not set or does not exist.");
-        }
-        if (!penDirExists || !cacheDirExists)
-        {
             return;
         }
 
-        var previousThreadPriority = Thread.CurrentThread.Priority;
+        var prevPriority = Thread.CurrentThread.Priority;
         Thread.CurrentThread.Priority = ThreadPriority.Lowest;
-        Logger.LogDebug("Getting files from {penumbra} and {storage}", penumbraDir, _configService.Current.CacheFolder);
 
-        Dictionary<string, string[]> penumbraFiles = new(StringComparer.Ordinal);
-        foreach (var folder in Directory.EnumerateDirectories(penumbraDir!))
+        try
         {
-            try
+            Logger.LogDebug("Getting files from {penumbra} and {storage}", penumbraDir, cacheFolder);
+
+            var onDiskPaths = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+            static bool IsExcludedPenumbraPath(string path)
+                => path.Contains(@"\bg\", StringComparison.OrdinalIgnoreCase)
+                || path.Contains(@"\bgcommon\", StringComparison.OrdinalIgnoreCase)
+                || path.Contains(@"\ui\", StringComparison.OrdinalIgnoreCase);
+
+            foreach (var folder in Directory.EnumerateDirectories(penumbraDir))
             {
-                penumbraFiles[folder] =
-                [
-                    .. Directory.GetFiles(folder, "*.*", SearchOption.AllDirectories)
-                                            .AsParallel()
-                                            .Where(f => HasAllowedExtension(f)
-                                                && !f.Contains(@"\bg\", StringComparison.OrdinalIgnoreCase)
-                                                && !f.Contains(@"\bgcommon\", StringComparison.OrdinalIgnoreCase)
-                                                && !f.Contains(@"\ui\", StringComparison.OrdinalIgnoreCase)),
-                ];
-            }
-            catch (Exception ex)
-            {
-                Logger.LogWarning(ex, "Could not enumerate path {path}", folder);
-            }
-            Thread.Sleep(50);
-            if (ct.IsCancellationRequested) return;
-        }
+                ct.ThrowIfCancellationRequested();
 
-        var allCacheFiles = Directory.GetFiles(_configService.Current.CacheFolder, "*.*", SearchOption.TopDirectoryOnly)
-                                .AsParallel()
-                                .Where(f =>
-                                {
-                                    var val = f.Split('\\')[^1];
-                                    return val.Length == 40 || (val.Split('.').FirstOrDefault()?.Length ?? 0) == 40;
-                                });
-
-        if (ct.IsCancellationRequested) return;
-
-        var allScannedFiles = (penumbraFiles.SelectMany(k => k.Value))
-            .Concat(allCacheFiles)
-            .Distinct(StringComparer.OrdinalIgnoreCase)
-            .ToDictionary(t => t.ToLowerInvariant(), t => false, StringComparer.OrdinalIgnoreCase);
-
-        TotalFiles = allScannedFiles.Count;
-        Thread.CurrentThread.Priority = previousThreadPriority;
-
-        Thread.Sleep(TimeSpan.FromSeconds(2));
-
-        if (ct.IsCancellationRequested) return;
-
-        // scan files from database
-        var threadCount = Math.Clamp((int)(Environment.ProcessorCount / 2.0f), 2, 8);
-
-        List<FileCacheEntity> entitiesToRemove = [];
-        List<FileCacheEntity> entitiesToUpdate = [];
-        Lock sync = new();
-        Thread[] workerThreads = new Thread[threadCount];
-
-        ConcurrentQueue<FileCacheEntity> fileCaches = new(_fileDbManager.GetAllFileCaches());
-
-        TotalFilesStorage = fileCaches.Count;
-
-        for (int i = 0; i < threadCount; i++)
-        {
-            Logger.LogTrace("Creating Thread {i}", i);
-            workerThreads[i] = new((tcounter) =>
-            {
-                var threadNr = (int)tcounter!;
-                Logger.LogTrace("Spawning Worker Thread {i}", threadNr);
-                while (!ct.IsCancellationRequested && fileCaches.TryDequeue(out var workload))
+                try
                 {
-                    try
+                    foreach (var file in Directory.EnumerateFiles(folder, "*.*", SearchOption.AllDirectories))
                     {
-                        if (ct.IsCancellationRequested) return;
+                        ct.ThrowIfCancellationRequested();
 
-                        if (!_ipcManager.Penumbra.APIAvailable)
-                        {
-                            Logger.LogWarning("Penumbra not available");
-                            return;
-                        }
+                        if (!HasAllowedExtension(file)) continue;
+                        if (IsExcludedPenumbraPath(file)) continue;
 
-                        var validatedCacheResult = _fileDbManager.ValidateFileCacheEntity(workload);
-                        if (validatedCacheResult.State != FileState.RequireDeletion)
-                        {
-                            lock (sync) { allScannedFiles[validatedCacheResult.FileCache.ResolvedFilepath] = true; }
-                        }
-                        if (validatedCacheResult.State == FileState.RequireUpdate)
-                        {
-                            Logger.LogTrace("To update: {path}", validatedCacheResult.FileCache.ResolvedFilepath);
-                            lock (sync) { entitiesToUpdate.Add(validatedCacheResult.FileCache); }
-                        }
-                        else if (validatedCacheResult.State == FileState.RequireDeletion)
-                        {
-                            Logger.LogTrace("To delete: {path}", validatedCacheResult.FileCache.ResolvedFilepath);
-                            lock (sync) { entitiesToRemove.Add(validatedCacheResult.FileCache); }
-                        }
+                        onDiskPaths.Add(file);
                     }
-                    catch (Exception ex)
-                    {
-                        if (workload != null)
-                        {
-                            Logger.LogWarning(ex, "Failed validating {path}", workload.ResolvedFilepath);
-                        }
-                        else
-                        {
-                            Logger.LogWarning(ex, "Failed validating unknown workload");
-                        }
-                    }
-                    Interlocked.Increment(ref _currentFileProgress);
                 }
-
-                Logger.LogTrace("Ending Worker Thread {i}", threadNr);
-            })
-            {
-                Priority = ThreadPriority.Lowest,
-                IsBackground = true
-            };
-            workerThreads[i].Start(i);
-        }
-
-        while (!ct.IsCancellationRequested && workerThreads.Any(u => u.IsAlive))
-        {
-            Thread.Sleep(1000);
-        }
-
-        if (ct.IsCancellationRequested) return;
-
-        Logger.LogTrace("Threads exited");
-
-        if (!_ipcManager.Penumbra.APIAvailable)
-        {
-            Logger.LogWarning("Penumbra not available");
-            return;
-        }
-
-        if (entitiesToUpdate.Count != 0 || entitiesToRemove.Count != 0)
-        {
-            foreach (var entity in entitiesToUpdate)
-            {
-                _fileDbManager.UpdateHashedFile(entity);
+                catch (Exception ex)
+                {
+                    Logger.LogWarning(ex, "Could not enumerate path {path}", folder);
+                }
             }
 
-            foreach (var entity in entitiesToRemove)
+            foreach (var file in Directory.EnumerateFiles(cacheFolder, "*.*", SearchOption.TopDirectoryOnly))
             {
-                _fileDbManager.RemoveHashedFile(entity.Hash, entity.PrefixedFilePath);
+                ct.ThrowIfCancellationRequested();
+
+                var name = Path.GetFileName(file);
+                var stem = Path.GetFileNameWithoutExtension(file);
+
+                if (name.Length == 40 || stem.Length == 40)
+                    onDiskPaths.Add(file);
             }
 
-            _fileDbManager.WriteOutFullCsv();
-        }
+            var threadCount = Math.Clamp((int)(Environment.ProcessorCount / 2.0f), 2, 8);
 
-        Logger.LogTrace("Scanner validated existing db files");
+            var fileCacheList = _fileDbManager.GetAllFileCaches();
+            var fileCaches = new ConcurrentQueue<FileCacheEntity>(fileCacheList);
 
-        if (!_ipcManager.Penumbra.APIAvailable)
-        {
-            Logger.LogWarning("Penumbra not available");
-            return;
-        }
+            TotalFilesStorage = fileCaches.Count;
+            TotalFiles = onDiskPaths.Count + TotalFilesStorage;
 
-        if (ct.IsCancellationRequested) return;
-
-        var newFiles = allScannedFiles.Where(c => !c.Value).Select(c => c.Key).ToList();
-        foreach (var cachePath in newFiles)
-        {
-            if (ct.IsCancellationRequested) break;
-            ProcessOne(cachePath);
-            Interlocked.Increment(ref _currentFileProgress);
-        }
-
-        Logger.LogTrace("Scanner added {count} new files to db", newFiles.Count);
-
-        void ProcessOne(string? cachePath)
-        {
-            if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null)
-            {
-                Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}",
-                    _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null);
-                return;
-            }
+            var validOrPresentInDb = new ConcurrentDictionary<string, byte>(StringComparer.OrdinalIgnoreCase);
+            var entitiesToUpdate = new ConcurrentBag<FileCacheEntity>();
+            var entitiesToRemove = new ConcurrentBag<FileCacheEntity>();
 
             if (!_ipcManager.Penumbra.APIAvailable)
             {
@@ -823,33 +753,161 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
                 return;
             }
 
-            try
+            Thread[] workerThreads = new Thread[threadCount];
+            for (int i = 0; i < threadCount; i++)
             {
-                var entry = _fileDbManager.CreateFileEntry(cachePath);
-                if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath);
+                workerThreads[i] = new Thread(tcounter =>
+                {
+                    var threadNr = (int)tcounter!;
+                    Logger.LogTrace("Spawning Worker Thread {i}", threadNr);
+
+                    while (!ct.IsCancellationRequested && fileCaches.TryDequeue(out var workload))
+                    {
+                        try
+                        {
+                            if (ct.IsCancellationRequested) break;
+
+                            if (!_ipcManager.Penumbra.APIAvailable)
+                                break;
+
+                            var validated = _fileDbManager.ValidateFileCacheEntity(workload);
+
+                            if (validated.State != FileState.RequireDeletion)
+                            {
+                                validOrPresentInDb.TryAdd(validated.FileCache.ResolvedFilepath, 0);
+                            }
+
+                            if (validated.State == FileState.RequireUpdate)
+                            {
+                                Logger.LogTrace("To update: {path}", validated.FileCache.ResolvedFilepath);
+                                entitiesToUpdate.Add(validated.FileCache);
+                            }
+                            else if (validated.State == FileState.RequireDeletion)
+                            {
+                                Logger.LogTrace("To delete: {path}", validated.FileCache.ResolvedFilepath);
+                                entitiesToRemove.Add(validated.FileCache);
+                            }
+                        }
+                        catch (Exception ex)
+                        {
+                            if (workload != null)
+                                Logger.LogWarning(ex, "Failed validating {path}", workload.ResolvedFilepath);
+                            else
+                                Logger.LogWarning(ex, "Failed validating unknown workload");
+                        }
+                        finally
+                        {
+                            Interlocked.Increment(ref _currentFileProgress);
+                        }
+                    }
+
+                    Logger.LogTrace("Ending Worker Thread {i}", threadNr);
+                })
+                {
+                    Priority = ThreadPriority.Lowest,
+                    IsBackground = true
+                };
+
+                workerThreads[i].Start(i);
             }
-            catch (IOException ioex)
+
+            while (!ct.IsCancellationRequested && workerThreads.Any(t => t.IsAlive))
             {
-                Logger.LogDebug(ioex, "File busy or locked: {file}", cachePath);
+                ct.WaitHandle.WaitOne(250);
             }
-            catch (Exception ex)
+
+            if (ct.IsCancellationRequested) return;
+
+            Logger.LogTrace("Scanner validated existing db files");
+
+            if (!_ipcManager.Penumbra.APIAvailable)
             {
-                Logger.LogWarning(ex, "Failed adding {file}", cachePath);
+                Logger.LogWarning("Penumbra not available");
+                return;
+            }
+
+            var didMutateDb = false;
+
+            foreach (var entity in entitiesToUpdate)
+            {
+                didMutateDb = true;
+                _fileDbManager.UpdateHashedFile(entity);
+            }
+
+            foreach (var entity in entitiesToRemove)
+            {
+                didMutateDb = true;
+                _fileDbManager.RemoveHashedFile(entity.Hash, entity.PrefixedFilePath);
+            }
+
+            if (didMutateDb)
+                _fileDbManager.WriteOutFullCsv();
+
+            if (ct.IsCancellationRequested) return;
+
+            var newFiles = onDiskPaths.Where(p => !validOrPresentInDb.ContainsKey(p)).ToList();
+
+            foreach (var path in newFiles)
+            {
+                if (ct.IsCancellationRequested) break;
+                ProcessOne(path);
+                Interlocked.Increment(ref _currentFileProgress);
+            }
+
+            Logger.LogTrace("Scanner added {count} new files to db", newFiles.Count);
+
+            void ProcessOne(string? filePath)
+            {
+                if (filePath == null)
+                    return;
+
+                if (!_ipcManager.Penumbra.APIAvailable)
+                {
+                    Logger.LogWarning("Penumbra not available");
+                    return;
+                }
+
+                try
+                {
+                    var entry = _fileDbManager.CreateFileEntry(filePath);
+                    if (entry == null)
+                        _ = _fileDbManager.CreateCacheEntry(filePath);
+                }
+                catch (IOException ioex)
+                {
+                    Logger.LogDebug(ioex, "File busy or locked: {file}", filePath);
+                }
+                catch (Exception ex)
+                {
+                    Logger.LogWarning(ex, "Failed adding {file}", filePath);
+                }
+            }
+
+            Logger.LogDebug("Scan complete");
+
+            TotalFiles = 0;
+            _currentFileProgress = 0;
+
+            if (!_configService.Current.InitialScanComplete)
+            {
+                _configService.Current.InitialScanComplete = true;
+                _configService.Save();
+
+                StartLightlessWatcher(cacheFolder);
+                StartPenumbraWatcher(penumbraDir);
             }
         }
-
-        Logger.LogDebug("Scan complete");
-        TotalFiles = 0;
-        _currentFileProgress = 0;
-        entitiesToRemove.Clear();
-        allScannedFiles.Clear();
-
-        if (!_configService.Current.InitialScanComplete)
+        catch (OperationCanceledException)
         {
-            _configService.Current.InitialScanComplete = true;
-            _configService.Save();
-            StartLightlessWatcher(_configService.Current.CacheFolder);
-            StartPenumbraWatcher(penumbraDir);
+            // normal cancellation
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Error during Full File Scan");
+        }
+        finally
+        {
+            Thread.CurrentThread.Priority = prevPriority;
         }
     }
 }

LightlessSync/FileCache/FileCompactor.cs

@@ -213,7 +213,7 @@ public sealed partial class FileCompactor : IDisposable
     /// <param name="bytes">Bytes that have to be written</param>
     /// <param name="token">Cancellation Token for interupts</param>
     /// <returns>Writing Task</returns>
-    public async Task WriteAllBytesAsync(string filePath, byte[] bytes, CancellationToken token)
+    public async Task WriteAllBytesAsync(string filePath, byte[] bytes, CancellationToken token, bool enqueueCompaction = true)
     {
         var dir = Path.GetDirectoryName(filePath);
         if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir))
@@ -221,6 +221,12 @@ public sealed partial class FileCompactor : IDisposable
 
         await File.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false);
 
+        if (enqueueCompaction && _lightlessConfigService.Current.UseCompactor)
+            EnqueueCompaction(filePath);
+    }
+
+    public void RequestCompaction(string filePath)
+    {
         if (_lightlessConfigService.Current.UseCompactor)
             EnqueueCompaction(filePath);
     }

LightlessSync/FileCache/TransientResourceManager.cs

@@ -321,7 +321,7 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
         {
             foreach (var handler in _playerRelatedPointers)
             {
-                var address = (nint)handler.Address;
+                var address =   handler.Address;
                 if (address != nint.Zero)
                 {
                     tempMap[address] = handler;