init

LightlessSync/FileCache/FileCacheManager.cs

@@ -27,6 +27,7 @@ public sealed class FileCacheManager : IHostedService
     private readonly Lock _fileWriteLock = new();
     private readonly IpcManager _ipcManager;
     private readonly ILogger<FileCacheManager> _logger;
+    private bool _csvHeaderEnsured;
     public string CacheFolder => _configService.Current.CacheFolder;
 
     public FileCacheManager(ILogger<FileCacheManager> logger, IpcManager ipcManager, LightlessConfigService configService, LightlessMediator lightlessMediator)
@@ -462,6 +463,7 @@ public sealed class FileCacheManager : IHostedService
         string[] existingLines = File.ReadAllLines(_csvPath);
         if (existingLines.Length > 0 && TryParseVersionHeader(existingLines[0], out var existingVersion) && existingVersion == FileCacheVersion)
         {
+            _csvHeaderEnsured = true;
             return;
         }
 
@@ -481,6 +483,18 @@ public sealed class FileCacheManager : IHostedService
         }
 
         File.WriteAllText(_csvPath, rebuilt.ToString());
+        _csvHeaderEnsured = true;
+    }
+
+    private void EnsureCsvHeaderLockedCached()
+    {
+        if (_csvHeaderEnsured)
+        {
+            return;
+        }
+
+        EnsureCsvHeaderLocked();
+        _csvHeaderEnsured = true;
     }
 
     private void BackupUnsupportedCache(string suffix)
@@ -540,10 +554,11 @@ public sealed class FileCacheManager : IHostedService
             if (!File.Exists(_csvPath))
             {
                 File.WriteAllLines(_csvPath, new[] { BuildVersionHeader(), entity.CsvEntry });
+                _csvHeaderEnsured = true;
             }
             else
             {
-                EnsureCsvHeaderLocked();
+                EnsureCsvHeaderLockedCached();
                 File.AppendAllLines(_csvPath, new[] { entity.CsvEntry });
             }
         }

LightlessSync/FileCache/FileCompactor.cs

@@ -2,25 +2,33 @@
 using LightlessSync.Services;
 using Microsoft.Extensions.Logging;
 using System.Runtime.InteropServices;
+using System.Collections.Concurrent;
+using System.Threading;
+using System.Threading.Channels;
+using System.Threading.Tasks;
 
 namespace LightlessSync.FileCache;
 
-public sealed class FileCompactor
+public sealed class FileCompactor : IDisposable
 {
     public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U;
     public const ulong WOF_PROVIDER_FILE = 2UL;
 
     private readonly Dictionary<string, int> _clusterSizes;
-
+    private readonly ConcurrentDictionary<string, byte> _pendingCompactions;
     private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo;
     private readonly ILogger<FileCompactor> _logger;
 
     private readonly LightlessConfigService _lightlessConfigService;
     private readonly DalamudUtilService _dalamudUtilService;
+    private readonly Channel<string> _compactionQueue;
+    private readonly CancellationTokenSource _compactionCts = new();
+    private readonly Task _compactionWorker;
 
     public FileCompactor(ILogger<FileCompactor> logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService)
     {
         _clusterSizes = new(StringComparer.Ordinal);
+        _pendingCompactions = new(StringComparer.OrdinalIgnoreCase);
         _logger = logger;
         _lightlessConfigService = lightlessConfigService;
         _dalamudUtilService = dalamudUtilService;
@@ -29,6 +37,18 @@ public sealed class FileCompactor
             Algorithm = CompressionAlgorithm.XPRESS8K,
             Flags = 0
         };
+
+        _compactionQueue = Channel.CreateUnbounded<string>(new UnboundedChannelOptions
+        {
+            SingleReader = true,
+            SingleWriter = false
+        });
+        _compactionWorker = Task.Factory.StartNew(
+                () => ProcessQueueAsync(_compactionCts.Token),
+                _compactionCts.Token,
+                TaskCreationOptions.LongRunning,
+                TaskScheduler.Default)
+            .Unwrap();
     }
 
     private enum CompressionAlgorithm
@@ -87,7 +107,30 @@ public sealed class FileCompactor
             return;
         }
 
-        CompactFile(filePath);
+        EnqueueCompaction(filePath);
+    }
+
+    public void Dispose()
+    {
+        _compactionQueue.Writer.TryComplete();
+        _compactionCts.Cancel();
+        try
+        {
+            if (!_compactionWorker.Wait(TimeSpan.FromSeconds(5)))
+            {
+                _logger.LogDebug("Compaction worker did not shut down within timeout");
+            }
+        }
+        catch (Exception ex) when (ex is not OperationCanceledException)
+        {
+            _logger.LogDebug(ex, "Error shutting down compaction worker");
+        }
+        finally
+        {
+            _compactionCts.Dispose();
+        }
+
+        GC.SuppressFinalize(this);
     }
 
     [DllImport("kernel32.dll")]
@@ -226,4 +269,67 @@ public sealed class FileCompactor
         public CompressionAlgorithm Algorithm;
         public ulong Flags;
     }
-}
+
+    private void EnqueueCompaction(string filePath)
+    {
+        if (!_pendingCompactions.TryAdd(filePath, 0))
+        {
+            return;
+        }
+
+        if (!_compactionQueue.Writer.TryWrite(filePath))
+        {
+            _pendingCompactions.TryRemove(filePath, out _);
+            _logger.LogDebug("Failed to enqueue compaction job for {file}", filePath);
+        }
+    }
+
+    private async Task ProcessQueueAsync(CancellationToken token)
+    {
+        try
+        {
+            while (await _compactionQueue.Reader.WaitToReadAsync(token).ConfigureAwait(false))
+            {
+                while (_compactionQueue.Reader.TryRead(out var filePath))
+                {
+                    try
+                    {
+                        if (token.IsCancellationRequested)
+                        {
+                            return;
+                        }
+
+                        if (_dalamudUtilService.IsWine || !_lightlessConfigService.Current.UseCompactor)
+                        {
+                            continue;
+                        }
+
+                        if (!File.Exists(filePath))
+                        {
+                            _logger.LogTrace("Skipping compaction for missing file {file}", filePath);
+                            continue;
+                        }
+
+                        CompactFile(filePath);
+                    }
+                    catch (OperationCanceledException)
+                    {
+                        return;
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogWarning(ex, "Error compacting file {file}", filePath);
+                    }
+                    finally
+                    {
+                        _pendingCompactions.TryRemove(filePath, out _);
+                    }
+                }
+            }
+        }
+        catch (OperationCanceledException)
+        {
+            // expected during shutdown
+        }
+    }
+}