Queue File compacting to let workers download as priority, Offload decompression task

LightlessSync/Interop/InteropModel/Enums/ResidentialAetheryteKind.cs

@@ -1,11 +0,0 @@
-namespace Lifestream.Enums;
-
-public enum ResidentialAetheryteKind
-{
-    None = -1,
-    Uldah = 9,
-    Gridania = 2,
-    Limsa = 8,
-    Foundation = 70,
-    Kugane = 111,
-}

LightlessSync/Interop/InteropModel/GlobalModels.cs

@@ -1 +0,0 @@
-global using AddressBookEntryTuple = (string Name, int World, int City, int Ward, int PropertyType, int Plot, int Apartment, bool ApartmentSubdivision, bool AliasEnabled, string Alias);

LightlessSync/Interop/Ipc/IpcCallerLifestream.cs

@@ -1,129 +0,0 @@
-using Dalamud.Plugin;
-using Dalamud.Plugin.Ipc;
-using Lifestream.Enums;
-using LightlessSync.Interop.Ipc.Framework;
-using LightlessSync.Services.Mediator;
-using Microsoft.Extensions.Logging;
-
-
-namespace LightlessSync.Interop.Ipc;
-
-public sealed class IpcCallerLifestream : IpcServiceBase
-{
-    private static readonly IpcServiceDescriptor LifestreamDescriptor = new("Lifestream", "Lifestream", new Version(0, 0, 0, 0));
-
-    private readonly ICallGateSubscriber<string, object> _executeLifestreamCommand;
-    private readonly ICallGateSubscriber<AddressBookEntryTuple, bool> _isHere;
-    private readonly ICallGateSubscriber<AddressBookEntryTuple, object> _goToHousingAddress;
-    private readonly ICallGateSubscriber<bool> _isBusy;
-    private readonly ICallGateSubscriber<object> _abort;
-    private readonly ICallGateSubscriber<string, bool> _changeWorld;
-    private readonly ICallGateSubscriber<uint, bool> _changeWorldById;
-    private readonly ICallGateSubscriber<string, bool> _aetheryteTeleport;
-    private readonly ICallGateSubscriber<uint, bool> _aetheryteTeleportById;
-    private readonly ICallGateSubscriber<bool> _canChangeInstance;
-    private readonly ICallGateSubscriber<int> _getCurrentInstance;
-    private readonly ICallGateSubscriber<int> _getNumberOfInstances;
-    private readonly ICallGateSubscriber<int, object> _changeInstance;
-    private readonly ICallGateSubscriber<(ResidentialAetheryteKind, int, int)> _getCurrentPlotInfo;
-
-    public IpcCallerLifestream(IDalamudPluginInterface pi, LightlessMediator lightlessMediator, ILogger<IpcCallerLifestream> logger)
-        : base(logger, lightlessMediator, pi, LifestreamDescriptor)
-    {
-        _executeLifestreamCommand = pi.GetIpcSubscriber<string, object>("Lifestream.ExecuteCommand");
-        _isHere = pi.GetIpcSubscriber<AddressBookEntryTuple, bool>("Lifestream.IsHere");
-        _goToHousingAddress = pi.GetIpcSubscriber<AddressBookEntryTuple, object>("Lifestream.GoToHousingAddress");
-        _isBusy = pi.GetIpcSubscriber<bool>("Lifestream.IsBusy");
-        _abort = pi.GetIpcSubscriber<object>("Lifestream.Abort");
-        _changeWorld = pi.GetIpcSubscriber<string, bool>("Lifestream.ChangeWorld");
-        _changeWorldById = pi.GetIpcSubscriber<uint, bool>("Lifestream.ChangeWorldById");
-        _aetheryteTeleport = pi.GetIpcSubscriber<string, bool>("Lifestream.AetheryteTeleport");
-        _aetheryteTeleportById = pi.GetIpcSubscriber<uint, bool>("Lifestream.AetheryteTeleportById");
-        _canChangeInstance = pi.GetIpcSubscriber<bool>("Lifestream.CanChangeInstance");
-        _getCurrentInstance = pi.GetIpcSubscriber<int>("Lifestream.GetCurrentInstance");
-        _getNumberOfInstances = pi.GetIpcSubscriber<int>("Lifestream.GetNumberOfInstances");
-        _changeInstance = pi.GetIpcSubscriber<int, object>("Lifestream.ChangeInstance");
-        _getCurrentPlotInfo = pi.GetIpcSubscriber<(ResidentialAetheryteKind, int, int)>("Lifestream.GetCurrentPlotInfo");
-        CheckAPI();
-    }
-
-    public void ExecuteLifestreamCommand(string command)
-    {
-        if (!APIAvailable) return;
-        _executeLifestreamCommand.InvokeAction(command);
-    }
-
-    public bool IsHere(AddressBookEntryTuple entry)
-    {
-        if (!APIAvailable) return false;
-        return _isHere.InvokeFunc(entry);
-    }
-
-    public void GoToHousingAddress(AddressBookEntryTuple entry)
-    {
-        if (!APIAvailable) return;
-        _goToHousingAddress.InvokeAction(entry);
-    }
-
-    public bool IsBusy()
-    {
-        if (!APIAvailable) return false;
-        return _isBusy.InvokeFunc();
-    }
-
-    public void Abort()
-    {
-        if (!APIAvailable) return;
-        _abort.InvokeAction();
-    }
-
-    public bool ChangeWorld(string worldName)
-    {
-        if (!APIAvailable) return false;
-        return _changeWorld.InvokeFunc(worldName);
-    }
-
-    public bool AetheryteTeleport(string aetheryteName)
-    {
-        if (!APIAvailable) return false;
-        return _aetheryteTeleport.InvokeFunc(aetheryteName);
-    }
-
-    public bool ChangeWorldById(uint worldId)
-    {
-        if (!APIAvailable) return false;
-        return _changeWorldById.InvokeFunc(worldId);
-    }
-
-    public bool AetheryteTeleportById(uint aetheryteId)
-    {
-        if (!APIAvailable) return false;
-        return _aetheryteTeleportById.InvokeFunc(aetheryteId);
-    }
-
-    public bool CanChangeInstance()
-    {
-        if (!APIAvailable) return false;
-        return _canChangeInstance.InvokeFunc();
-    }
-    public int GetCurrentInstance()
-    {
-        if (!APIAvailable) return -1;
-        return _getCurrentInstance.InvokeFunc();
-    }
-    public int GetNumberOfInstances()
-    {
-        if (!APIAvailable) return -1;
-        return _getNumberOfInstances.InvokeFunc();
-    }
-    public void ChangeInstance(int instanceNumber)
-    {
-        if (!APIAvailable) return;
-        _changeInstance.InvokeAction(instanceNumber);
-    }
-    public (ResidentialAetheryteKind, int, int)? GetCurrentPlotInfo()
-    {
-        if (!APIAvailable) return (ResidentialAetheryteKind.None, -1, -1);
-        return _getCurrentPlotInfo.InvokeFunc();
-    }
-}

LightlessSync/Interop/Ipc/IpcManager.cs

@@ -7,8 +7,7 @@ public sealed partial class IpcManager : DisposableMediatorSubscriberBase
 {
     public IpcManager(ILogger<IpcManager> logger, LightlessMediator mediator,
         IpcCallerPenumbra penumbraIpc, IpcCallerGlamourer glamourerIpc, IpcCallerCustomize customizeIpc, IpcCallerHeels heelsIpc,
-        IpcCallerHonorific honorificIpc, IpcCallerMoodles moodlesIpc, IpcCallerPetNames ipcCallerPetNames, IpcCallerBrio ipcCallerBrio,
+        IpcCallerHonorific honorificIpc, IpcCallerMoodles moodlesIpc, IpcCallerPetNames ipcCallerPetNames, IpcCallerBrio ipcCallerBrio) : base(logger, mediator)
-        IpcCallerLifestream ipcCallerLifestream) : base(logger, mediator)
     {
         CustomizePlus = customizeIpc;
         Heels = heelsIpc;
@@ -18,7 +17,6 @@ public sealed partial class IpcManager : DisposableMediatorSubscriberBase
         Moodles = moodlesIpc;
         PetNames = ipcCallerPetNames;
         Brio = ipcCallerBrio;
-        Lifestream = ipcCallerLifestream;
 
         if (Initialized)
         {
@@ -46,8 +44,8 @@ public sealed partial class IpcManager : DisposableMediatorSubscriberBase
     public IpcCallerPenumbra Penumbra { get; }
     public IpcCallerMoodles Moodles { get; }
     public IpcCallerPetNames PetNames { get; }
+
     public IpcCallerBrio Brio { get; }
-    public IpcCallerLifestream Lifestream { get; }
 
     private void PeriodicApiStateCheck()
     {
@@ -60,6 +58,5 @@ public sealed partial class IpcManager : DisposableMediatorSubscriberBase
         Moodles.CheckAPI();
         PetNames.CheckAPI();
         Brio.CheckAPI();
-        Lifestream.CheckAPI();
     }
 }

LightlessSync/Plugin.cs

@@ -373,11 +373,6 @@ public sealed class Plugin : IDalamudPlugin
             sp.GetRequiredService<DalamudUtilService>(),
             sp.GetRequiredService<LightlessMediator>()));
 
-        services.AddSingleton(sp => new IpcCallerLifestream(
-            pluginInterface,
-            sp.GetRequiredService<LightlessMediator>(),
-            sp.GetRequiredService<ILogger<IpcCallerLifestream>>()));
-
         services.AddSingleton(sp => new IpcManager(
             sp.GetRequiredService<ILogger<IpcManager>>(),
             sp.GetRequiredService<LightlessMediator>(),
@@ -388,9 +383,7 @@ public sealed class Plugin : IDalamudPlugin
             sp.GetRequiredService<IpcCallerHonorific>(),
             sp.GetRequiredService<IpcCallerMoodles>(),
             sp.GetRequiredService<IpcCallerPetNames>(),
-            sp.GetRequiredService<IpcCallerBrio>(),
+            sp.GetRequiredService<IpcCallerBrio>()));
-            sp.GetRequiredService<IpcCallerLifestream>()
-            ));
 
         // Notifications / HTTP
         services.AddSingleton(sp => new NotificationService(

LightlessSync/UI/SettingsUi.cs

@@ -5,7 +5,6 @@ using Dalamud.Interface.Colors;
 using Dalamud.Interface.Utility;
 using Dalamud.Interface.Utility.Raii;
 using Dalamud.Utility;
-using Lifestream.Enums;
 using LightlessSync.API.Data;
 using LightlessSync.API.Data.Comparer;
 using LightlessSync.API.Data.Enum;
@@ -1244,6 +1243,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
 
             ImGui.TreePop();
         }
+#endif
         if (_uiShared.IconTextButton(FontAwesomeIcon.Copy, "[DEBUG] Copy Last created Character Data to clipboard"))
         {
             if (LastCreatedCharacterData != null)
@@ -1259,39 +1259,6 @@ public class SettingsUi : WindowMediatorSubscriberBase
 
         UiSharedService.AttachToolTip("Use this when reporting mods being rejected from the server.");
 
-        if (_uiShared.IconTextButton(FontAwesomeIcon.Home, "Teleport to Limsa [LIFESTREAM TEST]") && _ipcManager.Lifestream.APIAvailable)
-        {
-            _ipcManager.Lifestream.ExecuteLifestreamCommand("limsa");
-        }
-
-        if (_uiShared.IconTextButton(FontAwesomeIcon.Home, "Teleport to JoyHouse [LIFESTREAM TEST]") && _ipcManager.Lifestream.APIAvailable)
-        {
-            var twintania = _dalamudUtilService.WorldData.Value
-               .FirstOrDefault(kvp => kvp.Value.Equals("Twintania", StringComparison.OrdinalIgnoreCase));
-
-            int ward = 29;
-            int plot = 7;
-
-            AddressBookEntryTuple addressEntry = (
-                Name: "",
-                World: (int)twintania.Key,
-                City: (int)ResidentialAetheryteKind.Kugane,
-                Ward: ward,
-                PropertyType: 0,
-                Plot: plot,
-                Apartment: 1,
-                ApartmentSubdivision: false,
-                AliasEnabled: false,
-                Alias: ""
-            );
-
-            _logger.LogInformation("going to: {address}", addressEntry);
-
-            _ipcManager.Lifestream.GoToHousingAddress(addressEntry);
-        }
-#endif
-
-
         _uiShared.DrawCombo("Log Level", Enum.GetValues<LogLevel>(), (l) => l.ToString(), (l) =>
         {
             _configService.Current.LogLevel = l;

LightlessSync/UI/UISharedService.cs

@@ -79,7 +79,6 @@ public partial class UiSharedService : DisposableMediatorSubscriberBase
     private readonly Dictionary<string, DateTime> _oauthTokenExpiry = [];
     private bool _penumbraExists = false;
     private bool _petNamesExists = false;
-    private bool _lifestreamExists = false;
     private int _serverSelectionIndex = -1;
     public UiSharedService(ILogger<UiSharedService> logger, IpcManager ipcManager, ApiController apiController,
         CacheMonitor cacheMonitor, FileDialogManager fileDialogManager,
@@ -113,7 +112,6 @@ public partial class UiSharedService : DisposableMediatorSubscriberBase
             _moodlesExists = _ipcManager.Moodles.APIAvailable;
             _petNamesExists = _ipcManager.PetNames.APIAvailable;
             _brioExists = _ipcManager.Brio.APIAvailable;
-            _lifestreamExists = _ipcManager.Lifestream.APIAvailable;
         });
 
         UidFont = _pluginInterface.UiBuilder.FontAtlas.NewDelegateFontHandle(e =>
@@ -1107,10 +1105,6 @@ public partial class UiSharedService : DisposableMediatorSubscriberBase
         ColorText("Brio", GetBoolColor(_brioExists));
         AttachToolTip(BuildPluginTooltip("Brio", _brioExists, _ipcManager.Brio.State));
 
-        ImGui.SameLine();
-        ColorText("Lifestream", GetBoolColor(_lifestreamExists));
-        AttachToolTip(BuildPluginTooltip("Lifestream", _lifestreamExists, _ipcManager.Lifestream.State));
-
         if (!_penumbraExists || !_glamourerExists)
         {
             ImGui.TextColored(ImGuiColors.DalamudRed, "You need to install both Penumbra and Glamourer and keep them up to date to use Lightless Sync.");

LightlessSync/WebAPI/Files/FileDownloadManager.cs

@@ -30,6 +30,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
     private readonly ConcurrentDictionary<ThrottledStream, byte> _activeDownloadStreams;
     private readonly SemaphoreSlim _decompressGate =
     new(Math.Max(1, Environment.ProcessorCount / 2), Math.Max(1, Environment.ProcessorCount / 2));
+    
+    private readonly ConcurrentQueue<string> _deferredCompressionQueue = new();
 
     private volatile bool _disableDirectDownloads;
     private int _consecutiveDirectDownloadFailures;
@@ -556,7 +558,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
 
                         if (len == 0)
                         {
-                            await _fileCompactor.WriteAllBytesAsync(filePath, Array.Empty<byte>(), ct).ConfigureAwait(false);
+                            await File.WriteAllBytesAsync(filePath, Array.Empty<byte>(), ct).ConfigureAwait(false);
                             PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale);
                             continue;
                         }
@@ -567,17 +569,21 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                         await _decompressGate.WaitAsync(ct).ConfigureAwait(false);
                         try
                         {
-                            var sw = System.Diagnostics.Stopwatch.StartNew();
+                            // offload CPU-intensive decompression to threadpool to free up worker
+                            await Task.Run(async () =>
+                            {
+                                var sw = System.Diagnostics.Stopwatch.StartNew();
 
-                            // decompress
+                                // decompress
-                            var decompressed = LZ4Wrapper.Unwrap(compressed);
+                                var decompressed = LZ4Wrapper.Unwrap(compressed);
 
-                            Logger.LogTrace("{dlName}: Unwrap {fileHash} took {ms}ms (compressed {c} bytes, decompressed {d} bytes)",
+                                Logger.LogTrace("{dlName}: Unwrap {fileHash} took {ms}ms (compressed {c} bytes, decompressed {d} bytes)",
-                                downloadLabel, fileHash, sw.ElapsedMilliseconds, compressed.Length, decompressed?.Length ?? -1);
+                                    downloadLabel, fileHash, sw.ElapsedMilliseconds, compressed.Length, decompressed?.Length ?? -1);
 
-                            // write to file
+                                // write to file without compacting during download
-                            await _fileCompactor.WriteAllBytesAsync(filePath, decompressed, ct).ConfigureAwait(false);
+                                await File.WriteAllBytesAsync(filePath, decompressed, ct).ConfigureAwait(false);
-                            PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale);
+                                PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale);
+                            }, ct).ConfigureAwait(false);
                         }
                         finally
                         {
@@ -752,8 +758,16 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
         if (gameObjectHandler is not null)
             Mediator.Publish(new DownloadStartedMessage(gameObjectHandler, _downloadStatus));
 
+        // work based on cpu count and slots
+        var coreCount = Environment.ProcessorCount;
+        var baseWorkers = Math.Min(slots, coreCount);
+
+        // only add buffer if decompression has capacity AND we have cores to spare
+        var availableDecompressSlots = _decompressGate.CurrentCount;
+        var extraWorkers = (availableDecompressSlots > 0 && coreCount >= 6) ? 2 : 0;
+
         // allow some extra workers so downloads can continue while earlier items decompress.
-        var workerDop = Math.Clamp(slots * 2, 2, 16);
+        var workerDop = Math.Clamp(baseWorkers + extraWorkers, 2, coreCount);
 
         // batch downloads
         Task batchTask = batchChunks.Length == 0
@@ -769,6 +783,9 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
 
         await Task.WhenAll(batchTask, directTask).ConfigureAwait(false);
 
+        // process deferred compressions after all downloads complete
+        await ProcessDeferredCompressionsAsync(ct).ConfigureAwait(false);
+
         Logger.LogDebug("Download end: {id}", objectName);
         ClearDownload();
     }
@@ -873,7 +890,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
             byte[] compressedBytes = await File.ReadAllBytesAsync(tempFilename, ct).ConfigureAwait(false);
             var decompressedBytes = LZ4Wrapper.Unwrap(compressedBytes);
 
-            await _fileCompactor.WriteAllBytesAsync(finalFilename, decompressedBytes, ct).ConfigureAwait(false);
+            await File.WriteAllBytesAsync(finalFilename, decompressedBytes, ct).ConfigureAwait(false);
             PersistFileToStorage(directDownload.Hash, finalFilename, repl.GamePath, skipDownscale);
 
             MarkTransferredFiles(directDownload.DirectDownloadUrl!, 1);
@@ -1001,6 +1018,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
         fi.LastAccessTime = DateTime.Today;
         fi.LastWriteTime = RandomDayInThePast().Invoke();
 
+        // queue file for deferred compression instead of compressing immediately
+        if (_configService.Current.UseCompactor)
+            _deferredCompressionQueue.Enqueue(filePath);
+
         try
         {
             var entry = _fileDbManager.CreateCacheEntry(filePath);
@@ -1026,6 +1047,52 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
 
     private static IProgress<long> CreateInlineProgress(Action<long> callback) => new InlineProgress(callback);
 
+    private async Task ProcessDeferredCompressionsAsync(CancellationToken ct)
+    {
+        if (_deferredCompressionQueue.IsEmpty)
+            return;
+
+        var filesToCompress = new List<string>();
+        while (_deferredCompressionQueue.TryDequeue(out var filePath))
+        {
+            if (File.Exists(filePath))
+                filesToCompress.Add(filePath);
+        }
+
+        if (filesToCompress.Count == 0)
+            return;
+
+        Logger.LogDebug("Starting deferred compression of {count} files", filesToCompress.Count);
+
+        var compressionWorkers = Math.Clamp(Environment.ProcessorCount / 4, 2, 4);
+
+        await Parallel.ForEachAsync(filesToCompress, 
+            new ParallelOptions 
+            { 
+                MaxDegreeOfParallelism = compressionWorkers, 
+                CancellationToken = ct 
+            },
+            async (filePath, token) =>
+            {
+                try 
+                { 
+                    await Task.Yield();
+                    if (_configService.Current.UseCompactor && File.Exists(filePath))
+                    {
+                        var bytes = await File.ReadAllBytesAsync(filePath, token).ConfigureAwait(false);
+                        await _fileCompactor.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false);
+                        Logger.LogTrace("Compressed file: {filePath}", filePath);
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Logger.LogWarning(ex, "Failed to compress file: {filePath}", filePath);
+                }
+            }).ConfigureAwait(false);
+
+        Logger.LogDebug("Completed deferred compression of {count} files", filesToCompress.Count);
+    }
+
     private sealed class InlineProgress : IProgress<long>
     {
         private readonly Action<long> _callback;

LightlessSync/packages.lock.json

@@ -76,19 +76,6 @@
           "Microsoft.AspNetCore.SignalR.Common": "10.0.1"
         }
       },
-      "Microsoft.Extensions.Caching.Memory": {
-        "type": "Direct",
-        "requested": "[10.0.1, )",
-        "resolved": "10.0.1",
-        "contentHash": "NxqSP0Ky4dZ5ybszdZCqs1X2C70s+dXflqhYBUh/vhcQVTIooNCXIYnLVbafoAFGZMs51d9+rHxveXs0ZC3SQQ==",
-        "dependencies": {
-          "Microsoft.Extensions.Caching.Abstractions": "10.0.1",
-          "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.1",
-          "Microsoft.Extensions.Logging.Abstractions": "10.0.1",
-          "Microsoft.Extensions.Options": "10.0.1",
-          "Microsoft.Extensions.Primitives": "10.0.1"
-        }
-      },
       "Microsoft.Extensions.Hosting": {
         "type": "Direct",
         "requested": "[10.0.1, )",
@@ -246,14 +233,6 @@
           "Microsoft.AspNetCore.SignalR.Common": "10.0.1"
         }
       },
-      "Microsoft.Extensions.Caching.Abstractions": {
-        "type": "Transitive",
-        "resolved": "10.0.1",
-        "contentHash": "Vb1vVAQDxHpXVdL9fpOX2BzeV7bbhzG4pAcIKRauRl0/VfkE8mq0f+fYC+gWICh3dlzTZInJ/cTeBS2MgU/XvQ==",
-        "dependencies": {
-          "Microsoft.Extensions.Primitives": "10.0.1"
-        }
-      },
       "Microsoft.Extensions.Configuration": {
         "type": "Transitive",
         "resolved": "10.0.1",
@@ -639,7 +618,7 @@
           "FlatSharp.Compiler": "[7.9.0, )",
           "FlatSharp.Runtime": "[7.9.0, )",
           "OtterGui": "[1.0.0, )",
-          "Penumbra.Api": "[5.13.1, )",
+          "Penumbra.Api": "[5.13.0, )",
           "Penumbra.String": "[1.0.7, )"
         }
       },