Testing decompression stuff

Reviewed-on: #121

LightlessSync/LightlessSync.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <Authors></Authors>
     <Company></Company>
-    <Version>2.0.2</Version>
+    <Version>2.0.1.71</Version>
     <Description></Description>
     <Copyright></Copyright>
     <PackageProjectUrl>https://github.com/Light-Public-Syncshells/LightlessClient</PackageProjectUrl>

LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs

@@ -1423,7 +1423,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
     private Task _visibilityGraceTask;
 
     private async Task DownloadAndApplyCharacterAsync(Guid applicationBase, CharacterData charaData, Dictionary<ObjectKind, HashSet<PlayerChanges>> updatedData,
-        bool updateModdedPaths, bool updateManip, Dictionary<(string GamePath, string? Hash), string>? cachedModdedPaths, CancellationToken downloadToken)
+     bool updateModdedPaths, bool updateManip, Dictionary<(string GamePath, string? Hash), string>? cachedModdedPaths, CancellationToken downloadToken)
     {
         var concurrencyLease = await _pairProcessingLimiter.AcquireAsync(downloadToken).ConfigureAwait(false);
         try
@@ -1577,24 +1577,37 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
                 RecordFailure("Handler not available for application", "HandlerUnavailable");
                 return;
             }
+            _applicationCancellationTokenSource = _applicationCancellationTokenSource.CancelRecreate() ?? new CancellationTokenSource();
 
-            var appToken = _applicationCancellationTokenSource?.Token;
+            if (_applicationTask != null && !_applicationTask.IsCompleted)
-            while ((!_applicationTask?.IsCompleted ?? false)
-                   && !downloadToken.IsCancellationRequested
-                   && (!appToken?.IsCancellationRequested ?? false))
             {
-                Logger.LogDebug("[BASE-{appBase}] Waiting for current data application (Id: {id}) for player ({handler}) to finish", applicationBase, _applicationId, PlayerName);
+                Logger.LogDebug("[BASE-{appBase}] Cancelling current data application (Id: {id}) for player ({handler})", applicationBase, _applicationId, PlayerName);
-                await Task.Delay(250).ConfigureAwait(false);
+
+                var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+                var combinedCts = CancellationTokenSource.CreateLinkedTokenSource(downloadToken, timeoutCts.Token);
+
+                try
+                {
+                    await _applicationTask.WaitAsync(combinedCts.Token).ConfigureAwait(false);
+                }
+                catch (OperationCanceledException)
+                {
+                    Logger.LogWarning("[BASE-{appBase}] Timeout waiting for application task {id} to complete, proceeding anyway", applicationBase, _applicationId);
+                }
+                finally
+                {
+                    timeoutCts.Dispose();
+                    combinedCts.Dispose();
+                }
             }
 
-            if (downloadToken.IsCancellationRequested || (appToken?.IsCancellationRequested ?? false))
+            if (downloadToken.IsCancellationRequested)
             {
                 _forceFullReapply = true;
                 RecordFailure("Application cancelled", "Cancellation");
                 return;
             }
 
-            _applicationCancellationTokenSource = _applicationCancellationTokenSource.CancelRecreate() ?? new CancellationTokenSource();
             var token = _applicationCancellationTokenSource.Token;
 
             _applicationTask = ApplyCharacterDataAsync(applicationBase, handlerForApply, charaData, updatedData, updateModdedPaths, updateManip, moddedPaths, wantsModApply, pendingModReapply, token);

LightlessSync/Services/UiFactory.cs

@@ -105,6 +105,7 @@ public class UiFactory
             groupData: groupData,
             isLightfinderContext: isLightfinderContext,
             lightfinderCid: lightfinderCid,
-            performanceCollector: _performanceCollectorService);
+            performanceCollector: _performanceCollectorService,
+            _apiController);
     }
 }

LightlessSync/UI/StandaloneProfileUi.cs

@@ -11,6 +11,7 @@ using LightlessSync.Services.ServerConfiguration;
 using LightlessSync.UI.Services;
 using LightlessSync.UI.Tags;
 using LightlessSync.Utils;
+using LightlessSync.WebAPI;
 using Microsoft.Extensions.Logging;
 using System.Numerics;
 
@@ -22,6 +23,7 @@ public class StandaloneProfileUi : WindowMediatorSubscriberBase
     private readonly PairUiService _pairUiService;
     private readonly ServerConfigurationManager _serverManager;
     private readonly ProfileTagService _profileTagService;
+    private readonly ApiController _apiController;
     private readonly UiSharedService _uiSharedService;
     private readonly UserData? _userData;
     private readonly GroupData? _groupData;
@@ -60,7 +62,8 @@ public class StandaloneProfileUi : WindowMediatorSubscriberBase
         GroupData? groupData,
         bool isLightfinderContext,
         string? lightfinderCid,
-        PerformanceCollectorService performanceCollector)
+        PerformanceCollectorService performanceCollector,
+        ApiController apiController)
         : base(logger, mediator, BuildWindowTitle(
             userData,
             groupData,
@@ -94,6 +97,7 @@ public class StandaloneProfileUi : WindowMediatorSubscriberBase
             .Apply();
 
         IsOpen = true;
+        _apiController = apiController;
     }
 
     public Pair? Pair { get; }
@@ -248,19 +252,33 @@ public class StandaloneProfileUi : WindowMediatorSubscriberBase
                 ResetBannerTexture();
                 _lastBannerPicture = bannerBytes;
             }
-
             string? noteText = null;
-            string statusLabel = _isLightfinderContext ? "Exploring" : "Offline";
+
+            var isSelfProfile = !_isLightfinderContext
+                                && _userData is not null
+                                && !string.IsNullOrEmpty(_apiController.UID)
+                                && string.Equals(_userData.UID, _apiController.UID, StringComparison.Ordinal);
+
+            string statusLabel = _isLightfinderContext
+                ? "Exploring"
+                : isSelfProfile ? "Online" : "Offline";
+
             string? visiblePlayerName = null;
             bool directPair = false;
             bool youPaused = false;
             bool theyPaused = false;
             List<string> syncshellLines = [];
 
+            if (!_isLightfinderContext)
+            {
+                noteText = _serverManager.GetNoteForUid(_userData!.UID);
+            }
+
             if (!_isLightfinderContext && Pair != null)
             {
                 var snapshot = _pairUiService.GetSnapshot();
                 noteText = _serverManager.GetNoteForUid(Pair.UserData.UID);
+
                 statusLabel = Pair.IsVisible ? "Visible" : (Pair.IsOnline ? "Online" : "Offline");
                 visiblePlayerName = Pair.IsVisible ? Pair.PlayerName : null;
 
@@ -282,11 +300,15 @@ public class StandaloneProfileUi : WindowMediatorSubscriberBase
                             var groupLabel = snapshot.GroupsByGid.TryGetValue(gid, out var groupInfo)
                                 ? groupInfo.GroupAliasOrGID
                                 : gid;
+
                             var groupNote = _serverManager.GetNoteForGid(gid);
                             syncshellLines.Add(string.IsNullOrEmpty(groupNote) ? groupLabel : $"{groupNote} ({groupLabel})");
                         }
                     }
                 }
+
+                if (isSelfProfile)
+                    statusLabel = "Online";
             }
 
             var presenceTokens = new List<PresenceToken>

LightlessSync/UI/SyncshellAdminUI.cs

@@ -116,7 +116,7 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
         var drawList = ImGui.GetWindowDrawList();
 
         var purple = UIColors.Get("LightlessPurple");
-        var gradLeft = purple.WithAlpha(0.0f); 
+        var gradLeft = purple.WithAlpha(0.0f);
         var gradRight = purple.WithAlpha(0.85f);
 
         uint colTopLeft = ImGui.ColorConvertFloat4ToU32(gradLeft);
@@ -162,7 +162,7 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
 
         var subtitlePos = new Vector2(
             pMin.X + 12f * scale,
-            titlePos.Y + titleHeight - 2f * scale); 
+            titlePos.Y + titleHeight - 2f * scale);
 
         ImGui.SetCursorScreenPos(subtitlePos);
         ImGui.PushStyleColor(ImGuiCol.Text, ImGuiColors.DalamudGrey);
@@ -392,25 +392,27 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
         }
         UiSharedService.AttachToolTip("When enabled, inactive non-pinned, non-moderator users will be pruned automatically on the server.");
 
-        ImGui.SameLine();
-        ImGui.SetNextItemWidth(150);
-
-        using (ImRaii.Disabled(!_autoPruneEnabled))
-        {
-            _uiSharedService.DrawCombo(
-                "Day(s) of inactivity",
-                [1, 3, 7, 14, 30, 90],
-                days => $"{days} day(s)",
-                selected =>
-                {
-                    _autoPruneDays = selected;
-                    SavePruneSettings();
-                },
-                _autoPruneDays);
-        }
 
         if (!_autoPruneEnabled)
         {
+            ImGui.BeginDisabled();
+        }
+        ImGui.SameLine();
+        ImGui.SetNextItemWidth(150);
+        _uiSharedService.DrawCombo(
+            "Day(s) of inactivity (gets checked hourly)",
+            [0, 1, 3, 7, 14, 30, 90],
+            (count) => count == 0 ? "2 hours(s)" : count + " day(s)",
+            selected =>
+            {
+                _autoPruneDays = selected;
+                SavePruneSettings();
+            },
+            _autoPruneDays);
+
+        if (!_autoPruneEnabled)
+        {
+            ImGui.EndDisabled();
             UiSharedService.ColorTextWrapped(
                 "Automatic prune is currently disabled. Enable it and choose an inactivity threshold to let the server clean up inactive users automatically.",
                 ImGuiColors.DalamudGrey);
@@ -593,7 +595,7 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
         _uiSharedService.DrawCombo(
             "Day(s) of inactivity",
             [0, 1, 3, 7, 14, 30, 90],
-            (count) => count == 0 ? "15 minute(s)" : count + " day(s)",
+            (count) => count == 0 ? "2 hours(s)" : count + " day(s)",
             (selected) =>
             {
                 _pruneDays = selected;
@@ -663,8 +665,8 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
         var style = ImGui.GetStyle();
         float fullW = ImGui.GetContentRegionAvail().X;
 
-        float colIdentity = fullW * 0.45f; 
+        float colIdentity = fullW * 0.45f;
-        float colMeta = fullW * 0.35f; 
+        float colMeta = fullW * 0.35f;
         float colActions = fullW - colIdentity - colMeta - style.ItemSpacing.X * 2.0f;
 
         // Header
@@ -873,7 +875,7 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
 
         var boolcolor = UiSharedService.GetBoolColor(pair.IsOnline);
         UiSharedService.ColorText(text, boolcolor);
-        
+
         if (ImGui.IsItemClicked())
             ImGui.SetClipboardText(pair.UserData.AliasOrUID);
 
@@ -1093,6 +1095,7 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
 
         ImGui.Dummy(new Vector2(0, 4 * ImGuiHelpers.GlobalScale));
     }
+
     private void SavePruneSettings()
     {
         if (_autoPruneDays <= 0)
@@ -1100,8 +1103,7 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
             _autoPruneEnabled = false;
         }
 
-        var enabled = _autoPruneEnabled && _autoPruneDays > 0;
+        var dto = new GroupPruneSettingsDto(Group: GroupFullInfo.Group, AutoPruneEnabled: _autoPruneEnabled, AutoPruneDays: _autoPruneDays);
-        var dto = new GroupPruneSettingsDto(Group: GroupFullInfo.Group, AutoPruneEnabled: enabled, AutoPruneDays: enabled ? _autoPruneDays : 0);
 
         try
         {

LightlessSync/WebAPI/Files/FileDownloadManager.cs

@@ -28,6 +28,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
     private readonly TextureMetadataHelper _textureMetadataHelper;
 
     private readonly ConcurrentDictionary<ThrottledStream, byte> _activeDownloadStreams;
+    private readonly SemaphoreSlim _decompressGate =
+    new(Math.Max(1, Environment.ProcessorCount / 2), Math.Max(1, Environment.ProcessorCount / 2));
 
     private volatile bool _disableDirectDownloads;
     private int _consecutiveDirectDownloadFailures;
@@ -500,6 +502,14 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
         }
     }
 
+    private void RemoveStatus(string key)
+    {
+        lock (_downloadStatusLock)
+        {
+            _downloadStatus.Remove(key);
+        }
+    }
+
     private async Task DecompressBlockFileAsync(
         string downloadStatusKey,
         string blockFilePath,
@@ -522,32 +532,57 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
 
                     try
                     {
+                        // sanity check length
                         if (fileLengthBytes < 0 || fileLengthBytes > int.MaxValue)
                             throw new InvalidDataException($"Invalid block entry length: {fileLengthBytes}");
 
+                        // safe cast after check
+                        var len = checked((int)fileLengthBytes);
+
                         if (!replacementLookup.TryGetValue(fileHash, out var repl))
                         {
                             Logger.LogWarning("{dlName}: No replacement mapping for {fileHash}", downloadLabel, fileHash);
-                            // still need to skip bytes:
+                            fileBlockStream.Seek(len, SeekOrigin.Current);
-                            var skip = checked((int)fileLengthBytes);
-                            fileBlockStream.Position += skip;
                             continue;
                         }
 
+                        // decompress
                         var filePath = _fileDbManager.GetCacheFilePath(fileHash, repl.Extension);
+                        Logger.LogTrace("{dlName}: Decompressing {file}:{len} => {dest}", downloadLabel, fileHash, fileLengthBytes, filePath);
 
-                        Logger.LogDebug("{dlName}: Decompressing {file}:{len} => {dest}", downloadLabel, fileHash, fileLengthBytes, filePath);
+                        // read compressed data
-
-                        var len = checked((int)fileLengthBytes);
                         var compressed = new byte[len];
-
                         await ReadExactlyAsync(fileBlockStream, compressed.AsMemory(0, len), ct).ConfigureAwait(false);
 
-                        MungeBuffer(compressed);
+                        if (len == 0)
-                        var decompressed = LZ4Wrapper.Unwrap(compressed);
+                        {
+                            await _fileCompactor.WriteAllBytesAsync(filePath, Array.Empty<byte>(), ct).ConfigureAwait(false);
+                            PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale);
+                            continue;
+                        }
 
-                        await _fileCompactor.WriteAllBytesAsync(filePath, decompressed, ct).ConfigureAwait(false);
+                        MungeBuffer(compressed);
-                        PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale);
+
+                        // limit concurrent decompressions
+                        await _decompressGate.WaitAsync(ct).ConfigureAwait(false);
+                        try
+                        {
+                            var sw = System.Diagnostics.Stopwatch.StartNew();
+
+                            // decompress
+                            var decompressed = LZ4Wrapper.Unwrap(compressed);
+
+                            Logger.LogTrace("{dlName}: Unwrap {fileHash} took {ms}ms (compressed {c} bytes, decompressed {d} bytes)",
+                                downloadLabel, fileHash, sw.ElapsedMilliseconds, compressed.Length, decompressed?.Length ?? -1);
+
+                            // write to file
+                            await _fileCompactor.WriteAllBytesAsync(filePath, decompressed, ct).ConfigureAwait(false);
+                            PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale);
+                        }
+                        finally
+                        {
+                            _decompressGate.Release();
+                        }
                     }
                     catch (EndOfStreamException)
                     {
@@ -568,6 +603,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
         {
             Logger.LogError(ex, "{dlName}: Error during block file read", downloadLabel);
         }
+        finally
+        {
+            RemoveStatus(downloadStatusKey);
+        }
     }
 
     public async Task<List<DownloadFileTransfer>> InitiateDownloadList(
@@ -605,20 +644,16 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
             .. await FilesGetSizes(hashes, ct).ConfigureAwait(false),
         ];
 
-        Logger.LogDebug("Files with size 0 or less: {files}",
-            string.Join(", ", downloadFileInfoFromService.Where(f => f.Size <= 0).Select(f => f.Hash)));
-
         foreach (var dto in downloadFileInfoFromService.Where(c => c.IsForbidden))
         {
             if (!_orchestrator.ForbiddenTransfers.Exists(f => string.Equals(f.Hash, dto.Hash, StringComparison.Ordinal)))
                 _orchestrator.ForbiddenTransfers.Add(new DownloadFileTransfer(dto));
         }
 
-        CurrentDownloads = downloadFileInfoFromService
+        CurrentDownloads = [.. downloadFileInfoFromService
             .Distinct()
             .Select(d => new DownloadFileTransfer(d))
-            .Where(d => d.CanBeTransferred)
+            .Where(d => d.CanBeTransferred)];
-            .ToList();
 
         return CurrentDownloads;
     }
@@ -843,6 +878,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
 
             MarkTransferredFiles(directDownload.DirectDownloadUrl!, 1);
             Logger.LogDebug("Finished direct download of {hash}.", directDownload.Hash);
+
+            RemoveStatus(directDownload.DirectDownloadUrl!);
         }
         catch (OperationCanceledException ex)
         {