Merge pull request 'Complete Decompression after try.' (#122) from decompression-bullshit into 2.0.2

Reviewed-on: #122

LightlessSync/LightlessSync.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <Authors></Authors>
     <Company></Company>
-    <Version>2.0.1.69</Version>
+    <Version>2.0.2</Version>
     <Description></Description>
     <Copyright></Copyright>
     <PackageProjectUrl>https://github.com/Light-Public-Syncshells/LightlessClient</PackageProjectUrl>

LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs

@@ -1423,7 +1423,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
     private Task _visibilityGraceTask;
 
     private async Task DownloadAndApplyCharacterAsync(Guid applicationBase, CharacterData charaData, Dictionary<ObjectKind, HashSet<PlayerChanges>> updatedData,
-        bool updateModdedPaths, bool updateManip, Dictionary<(string GamePath, string? Hash), string>? cachedModdedPaths, CancellationToken downloadToken)
+     bool updateModdedPaths, bool updateManip, Dictionary<(string GamePath, string? Hash), string>? cachedModdedPaths, CancellationToken downloadToken)
     {
         var concurrencyLease = await _pairProcessingLimiter.AcquireAsync(downloadToken).ConfigureAwait(false);
         try
@@ -1577,24 +1577,37 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
                 RecordFailure("Handler not available for application", "HandlerUnavailable");
                 return;
             }
+            _applicationCancellationTokenSource = _applicationCancellationTokenSource.CancelRecreate() ?? new CancellationTokenSource();
 
-            var appToken = _applicationCancellationTokenSource?.Token;
+            if (_applicationTask != null && !_applicationTask.IsCompleted)
-            while ((!_applicationTask?.IsCompleted ?? false)
-                   && !downloadToken.IsCancellationRequested
-                   && (!appToken?.IsCancellationRequested ?? false))
             {
-                Logger.LogDebug("[BASE-{appBase}] Waiting for current data application (Id: {id}) for player ({handler}) to finish", applicationBase, _applicationId, PlayerName);
+                Logger.LogDebug("[BASE-{appBase}] Cancelling current data application (Id: {id}) for player ({handler})", applicationBase, _applicationId, PlayerName);
-                await Task.Delay(250).ConfigureAwait(false);
+
+                var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+                var combinedCts = CancellationTokenSource.CreateLinkedTokenSource(downloadToken, timeoutCts.Token);
+
+                try
+                {
+                    await _applicationTask.WaitAsync(combinedCts.Token).ConfigureAwait(false);
+                }
+                catch (OperationCanceledException)
+                {
+                    Logger.LogWarning("[BASE-{appBase}] Timeout waiting for application task {id} to complete, proceeding anyway", applicationBase, _applicationId);
+                }
+                finally
+                {
+                    timeoutCts.Dispose();
+                    combinedCts.Dispose();
+                }
             }
 
-            if (downloadToken.IsCancellationRequested || (appToken?.IsCancellationRequested ?? false))
+            if (downloadToken.IsCancellationRequested)
             {
                 _forceFullReapply = true;
                 RecordFailure("Application cancelled", "Cancellation");
                 return;
             }
 
-            _applicationCancellationTokenSource = _applicationCancellationTokenSource.CancelRecreate() ?? new CancellationTokenSource();
             var token = _applicationCancellationTokenSource.Token;
 
             _applicationTask = ApplyCharacterDataAsync(applicationBase, handlerForApply, charaData, updatedData, updateModdedPaths, updateManip, moddedPaths, wantsModApply, pendingModReapply, token);

LightlessSync/Services/UiFactory.cs

@@ -105,6 +105,7 @@ public class UiFactory
             groupData: groupData,
             isLightfinderContext: isLightfinderContext,
             lightfinderCid: lightfinderCid,
-            performanceCollector: _performanceCollectorService);
+            performanceCollector: _performanceCollectorService,
+            _apiController);
     }
 }

LightlessSync/UI/SettingsUi.cs

@@ -84,6 +84,8 @@ public class SettingsUi : WindowMediatorSubscriberBase
     private bool _pairDiagnosticsEnabled;
     private string? _selectedPairDebugUid = null;
     private string _lightfinderIconInput = string.Empty;
+    private bool _showLightfinderRendererWarning = false;
+    private LightfinderLabelRenderer _pendingLightfinderRenderer = LightfinderLabelRenderer.Pictomancy;
     private bool _lightfinderIconInputInitialized = false;
     private int _lightfinderIconPresetIndex = -1;
     private static readonly LightlessConfig DefaultConfig = new();
@@ -2372,7 +2374,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
                 var labelRenderer = _configService.Current.LightfinderLabelRenderer;
                 var labelRendererLabel = labelRenderer switch
                 {
-                    LightfinderLabelRenderer.SignatureHook => "Native nameplate (sig hook)",
+                    LightfinderLabelRenderer.SignatureHook => "Native Nameplate Rendering",
                     _ => "ImGui Overlay",
                 };
 
@@ -2382,18 +2384,25 @@ public class SettingsUi : WindowMediatorSubscriberBase
                     {
                         var optionLabel = option switch
                         {
-                            LightfinderLabelRenderer.SignatureHook => "Native Nameplate (sig hook)",
+                            LightfinderLabelRenderer.SignatureHook => "Native Nameplate Rendering",
                             _ => "ImGui Overlay",
                         };
 
                         var selected = option == labelRenderer;
                         if (ImGui.Selectable(optionLabel, selected))
                         {
-                            _configService.Current.LightfinderLabelRenderer = option;
+                            if (option == LightfinderLabelRenderer.SignatureHook)
-                            _configService.Save();
+                            {
-                            _nameplateService.RequestRedraw();
+                                _pendingLightfinderRenderer = option;
+                                _showLightfinderRendererWarning = true;
+                            }
+                            else
+                            {
+                                _configService.Current.LightfinderLabelRenderer = option;
+                                _configService.Save();
+                                _nameplateService.RequestRedraw();
+                            }
                         }
-
                         if (selected)
                             ImGui.SetItemDefaultFocus();
                     }
@@ -2401,6 +2410,34 @@ public class SettingsUi : WindowMediatorSubscriberBase
                     ImGui.EndCombo();
                 }
 
+                if (_showLightfinderRendererWarning)
+                {
+                    ImGui.SetNextWindowSize(new Vector2(450f, 0f), ImGuiCond.Appearing);
+                    ImGui.OpenPopup("Nameplate Warning");
+                }
+
+                if (ImGui.BeginPopupModal("Nameplate Warning", ref _showLightfinderRendererWarning, ImGuiWindowFlags.AlwaysAutoResize))
+                {
+                    ImGui.TextColored(UIColors.Get("DimRed"), "USE AT YOUR RISK!");
+                    ImGui.Spacing();
+                    ImGui.TextWrapped("Writing on to the native Nameplates is known to be unstable and MAY cause crashes. DO NOT REPORT THOSE CRASHES TO DALAMUD. We will also not be supporting Nameplate crashes. You have been warned.");
+                    ImGui.Spacing();
+                    ImGui.TextWrapped("By accepting this warning, you understand that you are using this feature at risk of crashing.");
+                    ImGui.Spacing();
+
+                    var buttonWidth = ImGui.GetContentRegionAvail().X;
+                    if (ImGui.Button("I Understand", new Vector2(buttonWidth, 0)))
+                    {
+                        _configService.Current.LightfinderLabelRenderer = _pendingLightfinderRenderer;
+                        _configService.Save();
+                        _nameplateService.RequestRedraw();
+                        _showLightfinderRendererWarning = false;
+                        ImGui.CloseCurrentPopup();
+                    }
+
+                    ImGui.EndPopup();
+                }
+
                 _uiShared.DrawHelpText("Choose how Lightfinder labels render: the default ImGui overlay or native nameplate nodes via signature hook.");
 
                 UiSharedService.ColoredSeparator(UIColors.Get("LightlessPurpleDefault"), 1.5f);

LightlessSync/UI/StandaloneProfileUi.cs

@@ -11,6 +11,7 @@ using LightlessSync.Services.ServerConfiguration;
 using LightlessSync.UI.Services;
 using LightlessSync.UI.Tags;
 using LightlessSync.Utils;
+using LightlessSync.WebAPI;
 using Microsoft.Extensions.Logging;
 using System.Numerics;
 
@@ -22,6 +23,7 @@ public class StandaloneProfileUi : WindowMediatorSubscriberBase
     private readonly PairUiService _pairUiService;
     private readonly ServerConfigurationManager _serverManager;
     private readonly ProfileTagService _profileTagService;
+    private readonly ApiController _apiController;
     private readonly UiSharedService _uiSharedService;
     private readonly UserData? _userData;
     private readonly GroupData? _groupData;
@@ -60,7 +62,8 @@ public class StandaloneProfileUi : WindowMediatorSubscriberBase
         GroupData? groupData,
         bool isLightfinderContext,
         string? lightfinderCid,
-        PerformanceCollectorService performanceCollector)
+        PerformanceCollectorService performanceCollector,
+        ApiController apiController)
         : base(logger, mediator, BuildWindowTitle(
             userData,
             groupData,
@@ -94,6 +97,7 @@ public class StandaloneProfileUi : WindowMediatorSubscriberBase
             .Apply();
 
         IsOpen = true;
+        _apiController = apiController;
     }
 
     public Pair? Pair { get; }
@@ -248,19 +252,33 @@ public class StandaloneProfileUi : WindowMediatorSubscriberBase
                 ResetBannerTexture();
                 _lastBannerPicture = bannerBytes;
             }
-
             string? noteText = null;
-            string statusLabel = _isLightfinderContext ? "Exploring" : "Offline";
+
+            var isSelfProfile = !_isLightfinderContext
+                                && _userData is not null
+                                && !string.IsNullOrEmpty(_apiController.UID)
+                                && string.Equals(_userData.UID, _apiController.UID, StringComparison.Ordinal);
+
+            string statusLabel = _isLightfinderContext
+                ? "Exploring"
+                : isSelfProfile ? "Online" : "Offline";
+
             string? visiblePlayerName = null;
             bool directPair = false;
             bool youPaused = false;
             bool theyPaused = false;
             List<string> syncshellLines = [];
 
+            if (!_isLightfinderContext)
+            {
+                noteText = _serverManager.GetNoteForUid(_userData!.UID);
+            }
+
             if (!_isLightfinderContext && Pair != null)
             {
                 var snapshot = _pairUiService.GetSnapshot();
                 noteText = _serverManager.GetNoteForUid(Pair.UserData.UID);
+
                 statusLabel = Pair.IsVisible ? "Visible" : (Pair.IsOnline ? "Online" : "Offline");
                 visiblePlayerName = Pair.IsVisible ? Pair.PlayerName : null;
 
@@ -282,11 +300,15 @@ public class StandaloneProfileUi : WindowMediatorSubscriberBase
                             var groupLabel = snapshot.GroupsByGid.TryGetValue(gid, out var groupInfo)
                                 ? groupInfo.GroupAliasOrGID
                                 : gid;
+
                             var groupNote = _serverManager.GetNoteForGid(gid);
                             syncshellLines.Add(string.IsNullOrEmpty(groupNote) ? groupLabel : $"{groupNote} ({groupLabel})");
                         }
                     }
                 }
+
+                if (isSelfProfile)
+                    statusLabel = "Online";
             }
 
             var presenceTokens = new List<PresenceToken>

LightlessSync/UI/SyncshellAdminUI.cs

@@ -116,7 +116,7 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
         var drawList = ImGui.GetWindowDrawList();
 
         var purple = UIColors.Get("LightlessPurple");
-        var gradLeft = purple.WithAlpha(0.0f); 
+        var gradLeft = purple.WithAlpha(0.0f);
         var gradRight = purple.WithAlpha(0.85f);
 
         uint colTopLeft = ImGui.ColorConvertFloat4ToU32(gradLeft);
@@ -162,7 +162,7 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
 
         var subtitlePos = new Vector2(
             pMin.X + 12f * scale,
-            titlePos.Y + titleHeight - 2f * scale); 
+            titlePos.Y + titleHeight - 2f * scale);
 
         ImGui.SetCursorScreenPos(subtitlePos);
         ImGui.PushStyleColor(ImGuiCol.Text, ImGuiColors.DalamudGrey);
@@ -392,25 +392,27 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
         }
         UiSharedService.AttachToolTip("When enabled, inactive non-pinned, non-moderator users will be pruned automatically on the server.");
 
-        ImGui.SameLine();
-        ImGui.SetNextItemWidth(150);
-
-        using (ImRaii.Disabled(!_autoPruneEnabled))
-        {
-            _uiSharedService.DrawCombo(
-                "Day(s) of inactivity",
-                [1, 3, 7, 14, 30, 90],
-                days => $"{days} day(s)",
-                selected =>
-                {
-                    _autoPruneDays = selected;
-                    SavePruneSettings();
-                },
-                _autoPruneDays);
-        }
 
         if (!_autoPruneEnabled)
         {
+            ImGui.BeginDisabled();
+        }
+        ImGui.SameLine();
+        ImGui.SetNextItemWidth(150);
+        _uiSharedService.DrawCombo(
+            "Day(s) of inactivity (gets checked hourly)",
+            [0, 1, 3, 7, 14, 30, 90],
+            (count) => count == 0 ? "2 hours(s)" : count + " day(s)",
+            selected =>
+            {
+                _autoPruneDays = selected;
+                SavePruneSettings();
+            },
+            _autoPruneDays);
+
+        if (!_autoPruneEnabled)
+        {
+            ImGui.EndDisabled();
             UiSharedService.ColorTextWrapped(
                 "Automatic prune is currently disabled. Enable it and choose an inactivity threshold to let the server clean up inactive users automatically.",
                 ImGuiColors.DalamudGrey);
@@ -593,7 +595,7 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
         _uiSharedService.DrawCombo(
             "Day(s) of inactivity",
             [0, 1, 3, 7, 14, 30, 90],
-            (count) => count == 0 ? "15 minute(s)" : count + " day(s)",
+            (count) => count == 0 ? "2 hours(s)" : count + " day(s)",
             (selected) =>
             {
                 _pruneDays = selected;
@@ -663,8 +665,8 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
         var style = ImGui.GetStyle();
         float fullW = ImGui.GetContentRegionAvail().X;
 
-        float colIdentity = fullW * 0.45f; 
+        float colIdentity = fullW * 0.45f;
-        float colMeta = fullW * 0.35f; 
+        float colMeta = fullW * 0.35f;
         float colActions = fullW - colIdentity - colMeta - style.ItemSpacing.X * 2.0f;
 
         // Header
@@ -873,7 +875,7 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
 
         var boolcolor = UiSharedService.GetBoolColor(pair.IsOnline);
         UiSharedService.ColorText(text, boolcolor);
-        
+
         if (ImGui.IsItemClicked())
             ImGui.SetClipboardText(pair.UserData.AliasOrUID);
 
@@ -1093,6 +1095,7 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
 
         ImGui.Dummy(new Vector2(0, 4 * ImGuiHelpers.GlobalScale));
     }
+
     private void SavePruneSettings()
     {
         if (_autoPruneDays <= 0)
@@ -1100,8 +1103,7 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
             _autoPruneEnabled = false;
         }
 
-        var enabled = _autoPruneEnabled && _autoPruneDays > 0;
+        var dto = new GroupPruneSettingsDto(Group: GroupFullInfo.Group, AutoPruneEnabled: _autoPruneEnabled, AutoPruneDays: _autoPruneDays);
-        var dto = new GroupPruneSettingsDto(Group: GroupFullInfo.Group, AutoPruneEnabled: enabled, AutoPruneDays: enabled ? _autoPruneDays : 0);
 
         try
         {

LightlessSync/UI/ZoneChatUi.cs

@@ -1000,23 +1000,26 @@ public sealed class ZoneChatUi : WindowMediatorSubscriberBase
             if (sanitized is not null)
             {
                 TrackPendingDraftClear(channel.Key, sanitized);
+                draft = string.Empty;
+                _draftMessages[channel.Key] = draft;
+                _scrollToBottom = true;
 
-                if (TrySendDraft(channel, sanitized))
+                _ = Task.Run(async () =>
                 {
-                    _scrollToBottom = true;
+                    try
-
-                    if (_draftMessages.TryGetValue(channel.Key, out var current) &&
-                        string.Equals(current, draftAtSend, StringComparison.Ordinal))
                     {
-                        draft = string.Empty;
+                        var succeeded = await _zoneChatService.SendMessageAsync(channel.Descriptor, sanitized).ConfigureAwait(false);
-                        _draftMessages[channel.Key] = draft;
+                        if (!succeeded)
+                        {
+                            RemovePendingDraftClear(channel.Key, sanitized);
+                        }
                     }
-
+                    catch (Exception ex)
-                }
+                    {
-                else
+                        _logger.LogWarning(ex, "Failed to send chat message");
-                {
+                        RemovePendingDraftClear(channel.Key, sanitized);
-                    RemovePendingDraftClear(channel.Key, sanitized);
+                    }
-                }
+                });
             }
         }
     }

LightlessSync/WebAPI/Files/FileDownloadManager.cs

@@ -28,6 +28,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
     private readonly TextureMetadataHelper _textureMetadataHelper;
 
     private readonly ConcurrentDictionary<ThrottledStream, byte> _activeDownloadStreams;
+    private readonly SemaphoreSlim _decompressGate =
+    new(Math.Max(1, Environment.ProcessorCount / 2), Math.Max(1, Environment.ProcessorCount / 2));
 
     private volatile bool _disableDirectDownloads;
     private int _consecutiveDirectDownloadFailures;
@@ -500,6 +502,14 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
         }
     }
 
+    private void RemoveStatus(string key)
+    {
+        lock (_downloadStatusLock)
+        {
+            _downloadStatus.Remove(key);
+        }
+    }
+
     private async Task DecompressBlockFileAsync(
         string downloadStatusKey,
         string blockFilePath,
@@ -522,32 +532,57 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
 
                     try
                     {
+                        // sanity check length
                         if (fileLengthBytes < 0 || fileLengthBytes > int.MaxValue)
                             throw new InvalidDataException($"Invalid block entry length: {fileLengthBytes}");
 
+                        // safe cast after check
+                        var len = checked((int)fileLengthBytes);
+
                         if (!replacementLookup.TryGetValue(fileHash, out var repl))
                         {
                             Logger.LogWarning("{dlName}: No replacement mapping for {fileHash}", downloadLabel, fileHash);
-                            // still need to skip bytes:
+                            fileBlockStream.Seek(len, SeekOrigin.Current);
-                            var skip = checked((int)fileLengthBytes);
-                            fileBlockStream.Position += skip;
                             continue;
                         }
 
+                        // decompress
                         var filePath = _fileDbManager.GetCacheFilePath(fileHash, repl.Extension);
+                        Logger.LogTrace("{dlName}: Decompressing {file}:{len} => {dest}", downloadLabel, fileHash, fileLengthBytes, filePath);
 
-                        Logger.LogDebug("{dlName}: Decompressing {file}:{len} => {dest}", downloadLabel, fileHash, fileLengthBytes, filePath);
+                        // read compressed data
-
-                        var len = checked((int)fileLengthBytes);
                         var compressed = new byte[len];
-
                         await ReadExactlyAsync(fileBlockStream, compressed.AsMemory(0, len), ct).ConfigureAwait(false);
 
-                        MungeBuffer(compressed);
+                        if (len == 0)
-                        var decompressed = LZ4Wrapper.Unwrap(compressed);
+                        {
+                            await _fileCompactor.WriteAllBytesAsync(filePath, Array.Empty<byte>(), ct).ConfigureAwait(false);
+                            PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale);
+                            continue;
+                        }
 
-                        await _fileCompactor.WriteAllBytesAsync(filePath, decompressed, ct).ConfigureAwait(false);
+                        MungeBuffer(compressed);
-                        PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale);
+
+                        // limit concurrent decompressions
+                        await _decompressGate.WaitAsync(ct).ConfigureAwait(false);
+                        try
+                        {
+                            var sw = System.Diagnostics.Stopwatch.StartNew();
+
+                            // decompress
+                            var decompressed = LZ4Wrapper.Unwrap(compressed);
+
+                            Logger.LogTrace("{dlName}: Unwrap {fileHash} took {ms}ms (compressed {c} bytes, decompressed {d} bytes)",
+                                downloadLabel, fileHash, sw.ElapsedMilliseconds, compressed.Length, decompressed?.Length ?? -1);
+
+                            // write to file
+                            await _fileCompactor.WriteAllBytesAsync(filePath, decompressed, ct).ConfigureAwait(false);
+                            PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale);
+                        }
+                        finally
+                        {
+                            _decompressGate.Release();
+                        }
                     }
                     catch (EndOfStreamException)
                     {
@@ -568,6 +603,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
         {
             Logger.LogError(ex, "{dlName}: Error during block file read", downloadLabel);
         }
+        finally
+        {
+            RemoveStatus(downloadStatusKey);
+        }
     }
 
     public async Task<List<DownloadFileTransfer>> InitiateDownloadList(
@@ -605,20 +644,16 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
             .. await FilesGetSizes(hashes, ct).ConfigureAwait(false),
         ];
 
-        Logger.LogDebug("Files with size 0 or less: {files}",
-            string.Join(", ", downloadFileInfoFromService.Where(f => f.Size <= 0).Select(f => f.Hash)));
-
         foreach (var dto in downloadFileInfoFromService.Where(c => c.IsForbidden))
         {
             if (!_orchestrator.ForbiddenTransfers.Exists(f => string.Equals(f.Hash, dto.Hash, StringComparison.Ordinal)))
                 _orchestrator.ForbiddenTransfers.Add(new DownloadFileTransfer(dto));
         }
 
-        CurrentDownloads = downloadFileInfoFromService
+        CurrentDownloads = [.. downloadFileInfoFromService
             .Distinct()
             .Select(d => new DownloadFileTransfer(d))
-            .Where(d => d.CanBeTransferred)
+            .Where(d => d.CanBeTransferred)];
-            .ToList();
 
         return CurrentDownloads;
     }
@@ -843,6 +878,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
 
             MarkTransferredFiles(directDownload.DirectDownloadUrl!, 1);
             Logger.LogDebug("Finished direct download of {hash}.", directDownload.Hash);
+
+            RemoveStatus(directDownload.DirectDownloadUrl!);
         }
         catch (OperationCanceledException ex)
         {