2.0.0 (#92)

2.0.0 Changes:

- Reworked shell finder UI with compact or list view with profile tags showing with the listing, allowing moderators to broadcast the syncshell as well to have it be used more.
- Reworked user list in syncshell admin screen to have filter visible and moved away from table to its own thing, allowing to copy uid/note/alias when clicking on the name.
- Reworked download bars and download box to make it look more modern, removed the jitter around, so it shouldn't vibrate around much.
- Chat has been added to the top menu, working in Zone or in Syncshells to be used there.
- Paired system has been revamped to make pausing and unpausing faster, and loading people should be faster as well.
- Moved to the internal object table to have faster load times for users; people should load in faster
- Compactor is running on a multi-threaded level instead of single-threaded; this should increase the speed of compacting files
- Nameplate Service has been reworked so it wouldn't use the nameplate handler anymore.
- Files can be resized when downloading to reduce load on users if they aren't compressed. (can be toggled to resize all).
- Penumbra Collections are now only made when people are visible, reducing the load on boot-up when having many syncshells in your list.
- Lightfinder plates have been moved away from using Nameplates, but will use an overlay.
- Main UI has been changed a bit with a gradient, and on hover will glow up now.
- Reworked Profile UI for Syncshell and Users to be more user-facing with more customizable items.
- Reworked Settings UI to look more modern.
- Performance should be better due to new systems that would dispose of the collections and better caching of items.

Co-authored-by: defnotken <itsdefnotken@gmail.com>
Co-authored-by: azyges <aaaaaa@aaa.aaa>
Co-authored-by: choco <choco@patat.nl>
Co-authored-by: cake <admin@cakeandbanana.nl>
Co-authored-by: Minmoose <KennethBohr@outlook.com>
Reviewed-on: #92

LightlessSync/WebAPI/Files/FileDownloadManager.cs

@@ -1,21 +1,16 @@
-using Dalamud.Utility;
 using K4os.Compression.LZ4.Legacy;
 using LightlessSync.API.Data;
 using LightlessSync.API.Dto.Files;
 using LightlessSync.API.Routes;
 using LightlessSync.FileCache;
 using LightlessSync.PlayerData.Handlers;
-using LightlessSync.Services;
 using LightlessSync.Services.Mediator;
+using LightlessSync.Services.TextureCompression;
 using LightlessSync.WebAPI.Files.Models;
 using Microsoft.Extensions.Logging;
-using System;
 using System.Collections.Concurrent;
-using System.IO;
 using System.Net;
 using System.Net.Http.Json;
-using System.Threading;
-using System.Threading.Tasks;
 using LightlessSync.LightlessConfiguration;
 
 namespace LightlessSync.WebAPI.Files;
@@ -26,25 +21,30 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
     private readonly FileCompactor _fileCompactor;
     private readonly FileCacheManager _fileDbManager;
     private readonly FileTransferOrchestrator _orchestrator;
-    private readonly PairProcessingLimiter _pairProcessingLimiter;
     private readonly LightlessConfigService _configService;
+    private readonly TextureDownscaleService _textureDownscaleService;
+    private readonly TextureMetadataHelper _textureMetadataHelper;
     private readonly ConcurrentDictionary<ThrottledStream, byte> _activeDownloadStreams;
-    private static readonly TimeSpan DownloadStallTimeout = TimeSpan.FromSeconds(30);
     private volatile bool _disableDirectDownloads;
     private int _consecutiveDirectDownloadFailures;
     private bool _lastConfigDirectDownloadsState;
 
-    public FileDownloadManager(ILogger<FileDownloadManager> logger, LightlessMediator mediator,
+    public FileDownloadManager(
+        ILogger<FileDownloadManager> logger,
+        LightlessMediator mediator,
         FileTransferOrchestrator orchestrator,
-        FileCacheManager fileCacheManager, FileCompactor fileCompactor,
-        PairProcessingLimiter pairProcessingLimiter, LightlessConfigService configService) : base(logger, mediator)
+        FileCacheManager fileCacheManager,
+        FileCompactor fileCompactor,
+        LightlessConfigService configService,
+        TextureDownscaleService textureDownscaleService, TextureMetadataHelper textureMetadataHelper) : base(logger, mediator)
     {
         _downloadStatus = new Dictionary<string, FileDownloadStatus>(StringComparer.Ordinal);
         _orchestrator = orchestrator;
         _fileDbManager = fileCacheManager;
         _fileCompactor = fileCompactor;
-        _pairProcessingLimiter = pairProcessingLimiter;
         _configService = configService;
+        _textureDownscaleService = textureDownscaleService;
+        _textureMetadataHelper = textureMetadataHelper;
         _activeDownloadStreams = new();
         _lastConfigDirectDownloadsState = _configService.Current.EnableDirectDownloads;
 
@@ -63,6 +63,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
     public List<DownloadFileTransfer> CurrentDownloads { get; private set; } = [];
 
     public List<FileTransfer> ForbiddenTransfers => _orchestrator.ForbiddenTransfers;
+    public Guid? CurrentOwnerToken { get; private set; }
 
     public bool IsDownloading => CurrentDownloads.Any();
 
@@ -83,14 +84,15 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
     {
         CurrentDownloads.Clear();
         _downloadStatus.Clear();
+        CurrentOwnerToken = null;
     }
 
-    public async Task DownloadFiles(GameObjectHandler gameObject, List<FileReplacementData> fileReplacementDto, CancellationToken ct)
+    public async Task DownloadFiles(GameObjectHandler? gameObject, List<FileReplacementData> fileReplacementDto, CancellationToken ct, bool skipDownscale = false)
     {
         Mediator.Publish(new HaltScanMessage(nameof(DownloadFiles)));
         try
         {
-            await DownloadFilesInternal(gameObject, fileReplacementDto, ct).ConfigureAwait(false);
+            await DownloadFilesInternal(gameObject, fileReplacementDto, ct, skipDownscale).ConfigureAwait(false);
         }
         catch
         {
@@ -98,7 +100,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
         }
         finally
         {
-            Mediator.Publish(new DownloadFinishedMessage(gameObject));
+            if (gameObject is not null)
+            {
+                Mediator.Publish(new DownloadFinishedMessage(gameObject));
+            }
             Mediator.Publish(new ResumeScanMessage(nameof(DownloadFiles)));
         }
     }
@@ -272,30 +277,11 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                     int bytesRead;
                     try
                     {
-                        var readTask = stream.ReadAsync(buffer.AsMemory(0, buffer.Length), ct).AsTask();
-                        while (!readTask.IsCompleted)
-                        {
-                            var completedTask = await Task.WhenAny(readTask, Task.Delay(DownloadStallTimeout)).ConfigureAwait(false);
-                            if (completedTask == readTask)
-                            {
-                                break;
-                            }
-
-                            ct.ThrowIfCancellationRequested();
-
-                            var snapshot = _pairProcessingLimiter.GetSnapshot();
-                            if (snapshot.Waiting > 0)
-                            {
-                                throw new TimeoutException($"No data received for {DownloadStallTimeout.TotalSeconds} seconds while downloading {requestUrl} (waiting: {snapshot.Waiting})");
-                            }
-
-                            Logger.LogTrace("Download stalled for {requestUrl} but no queued pairs, continuing to wait", requestUrl);
-                        }
-
-                        bytesRead = await readTask.ConfigureAwait(false);
+                        bytesRead = await stream.ReadAsync(buffer.AsMemory(0, buffer.Length), ct).ConfigureAwait(false);
                     }
-                    catch (OperationCanceledException)
+                    catch (OperationCanceledException ex)
                     {
+                        Logger.LogWarning(ex, "Request got cancelled : {url}", requestUrl);
                         throw;
                     }
 
@@ -314,11 +300,6 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                 Logger.LogDebug("{requestUrl} downloaded to {destination}", requestUrl, destinationFilename);
             }
         }
-        catch (TimeoutException ex)
-        {
-            Logger.LogWarning(ex, "Detected stalled download for {requestUrl}, aborting transfer", requestUrl);
-            throw;
-        }
         catch (OperationCanceledException)
         {
             throw;
@@ -352,7 +333,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
         }
     }
 
-    private async Task DecompressBlockFileAsync(string downloadStatusKey, string blockFilePath, List<FileReplacementData> fileReplacement, string downloadLabel)
+    private async Task DecompressBlockFileAsync(string downloadStatusKey, string blockFilePath, List<FileReplacementData> fileReplacement, string downloadLabel, bool skipDownscale)
     {
         if (_downloadStatus.TryGetValue(downloadStatusKey, out var status))
         {
@@ -385,7 +366,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                     var decompressedFile = LZ4Wrapper.Unwrap(compressedFileContent);
                     await _fileCompactor.WriteAllBytesAsync(filePath, decompressedFile, CancellationToken.None).ConfigureAwait(false);
 
-                    PersistFileToStorage(fileHash, filePath);
+                    var gamePath = fileReplacement.FirstOrDefault(f => string.Equals(f.Hash, fileHash, StringComparison.OrdinalIgnoreCase))?.GamePaths.FirstOrDefault() ?? string.Empty;
+                    PersistFileToStorage(fileHash, filePath, gamePath, skipDownscale);
                 }
                 catch (EndOfStreamException)
                 {
@@ -413,7 +395,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
     }
 
     private async Task PerformDirectDownloadFallbackAsync(DownloadFileTransfer directDownload, List<FileReplacementData> fileReplacement,
-        IProgress<long> progress, CancellationToken token, bool slotAlreadyAcquired)
+        IProgress<long> progress, CancellationToken token, bool skipDownscale, bool slotAlreadyAcquired)
     {
         if (string.IsNullOrEmpty(directDownload.DirectDownloadUrl))
         {
@@ -455,7 +437,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                 throw new FileNotFoundException("Block file missing after direct download fallback.", blockFile);
             }
 
-            await DecompressBlockFileAsync(downloadKey, blockFile, fileReplacement, $"fallback-{directDownload.Hash}").ConfigureAwait(false);
+            await DecompressBlockFileAsync(downloadKey, blockFile, fileReplacement, $"fallback-{directDownload.Hash}", skipDownscale).ConfigureAwait(false);
         }
         finally
         {
@@ -478,8 +460,9 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
         }
     }
 
-    public async Task<List<DownloadFileTransfer>> InitiateDownloadList(GameObjectHandler gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct)
+    public async Task<List<DownloadFileTransfer>> InitiateDownloadList(GameObjectHandler? gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct, Guid? ownerToken = null)
     {
+        CurrentOwnerToken = ownerToken;
         var objectName = gameObjectHandler?.Name ?? "Unknown";
         Logger.LogDebug("Download start: {id}", objectName);
 
@@ -520,7 +503,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
         return CurrentDownloads;
     }
 
-    private async Task DownloadFilesInternal(GameObjectHandler gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct)
+    private async Task DownloadFilesInternal(GameObjectHandler? gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct, bool skipDownscale)
     {
         var objectName = gameObjectHandler?.Name ?? "Unknown";
 
@@ -583,7 +566,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
             Logger.LogWarning("Downloading {direct} files directly, and {batchtotal} in {batches} batches.", directDownloads.Count, batchDownloads.Count, downloadBatches.Length);
         }
 
-        Mediator.Publish(new DownloadStartedMessage(gameObjectHandler, _downloadStatus));
+        if (gameObjectHandler is not null)
+        {
+            Mediator.Publish(new DownloadStartedMessage(gameObjectHandler, _downloadStatus));
+        }
 
         Task batchDownloadsTask = downloadBatches.Length == 0 ? Task.CompletedTask : Parallel.ForEachAsync(downloadBatches, new ParallelOptions()
         {
@@ -651,7 +637,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                     return;
                 }
 
-                await DecompressBlockFileAsync(fileGroup.Key, blockFile, fileReplacement, fi.Name).ConfigureAwait(false);
+                await DecompressBlockFileAsync(fileGroup.Key, blockFile, fileReplacement, fi.Name, skipDownscale).ConfigureAwait(false);
             }
             finally
             {
@@ -690,14 +676,13 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
 
             if (!ShouldUseDirectDownloads())
             {
-                await PerformDirectDownloadFallbackAsync(directDownload, fileReplacement, progress, token, slotAlreadyAcquired: false).ConfigureAwait(false);
+                await PerformDirectDownloadFallbackAsync(directDownload, fileReplacement, progress, token, skipDownscale, slotAlreadyAcquired: false).ConfigureAwait(false);
                 return;
             }
 
             var tempFilename = _fileDbManager.GetCacheFilePath(directDownload.Hash, "bin");
             var slotAcquired = false;
 
-
             try
             {
                 downloadTracker.DownloadStatus = DownloadStatus.WaitingForSlot;
@@ -727,7 +712,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                     byte[] compressedBytes = await File.ReadAllBytesAsync(tempFilename).ConfigureAwait(false);
                     var decompressedBytes = LZ4Wrapper.Unwrap(compressedBytes);
                     await _fileCompactor.WriteAllBytesAsync(finalFilename, decompressedBytes, CancellationToken.None).ConfigureAwait(false);
-                    PersistFileToStorage(directDownload.Hash, finalFilename);
+                    PersistFileToStorage(directDownload.Hash, finalFilename, replacement.GamePaths[0], skipDownscale);
 
                     downloadTracker.TransferredFiles = 1;
                     Logger.LogDebug("Finished direct download of {hash}.", directDownload.Hash);
@@ -739,8 +724,15 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
             }
             catch (OperationCanceledException ex)
             {
-                Logger.LogDebug("{hash}: Detected cancellation of direct download, discarding file.", directDownload.Hash);
-                Logger.LogError(ex, "{hash}: Error during direct download.", directDownload.Hash);
+                if (token.IsCancellationRequested)
+                {
+                    Logger.LogDebug("{hash}: Direct download cancelled by caller, discarding file.", directDownload.Hash);
+                }
+                else
+                {
+                    Logger.LogWarning(ex, "{hash}: Direct download cancelled unexpectedly.", directDownload.Hash);
+                }
+
                 ClearDownload();
                 return;
             }
@@ -762,7 +754,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                 try
                 {
                     downloadTracker.DownloadStatus = DownloadStatus.WaitingForQueue;
-                    await PerformDirectDownloadFallbackAsync(directDownload, fileReplacement, progress, token, slotAcquired).ConfigureAwait(false);
+                    await PerformDirectDownloadFallbackAsync(directDownload, fileReplacement, progress, token, skipDownscale, slotAcquired).ConfigureAwait(false);
 
                     if (!expectedDirectDownloadFailure && failureCount >= 3 && !_disableDirectDownloads)
                     {
@@ -815,7 +807,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
         return await response.Content.ReadFromJsonAsync<List<DownloadFileDto>>(cancellationToken: ct).ConfigureAwait(false) ?? [];
     }
 
-    private void PersistFileToStorage(string fileHash, string filePath)
+    private void PersistFileToStorage(string fileHash, string filePath, string gamePath, bool skipDownscale)
     {
         var fi = new FileInfo(filePath);
         Func<DateTime> RandomDayInThePast()
@@ -832,6 +824,11 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
         try
         {
             var entry = _fileDbManager.CreateCacheEntry(filePath);
+            var mapKind = _textureMetadataHelper.DetermineMapKind(gamePath, filePath);
+            if (!skipDownscale)
+            {
+                _textureDownscaleService.ScheduleDownscale(fileHash, filePath, mapKind);
+            }
             if (entry != null && !string.Equals(entry.Hash, fileHash, StringComparison.OrdinalIgnoreCase))
             {
                 Logger.LogError("Hash mismatch after extracting, got {hash}, expected {expectedHash}, deleting file", entry.Hash, fileHash);