Merge branch '2.0.0' into dotnet10-api14-migration

2025-11-26 16:20:38 -06:00
parent c94c785e0b e350e8007a
commit d1f5122cd4
116 changed files with 20468 additions and 3311 deletions
--- a/LightlessSync/WebAPI/Files/FileDownloadManager.cs
+++ b/LightlessSync/WebAPI/Files/FileDownloadManager.cs
@@ -7,6 +7,7 @@ using LightlessSync.FileCache;
 using LightlessSync.PlayerData.Handlers;
 using LightlessSync.Services;
 using LightlessSync.Services.Mediator;
+using LightlessSync.Services.TextureCompression;
 using LightlessSync.WebAPI.Files.Models;
 using Microsoft.Extensions.Logging;
 using System;
@@ -28,16 +29,23 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
    private readonly FileTransferOrchestrator _orchestrator;
    private readonly PairProcessingLimiter _pairProcessingLimiter;
    private readonly LightlessConfigService _configService;
+    private readonly TextureDownscaleService _textureDownscaleService;
+    private readonly TextureMetadataHelper _textureMetadataHelper;
    private readonly ConcurrentDictionary<ThrottledStream, byte> _activeDownloadStreams;
    private static readonly TimeSpan DownloadStallTimeout = TimeSpan.FromSeconds(30);
    private volatile bool _disableDirectDownloads;
    private int _consecutiveDirectDownloadFailures;
    private bool _lastConfigDirectDownloadsState;

-    public FileDownloadManager(ILogger<FileDownloadManager> logger, LightlessMediator mediator,
+    public FileDownloadManager(
+        ILogger<FileDownloadManager> logger,
+        LightlessMediator mediator,
        FileTransferOrchestrator orchestrator,
-        FileCacheManager fileCacheManager, FileCompactor fileCompactor,
-        PairProcessingLimiter pairProcessingLimiter, LightlessConfigService configService) : base(logger, mediator)
+        FileCacheManager fileCacheManager,
+        FileCompactor fileCompactor,
+        PairProcessingLimiter pairProcessingLimiter,
+        LightlessConfigService configService,
+        TextureDownscaleService textureDownscaleService, TextureMetadataHelper textureMetadataHelper) : base(logger, mediator)
    {
        _downloadStatus = new Dictionary<string, FileDownloadStatus>(StringComparer.Ordinal);
        _orchestrator = orchestrator;
@@ -45,6 +53,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
        _fileCompactor = fileCompactor;
        _pairProcessingLimiter = pairProcessingLimiter;
        _configService = configService;
+        _textureDownscaleService = textureDownscaleService;
+        _textureMetadataHelper = textureMetadataHelper;
        _activeDownloadStreams = new();
        _lastConfigDirectDownloadsState = _configService.Current.EnableDirectDownloads;

@@ -63,6 +73,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
    public List<DownloadFileTransfer> CurrentDownloads { get; private set; } = [];

    public List<FileTransfer> ForbiddenTransfers => _orchestrator.ForbiddenTransfers;
+    public Guid? CurrentOwnerToken { get; private set; }

    public bool IsDownloading => CurrentDownloads.Any();

@@ -83,14 +94,15 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
    {
        CurrentDownloads.Clear();
        _downloadStatus.Clear();
+        CurrentOwnerToken = null;
    }

-    public async Task DownloadFiles(GameObjectHandler gameObject, List<FileReplacementData> fileReplacementDto, CancellationToken ct)
+    public async Task DownloadFiles(GameObjectHandler? gameObject, List<FileReplacementData> fileReplacementDto, CancellationToken ct, bool skipDownscale = false)
    {
        Mediator.Publish(new HaltScanMessage(nameof(DownloadFiles)));
        try
        {
-            await DownloadFilesInternal(gameObject, fileReplacementDto, ct).ConfigureAwait(false);
+            await DownloadFilesInternal(gameObject, fileReplacementDto, ct, skipDownscale).ConfigureAwait(false);
        }
        catch
        {
@@ -98,7 +110,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
        }
        finally
        {
-            Mediator.Publish(new DownloadFinishedMessage(gameObject));
+            if (gameObject is not null)
+            {
+                Mediator.Publish(new DownloadFinishedMessage(gameObject));
+            }
            Mediator.Publish(new ResumeScanMessage(nameof(DownloadFiles)));
        }
    }
@@ -272,7 +287,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                    int bytesRead;
                    try
                    {
-                        var readTask = stream.ReadAsync(buffer.AsMemory(0, buffer.Length), ct).AsTask();
+                        using var readCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct);
+                        var readTask = stream.ReadAsync(buffer.AsMemory(0, buffer.Length), readCancellation.Token).AsTask();
                        while (!readTask.IsCompleted)
                        {
                            var completedTask = await Task.WhenAny(readTask, Task.Delay(DownloadStallTimeout)).ConfigureAwait(false);
@@ -286,6 +302,20 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                            var snapshot = _pairProcessingLimiter.GetSnapshot();
                            if (snapshot.Waiting > 0)
                            {
+                                readCancellation.Cancel();
+                                try
+                                {
+                                    await readTask.ConfigureAwait(false);
+                                }
+                                catch (OperationCanceledException)
+                                {
+                                    // expected when cancelling the read due to timeout
+                                }
+                                catch (Exception ex)
+                                {
+                                    Logger.LogDebug(ex, "Error finishing read task after stall detection for {requestUrl}", requestUrl);
+                                }
+
                                throw new TimeoutException($"No data received for {DownloadStallTimeout.TotalSeconds} seconds while downloading {requestUrl} (waiting: {snapshot.Waiting})");
                            }

@@ -353,7 +383,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
        }
    }

-    private async Task DecompressBlockFileAsync(string downloadStatusKey, string blockFilePath, List<FileReplacementData> fileReplacement, string downloadLabel)
+    private async Task DecompressBlockFileAsync(string downloadStatusKey, string blockFilePath, List<FileReplacementData> fileReplacement, string downloadLabel, bool skipDownscale)
    {
        if (_downloadStatus.TryGetValue(downloadStatusKey, out var status))
        {
@@ -386,7 +416,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                    var decompressedFile = LZ4Wrapper.Unwrap(compressedFileContent);
                    await _fileCompactor.WriteAllBytesAsync(filePath, decompressedFile, CancellationToken.None).ConfigureAwait(false);

-                    PersistFileToStorage(fileHash, filePath);
+                    var gamePath = fileReplacement.FirstOrDefault(f => string.Equals(f.Hash, fileHash, StringComparison.OrdinalIgnoreCase))?.GamePaths.FirstOrDefault() ?? string.Empty;
+                    PersistFileToStorage(fileHash, filePath, gamePath, skipDownscale);
                }
                catch (EndOfStreamException)
                {
@@ -414,7 +445,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
    }

    private async Task PerformDirectDownloadFallbackAsync(DownloadFileTransfer directDownload, List<FileReplacementData> fileReplacement,
-        IProgress<long> progress, CancellationToken token, bool slotAlreadyAcquired)
+        IProgress<long> progress, CancellationToken token, bool skipDownscale, bool slotAlreadyAcquired)
    {
        if (string.IsNullOrEmpty(directDownload.DirectDownloadUrl))
        {
@@ -456,7 +487,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                throw new FileNotFoundException("Block file missing after direct download fallback.", blockFile);
            }

-            await DecompressBlockFileAsync(downloadKey, blockFile, fileReplacement, $"fallback-{directDownload.Hash}").ConfigureAwait(false);
+            await DecompressBlockFileAsync(downloadKey, blockFile, fileReplacement, $"fallback-{directDownload.Hash}", skipDownscale).ConfigureAwait(false);
        }
        finally
        {
@@ -479,8 +510,9 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
        }
    }

-    public async Task<List<DownloadFileTransfer>> InitiateDownloadList(GameObjectHandler gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct)
+    public async Task<List<DownloadFileTransfer>> InitiateDownloadList(GameObjectHandler? gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct, Guid? ownerToken = null)
    {
+        CurrentOwnerToken = ownerToken;
        var objectName = gameObjectHandler?.Name ?? "Unknown";
        Logger.LogDebug("Download start: {id}", objectName);

@@ -521,7 +553,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
        return CurrentDownloads;
    }

-    private async Task DownloadFilesInternal(GameObjectHandler gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct)
+    private async Task DownloadFilesInternal(GameObjectHandler? gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct, bool skipDownscale)
    {
        var objectName = gameObjectHandler?.Name ?? "Unknown";

@@ -584,7 +616,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
            Logger.LogWarning("Downloading {direct} files directly, and {batchtotal} in {batches} batches.", directDownloads.Count, batchDownloads.Count, downloadBatches.Length);
        }

-        Mediator.Publish(new DownloadStartedMessage(gameObjectHandler, _downloadStatus));
+        if (gameObjectHandler is not null)
+        {
+            Mediator.Publish(new DownloadStartedMessage(gameObjectHandler, _downloadStatus));
+        }

        Task batchDownloadsTask = downloadBatches.Length == 0 ? Task.CompletedTask : Parallel.ForEachAsync(downloadBatches, new ParallelOptions()
        {
@@ -652,7 +687,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                    return;
                }

-                await DecompressBlockFileAsync(fileGroup.Key, blockFile, fileReplacement, fi.Name).ConfigureAwait(false);
+                await DecompressBlockFileAsync(fileGroup.Key, blockFile, fileReplacement, fi.Name, skipDownscale).ConfigureAwait(false);
            }
            finally
            {
@@ -691,14 +726,13 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase

            if (!ShouldUseDirectDownloads())
            {
-                await PerformDirectDownloadFallbackAsync(directDownload, fileReplacement, progress, token, slotAlreadyAcquired: false).ConfigureAwait(false);
+                await PerformDirectDownloadFallbackAsync(directDownload, fileReplacement, progress, token, skipDownscale, slotAlreadyAcquired: false).ConfigureAwait(false);
                return;
            }

            var tempFilename = _fileDbManager.GetCacheFilePath(directDownload.Hash, "bin");
            var slotAcquired = false;

-
            try
            {
                downloadTracker.DownloadStatus = DownloadStatus.WaitingForSlot;
@@ -728,7 +762,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                    byte[] compressedBytes = await File.ReadAllBytesAsync(tempFilename).ConfigureAwait(false);
                    var decompressedBytes = LZ4Wrapper.Unwrap(compressedBytes);
                    await _fileCompactor.WriteAllBytesAsync(finalFilename, decompressedBytes, CancellationToken.None).ConfigureAwait(false);
-                    PersistFileToStorage(directDownload.Hash, finalFilename);
+                    PersistFileToStorage(directDownload.Hash, finalFilename, replacement.GamePaths[0], skipDownscale);

                    downloadTracker.TransferredFiles = 1;
                    Logger.LogDebug("Finished direct download of {hash}.", directDownload.Hash);
@@ -740,8 +774,15 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
            }
            catch (OperationCanceledException ex)
            {
-                Logger.LogDebug("{hash}: Detected cancellation of direct download, discarding file.", directDownload.Hash);
-                Logger.LogError(ex, "{hash}: Error during direct download.", directDownload.Hash);
+                if (token.IsCancellationRequested)
+                {
+                    Logger.LogDebug("{hash}: Direct download cancelled by caller, discarding file.", directDownload.Hash);
+                }
+                else
+                {
+                    Logger.LogWarning(ex, "{hash}: Direct download cancelled unexpectedly.", directDownload.Hash);
+                }
+
                ClearDownload();
                return;
            }
@@ -763,7 +804,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                try
                {
                    downloadTracker.DownloadStatus = DownloadStatus.WaitingForQueue;
-                    await PerformDirectDownloadFallbackAsync(directDownload, fileReplacement, progress, token, slotAcquired).ConfigureAwait(false);
+                    await PerformDirectDownloadFallbackAsync(directDownload, fileReplacement, progress, token, skipDownscale, slotAcquired).ConfigureAwait(false);

                    if (!expectedDirectDownloadFailure && failureCount >= 3 && !_disableDirectDownloads)
                    {
@@ -816,7 +857,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
        return await response.Content.ReadFromJsonAsync<List<DownloadFileDto>>(cancellationToken: ct).ConfigureAwait(false) ?? [];
    }

-    private void PersistFileToStorage(string fileHash, string filePath)
+    private void PersistFileToStorage(string fileHash, string filePath, string gamePath, bool skipDownscale)
    {
        var fi = new FileInfo(filePath);
        Func<DateTime> RandomDayInThePast()
@@ -833,6 +874,11 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
        try
        {
            var entry = _fileDbManager.CreateCacheEntry(filePath);
+            var mapKind = _textureMetadataHelper.DetermineMapKind(gamePath, filePath);
+            if (!skipDownscale)
+            {
+                _textureDownscaleService.ScheduleDownscale(fileHash, filePath, mapKind);
+            }
            if (entry != null && !string.Equals(entry.Hash, fileHash, StringComparison.OrdinalIgnoreCase))
            {
                Logger.LogError("Hash mismatch after extracting, got {hash}, expected {expectedHash}, deleting file", entry.Hash, fileHash);