init

LightlessSync/WebAPI/Files/FileDownloadManager.cs

@@ -5,12 +5,18 @@ using LightlessSync.API.Dto.Files;
 using LightlessSync.API.Routes;
 using LightlessSync.FileCache;
 using LightlessSync.PlayerData.Handlers;
+using LightlessSync.Services;
 using LightlessSync.Services.Mediator;
 using LightlessSync.WebAPI.Files.Models;
 using Microsoft.Extensions.Logging;
+using System;
 using System.Collections.Concurrent;
+using System.IO;
 using System.Net;
 using System.Net.Http.Json;
+using System.Threading;
+using System.Threading.Tasks;
+using LightlessSync.LightlessConfiguration;
 
 namespace LightlessSync.WebAPI.Files;
 
@@ -20,17 +26,27 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
     private readonly FileCompactor _fileCompactor;
     private readonly FileCacheManager _fileDbManager;
     private readonly FileTransferOrchestrator _orchestrator;
+    private readonly PairProcessingLimiter _pairProcessingLimiter;
+    private readonly LightlessConfigService _configService;
     private readonly ConcurrentDictionary<ThrottledStream, byte> _activeDownloadStreams;
+    private static readonly TimeSpan DownloadStallTimeout = TimeSpan.FromSeconds(30);
+    private volatile bool _disableDirectDownloads;
+    private int _consecutiveDirectDownloadFailures;
+    private bool _lastConfigDirectDownloadsState;
 
     public FileDownloadManager(ILogger<FileDownloadManager> logger, LightlessMediator mediator,
         FileTransferOrchestrator orchestrator,
-        FileCacheManager fileCacheManager, FileCompactor fileCompactor) : base(logger, mediator)
+        FileCacheManager fileCacheManager, FileCompactor fileCompactor,
+        PairProcessingLimiter pairProcessingLimiter, LightlessConfigService configService) : base(logger, mediator)
     {
         _downloadStatus = new Dictionary<string, FileDownloadStatus>(StringComparer.Ordinal);
         _orchestrator = orchestrator;
         _fileDbManager = fileCacheManager;
         _fileCompactor = fileCompactor;
+        _pairProcessingLimiter = pairProcessingLimiter;
+        _configService = configService;
         _activeDownloadStreams = new();
+        _lastConfigDirectDownloadsState = _configService.Current.EnableDirectDownloads;
 
         Mediator.Subscribe<DownloadLimitChangedMessage>(this, (msg) =>
         {
@@ -50,6 +66,11 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
 
     public bool IsDownloading => CurrentDownloads.Any();
 
+    private bool ShouldUseDirectDownloads()
+    {
+        return _configService.Current.EnableDirectDownloads && !_disableDirectDownloads;
+    }
+
     public static void MungeBuffer(Span<byte> buffer)
     {
         for (int i = 0; i < buffer.Length; ++i)
@@ -156,39 +177,47 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
             Logger.LogWarning("Download status missing for {group} when starting download", downloadGroup);
         }
 
+        var requestUrl = LightlessFiles.CacheGetFullPath(fileTransfer[0].DownloadUri, requestId);
+
+        await DownloadFileThrottled(requestUrl, tempPath, progress, MungeBuffer, ct, withToken: true).ConfigureAwait(false);
+    }
+
+    private delegate void DownloadDataCallback(Span<byte> data);
+
+    private async Task DownloadFileThrottled(Uri requestUrl, string destinationFilename, IProgress<long> progress, DownloadDataCallback? callback, CancellationToken ct, bool withToken)
+    {
         const int maxRetries = 3;
         int retryCount = 0;
         TimeSpan retryDelay = TimeSpan.FromSeconds(2);
-
-        HttpResponseMessage response = null!;
-        var requestUrl = LightlessFiles.CacheGetFullPath(fileTransfer[0].DownloadUri, requestId);
+        HttpResponseMessage? response = null;
 
         while (true)
         {
             try
             {
-                Logger.LogDebug("Attempt {attempt} - Downloading {requestUrl} for request {id}", retryCount + 1, requestUrl, requestId);
-
-                response = await _orchestrator.SendRequestAsync(HttpMethod.Get, requestUrl, ct, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
+                Logger.LogDebug("Attempt {attempt} - Downloading {requestUrl}", retryCount + 1, requestUrl);
+                response = await _orchestrator.SendRequestAsync(HttpMethod.Get, requestUrl, ct, HttpCompletionOption.ResponseHeadersRead, withToken).ConfigureAwait(false);
                 response.EnsureSuccessStatusCode();
                 break;
             }
             catch (HttpRequestException ex) when (ex.InnerException is TimeoutException || ex.StatusCode == null)
             {
+                response?.Dispose();
                 retryCount++;
 
                 Logger.LogWarning(ex, "Timeout during download of {requestUrl}. Attempt {attempt} of {maxRetries}", requestUrl, retryCount, maxRetries);
 
                 if (retryCount >= maxRetries || ct.IsCancellationRequested)
                 {
-                    Logger.LogError($"Max retries reached or cancelled. Failing download for {requestUrl}");
+                    Logger.LogError("Max retries reached or cancelled. Failing download for {requestUrl}", requestUrl);
                     throw;
                 }
 
-                await Task.Delay(retryDelay, ct).ConfigureAwait(false); // Wait before retrying
+                await Task.Delay(retryDelay, ct).ConfigureAwait(false);
             }
             catch (HttpRequestException ex)
             {
+                response?.Dispose();
                 Logger.LogWarning(ex, "Error during download of {requestUrl}, HttpStatusCode: {code}", requestUrl, ex.StatusCode);
 
                 if (ex.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Unauthorized)
@@ -196,42 +225,80 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                     throw new InvalidDataException($"Http error {ex.StatusCode} (cancelled: {ct.IsCancellationRequested}): {requestUrl}", ex);
                 }
 
-                throw; 
+                throw;
             }
         }
+
         ThrottledStream? stream = null;
         FileStream? fileStream = null;
 
         try
         {
-            fileStream = File.Create(tempPath);
+            fileStream = File.Create(destinationFilename);
             await using (fileStream.ConfigureAwait(false))
             {
-                var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 65536 : 8196;
+                var bufferSize = response!.Content.Headers.ContentLength > 1024 * 1024 ? 65536 : 8196;
                 var buffer = new byte[bufferSize];
 
-                var bytesRead = 0;
                 var limit = _orchestrator.DownloadLimitPerSlot();
-                Logger.LogTrace("Starting Download of {id} with a speed limit of {limit} to {tempPath}", requestId, limit, tempPath);
+                Logger.LogTrace("Starting Download with a speed limit of {limit} to {destination}", limit, destinationFilename);
 
                 stream = new(await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false), limit);
-
                 _activeDownloadStreams.TryAdd(stream, 0);
 
-                while ((bytesRead = await stream.ReadAsync(buffer, ct).ConfigureAwait(false)) > 0)
+                while (true)
                 {
                     ct.ThrowIfCancellationRequested();
+                    int bytesRead;
+                    try
+                    {
+                        var readTask = stream.ReadAsync(buffer.AsMemory(0, buffer.Length), ct).AsTask();
+                        while (!readTask.IsCompleted)
+                        {
+                            var completedTask = await Task.WhenAny(readTask, Task.Delay(DownloadStallTimeout)).ConfigureAwait(false);
+                            if (completedTask == readTask)
+                            {
+                                break;
+                            }
 
-                    MungeBuffer(buffer.AsSpan(0, bytesRead));
+                            ct.ThrowIfCancellationRequested();
+
+                            var snapshot = _pairProcessingLimiter.GetSnapshot();
+                            if (snapshot.Waiting > 0)
+                            {
+                                throw new TimeoutException($"No data received for {DownloadStallTimeout.TotalSeconds} seconds while downloading {requestUrl} (waiting: {snapshot.Waiting})");
+                            }
+
+                            Logger.LogTrace("Download stalled for {requestUrl} but no queued pairs, continuing to wait", requestUrl);
+                        }
+
+                        bytesRead = await readTask.ConfigureAwait(false);
+                    }
+                    catch (OperationCanceledException)
+                    {
+                        throw;
+                    }
+
+                    if (bytesRead == 0)
+                    {
+                        break;
+                    }
+
+                    callback?.Invoke(buffer.AsSpan(0, bytesRead));
 
                     await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead), ct).ConfigureAwait(false);
 
                     progress.Report(bytesRead);
                 }
 
-                Logger.LogDebug("{requestUrl} downloaded to {tempPath}", requestUrl, tempPath);
+                Logger.LogDebug("{requestUrl} downloaded to {destination}", requestUrl, destinationFilename);
             }
         }
+        catch (TimeoutException ex)
+        {
+            Logger.LogWarning(ex, "Detected stalled download for {requestUrl}, aborting transfer", requestUrl);
+            throw;
+        }
         catch (OperationCanceledException)
         {
             throw;
@@ -240,18 +307,18 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
         {
             try
             {
-                fileStream?.Close(); 
+                fileStream?.Close();
 
-                if (!string.IsNullOrEmpty(tempPath) && File.Exists(tempPath))
+                if (!string.IsNullOrEmpty(destinationFilename) && File.Exists(destinationFilename))
                 {
-                    File.Delete(tempPath);
+                    File.Delete(destinationFilename);
                 }
             }
             catch
             {
-                // Ignore errors during cleanup
+                // ignore cleanup errors
             }
-            throw; 
+            throw;
         }
         finally
         {
@@ -260,6 +327,134 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                 _activeDownloadStreams.TryRemove(stream, out _);
                 await stream.DisposeAsync().ConfigureAwait(false);
             }
+
+            response?.Dispose();
+        }
+    }
+
+    private async Task DecompressBlockFileAsync(string downloadStatusKey, string blockFilePath, List<FileReplacementData> fileReplacement, string downloadLabel)
+    {
+        if (_downloadStatus.TryGetValue(downloadStatusKey, out var status))
+        {
+            status.TransferredFiles = 1;
+            status.DownloadStatus = DownloadStatus.Decompressing;
+        }
+
+        FileStream? fileBlockStream = null;
+        try
+        {
+            fileBlockStream = File.OpenRead(blockFilePath);
+            while (fileBlockStream.Position < fileBlockStream.Length)
+            {
+                (string fileHash, long fileLengthBytes) = ReadBlockFileHeader(fileBlockStream);
+
+                try
+                {
+                    var fileExtension = fileReplacement.First(f => string.Equals(f.Hash, fileHash, StringComparison.OrdinalIgnoreCase)).GamePaths[0].Split(".")[^1];
+                    var filePath = _fileDbManager.GetCacheFilePath(fileHash, fileExtension);
+                    Logger.LogDebug("{dlName}: Decompressing {file}:{le} => {dest}", downloadLabel, fileHash, fileLengthBytes, filePath);
+
+                    byte[] compressedFileContent = new byte[fileLengthBytes];
+                    var readBytes = await fileBlockStream.ReadAsync(compressedFileContent, CancellationToken.None).ConfigureAwait(false);
+                    if (readBytes != fileLengthBytes)
+                    {
+                        throw new EndOfStreamException();
+                    }
+                    MungeBuffer(compressedFileContent);
+
+                    var decompressedFile = LZ4Wrapper.Unwrap(compressedFileContent);
+                    await _fileCompactor.WriteAllBytesAsync(filePath, decompressedFile, CancellationToken.None).ConfigureAwait(false);
+
+                    PersistFileToStorage(fileHash, filePath);
+                }
+                catch (EndOfStreamException)
+                {
+                    Logger.LogWarning("{dlName}: Failure to extract file {fileHash}, stream ended prematurely", downloadLabel, fileHash);
+                }
+                catch (Exception e)
+                {
+                    Logger.LogWarning(e, "{dlName}: Error during decompression", downloadLabel);
+                }
+            }
+        }
+        catch (EndOfStreamException)
+        {
+            Logger.LogDebug("{dlName}: Failure to extract file header data, stream ended", downloadLabel);
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "{dlName}: Error during block file read", downloadLabel);
+        }
+        finally
+        {
+            if (fileBlockStream != null)
+                await fileBlockStream.DisposeAsync().ConfigureAwait(false);
+        }
+    }
+
+    private async Task PerformDirectDownloadFallbackAsync(DownloadFileTransfer directDownload, List<FileReplacementData> fileReplacement,
+        IProgress<long> progress, CancellationToken token, bool slotAlreadyAcquired)
+    {
+        if (string.IsNullOrEmpty(directDownload.DirectDownloadUrl))
+        {
+            throw new InvalidOperationException("Direct download fallback requested without a direct download URL.");
+        }
+
+        var downloadKey = directDownload.DirectDownloadUrl!;
+        bool slotAcquiredHere = false;
+        string? blockFile = null;
+
+        try
+        {
+            if (!slotAlreadyAcquired)
+            {
+                if (_downloadStatus.TryGetValue(downloadKey, out var tracker))
+                {
+                    tracker.DownloadStatus = DownloadStatus.WaitingForSlot;
+                }
+
+                await _orchestrator.WaitForDownloadSlotAsync(token).ConfigureAwait(false);
+                slotAcquiredHere = true;
+            }
+
+            if (_downloadStatus.TryGetValue(downloadKey, out var queueTracker))
+            {
+                queueTracker.DownloadStatus = DownloadStatus.WaitingForQueue;
+            }
+
+            var requestIdResponse = await _orchestrator.SendRequestAsync(HttpMethod.Post, LightlessFiles.RequestEnqueueFullPath(directDownload.DownloadUri),
+                new[] { directDownload.Hash }, token).ConfigureAwait(false);
+            var requestId = Guid.Parse((await requestIdResponse.Content.ReadAsStringAsync().ConfigureAwait(false)).Trim('"'));
+
+            blockFile = _fileDbManager.GetCacheFilePath(requestId.ToString("N"), "blk");
+
+            await DownloadAndMungeFileHttpClient(downloadKey, requestId, [directDownload], blockFile, progress, token).ConfigureAwait(false);
+
+            if (!File.Exists(blockFile))
+            {
+                throw new FileNotFoundException("Block file missing after direct download fallback.", blockFile);
+            }
+
+            await DecompressBlockFileAsync(downloadKey, blockFile, fileReplacement, $"fallback-{directDownload.Hash}").ConfigureAwait(false);
+        }
+        finally
+        {
+            if (slotAcquiredHere)
+            {
+                _orchestrator.ReleaseDownloadSlot();
+            }
+
+            if (!string.IsNullOrEmpty(blockFile))
+            {
+                try
+                {
+                    File.Delete(blockFile);
+                }
+                catch
+                {
+                    // ignore cleanup errors
+                }
+            }
         }
     }
 
@@ -307,30 +502,76 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
 
     private async Task DownloadFilesInternal(GameObjectHandler gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct)
     {
-        var downloadGroups = CurrentDownloads.GroupBy(f => f.DownloadUri.Host + ":" + f.DownloadUri.Port, StringComparer.Ordinal);
+        var objectName = gameObjectHandler?.Name ?? "Unknown";
 
-        foreach (var downloadGroup in downloadGroups)
+        var configAllowsDirect = _configService.Current.EnableDirectDownloads;
+        if (configAllowsDirect != _lastConfigDirectDownloadsState)
         {
-            _downloadStatus[downloadGroup.Key] = new FileDownloadStatus()
+            _lastConfigDirectDownloadsState = configAllowsDirect;
+            if (configAllowsDirect)
+            {
+                _disableDirectDownloads = false;
+                _consecutiveDirectDownloadFailures = 0;
+            }
+        }
+
+        var allowDirectDownloads = ShouldUseDirectDownloads();
+
+        var directDownloads = new List<DownloadFileTransfer>();
+        var batchDownloads = new List<DownloadFileTransfer>();
+
+        foreach (var download in CurrentDownloads)
+        {
+            if (!string.IsNullOrEmpty(download.DirectDownloadUrl) && allowDirectDownloads)
+            {
+                directDownloads.Add(download);
+            }
+            else
+            {
+                batchDownloads.Add(download);
+            }
+        }
+
+        var downloadBatches = batchDownloads.GroupBy(f => f.DownloadUri.Host + ":" + f.DownloadUri.Port, StringComparer.Ordinal).ToArray();
+
+        foreach (var directDownload in directDownloads)
+        {
+            _downloadStatus[directDownload.DirectDownloadUrl!] = new FileDownloadStatus()
             {
                 DownloadStatus = DownloadStatus.Initializing,
-                TotalBytes = downloadGroup.Sum(c => c.Total),
+                TotalBytes = directDownload.Total,
                 TotalFiles = 1,
                 TransferredBytes = 0,
                 TransferredFiles = 0
             };
         }
 
+        foreach (var downloadBatch in downloadBatches)
+        {
+            _downloadStatus[downloadBatch.Key] = new FileDownloadStatus()
+            {
+                DownloadStatus = DownloadStatus.Initializing,
+                TotalBytes = downloadBatch.Sum(c => c.Total),
+                TotalFiles = 1,
+                TransferredBytes = 0,
+                TransferredFiles = 0
+            };
+        }
+
+        if (directDownloads.Count > 0 || downloadBatches.Length > 0)
+        {
+            Logger.LogWarning("Downloading {direct} files directly, and {batchtotal} in {batches} batches.", directDownloads.Count, batchDownloads.Count, downloadBatches.Length);
+        }
+
         Mediator.Publish(new DownloadStartedMessage(gameObjectHandler, _downloadStatus));
 
-        await Parallel.ForEachAsync(downloadGroups, new ParallelOptions()
+        Task batchDownloadsTask = downloadBatches.Length == 0 ? Task.CompletedTask : Parallel.ForEachAsync(downloadBatches, new ParallelOptions()
         {
-            MaxDegreeOfParallelism = downloadGroups.Count(),
+            MaxDegreeOfParallelism = downloadBatches.Length,
             CancellationToken = ct,
         },
         async (fileGroup, token) =>
         {
-            // let server predownload files
             var requestIdResponse = await _orchestrator.SendRequestAsync(HttpMethod.Post, LightlessFiles.RequestEnqueueFullPath(fileGroup.First().DownloadUri),
                 fileGroup.Select(c => c.Hash), token).ConfigureAwait(false);
             Logger.LogDebug("Sent request for {n} files on server {uri} with result {result}", fileGroup.Count(), fileGroup.First().DownloadUri,
@@ -353,7 +594,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                 downloadStatus.DownloadStatus = DownloadStatus.WaitingForSlot;
                 await _orchestrator.WaitForDownloadSlotAsync(token).ConfigureAwait(false);
                 downloadStatus.DownloadStatus = DownloadStatus.WaitingForQueue;
-                Progress<long> progress = new((bytesDownloaded) =>
+                var progress = CreateInlineProgress((bytesDownloaded) =>
                 {
                     try
                     {
@@ -371,7 +612,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
             }
             catch (OperationCanceledException)
             {
-                Logger.LogDebug("{dlName}: Detected cancellation of download, partially extracting files for {id}", fi.Name, gameObjectHandler);
+                Logger.LogDebug("{dlName}: Detected cancellation of download, partially extracting files for {id}", fi.Name, objectName);
             }
             catch (Exception ex)
             {
@@ -382,72 +623,167 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                 return;
             }
 
-            FileStream? fileBlockStream = null;
             try
             {
-                if (_downloadStatus.TryGetValue(fileGroup.Key, out var status))
-                {
-                    status.TransferredFiles = 1;
-                    status.DownloadStatus = DownloadStatus.Decompressing;
-                }
                 if (!File.Exists(blockFile))
                 {
                     Logger.LogWarning("{dlName}: Block file missing before extraction, skipping", fi.Name);
                     return;
                 }
 
-                fileBlockStream = File.OpenRead(blockFile);
-                while (fileBlockStream.Position < fileBlockStream.Length)
-                {
-                    (string fileHash, long fileLengthBytes) = ReadBlockFileHeader(fileBlockStream);
-
-                    try
-                    {
-                        var fileExtension = fileReplacement.First(f => string.Equals(f.Hash, fileHash, StringComparison.OrdinalIgnoreCase)).GamePaths[0].Split(".")[^1];
-                        var filePath = _fileDbManager.GetCacheFilePath(fileHash, fileExtension);
-                        Logger.LogDebug("{dlName}: Decompressing {file}:{le} => {dest}", fi.Name, fileHash, fileLengthBytes, filePath);
-
-                        byte[] compressedFileContent = new byte[fileLengthBytes];
-                        var readBytes = await fileBlockStream.ReadAsync(compressedFileContent, CancellationToken.None).ConfigureAwait(false);
-                        if (readBytes != fileLengthBytes)
-                        {
-                            throw new EndOfStreamException();
-                        }
-                        MungeBuffer(compressedFileContent);
-
-                        var decompressedFile = LZ4Wrapper.Unwrap(compressedFileContent);
-                        await _fileCompactor.WriteAllBytesAsync(filePath, decompressedFile, CancellationToken.None).ConfigureAwait(false);
-
-                        PersistFileToStorage(fileHash, filePath);
-                    }
-                    catch (EndOfStreamException)
-                    {
-                        Logger.LogWarning("{dlName}: Failure to extract file {fileHash}, stream ended prematurely", fi.Name, fileHash);
-                    }
-                    catch (Exception e)
-                    {
-                        Logger.LogWarning(e, "{dlName}: Error during decompression", fi.Name);
-                    }
-                }
-            }
-            catch (EndOfStreamException)
-            {
-                Logger.LogDebug("{dlName}: Failure to extract file header data, stream ended", fi.Name);
-            }
-            catch (Exception ex)
-            {
-                Logger.LogError(ex, "{dlName}: Error during block file read", fi.Name);
+                await DecompressBlockFileAsync(fileGroup.Key, blockFile, fileReplacement, fi.Name).ConfigureAwait(false);
             }
             finally
             {
                 _orchestrator.ReleaseDownloadSlot();
-                if (fileBlockStream != null)
-                    await fileBlockStream.DisposeAsync().ConfigureAwait(false);
                 File.Delete(blockFile);
             }
-        }).ConfigureAwait(false);
+        });
 
-        Logger.LogDebug("Download end: {id}", gameObjectHandler);
+        Task directDownloadsTask = directDownloads.Count == 0 ? Task.CompletedTask : Parallel.ForEachAsync(directDownloads, new ParallelOptions()
+        {
+            MaxDegreeOfParallelism = directDownloads.Count,
+            CancellationToken = ct,
+        },
+        async (directDownload, token) =>
+        {
+            if (!_downloadStatus.TryGetValue(directDownload.DirectDownloadUrl!, out var downloadTracker))
+            {
+                Logger.LogWarning("Download status missing for direct URL {url}", directDownload.DirectDownloadUrl);
+                return;
+            }
+
+            var progress = CreateInlineProgress((bytesDownloaded) =>
+            {
+                try
+                {
+                    if (_downloadStatus.TryGetValue(directDownload.DirectDownloadUrl!, out FileDownloadStatus? value))
+                    {
+                        value.TransferredBytes += bytesDownloaded;
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Logger.LogWarning(ex, "Could not set download progress");
+                }
+            });
+
+            if (!ShouldUseDirectDownloads())
+            {
+                await PerformDirectDownloadFallbackAsync(directDownload, fileReplacement, progress, token, slotAlreadyAcquired: false).ConfigureAwait(false);
+                return;
+            }
+
+            var tempFilename = _fileDbManager.GetCacheFilePath(directDownload.Hash, "bin");
+            var slotAcquired = false;
+
+
+            try
+            {
+                downloadTracker.DownloadStatus = DownloadStatus.WaitingForSlot;
+                await _orchestrator.WaitForDownloadSlotAsync(token).ConfigureAwait(false);
+                slotAcquired = true;
+
+                downloadTracker.DownloadStatus = DownloadStatus.Downloading;
+                Logger.LogDebug("Beginning direct download of {hash} from {url}", directDownload.Hash, directDownload.DirectDownloadUrl);
+                await DownloadFileThrottled(new Uri(directDownload.DirectDownloadUrl!), tempFilename, progress, null, token, withToken: false).ConfigureAwait(false);
+
+                Interlocked.Exchange(ref _consecutiveDirectDownloadFailures, 0);
+
+                downloadTracker.DownloadStatus = DownloadStatus.Decompressing;
+
+                try
+                {
+                    var replacement = fileReplacement.FirstOrDefault(f => string.Equals(f.Hash, directDownload.Hash, StringComparison.OrdinalIgnoreCase));
+                    if (replacement == null || replacement.GamePaths.Length == 0)
+                    {
+                        Logger.LogWarning("{hash}: No replacement data found for direct download.", directDownload.Hash);
+                        return;
+                    }
+
+                    var fileExtension = replacement.GamePaths[0].Split(".")[^1];
+                    var finalFilename = _fileDbManager.GetCacheFilePath(directDownload.Hash, fileExtension);
+                    Logger.LogDebug("Decompressing direct download {hash} from {compressedFile} to {finalFile}", directDownload.Hash, tempFilename, finalFilename);
+                    byte[] compressedBytes = await File.ReadAllBytesAsync(tempFilename).ConfigureAwait(false);
+                    var decompressedBytes = LZ4Wrapper.Unwrap(compressedBytes);
+                    await _fileCompactor.WriteAllBytesAsync(finalFilename, decompressedBytes, CancellationToken.None).ConfigureAwait(false);
+                    PersistFileToStorage(directDownload.Hash, finalFilename);
+
+                    downloadTracker.TransferredFiles = 1;
+                    Logger.LogDebug("Finished direct download of {hash}.", directDownload.Hash);
+                }
+                catch (Exception ex)
+                {
+                    Logger.LogError(ex, "Exception downloading {hash} from {url}", directDownload.Hash, directDownload.DirectDownloadUrl);
+                }
+            }
+            catch (OperationCanceledException ex)
+            {
+                Logger.LogDebug("{hash}: Detected cancellation of direct download, discarding file.", directDownload.Hash);
+                Logger.LogError(ex, "{hash}: Error during direct download.", directDownload.Hash);
+                ClearDownload();
+                return;
+            }
+            catch (Exception ex)
+            {
+                var expectedDirectDownloadFailure = ex is InvalidDataException;
+                var failureCount = 0;
+
+                if (expectedDirectDownloadFailure)
+                {
+                    Logger.LogInformation(ex, "{hash}: Direct download unavailable, attempting queued fallback.", directDownload.Hash);
+                }
+                else
+                {
+                    failureCount = Interlocked.Increment(ref _consecutiveDirectDownloadFailures);
+                    Logger.LogWarning(ex, "{hash}: Direct download failed, attempting queued fallback.", directDownload.Hash);
+                }
+
+                try
+                {
+                    downloadTracker.DownloadStatus = DownloadStatus.WaitingForQueue;
+                    await PerformDirectDownloadFallbackAsync(directDownload, fileReplacement, progress, token, slotAcquired).ConfigureAwait(false);
+
+                    if (!expectedDirectDownloadFailure && failureCount >= 3 && !_disableDirectDownloads)
+                    {
+                        _disableDirectDownloads = true;
+                        Logger.LogWarning("Disabling direct downloads for this session after {count} consecutive failures.", failureCount);
+                    }
+                }
+                catch (Exception fallbackEx)
+                {
+                    if (slotAcquired)
+                    {
+                        _orchestrator.ReleaseDownloadSlot();
+                        slotAcquired = false;
+                    }
+
+                    Logger.LogError(fallbackEx, "{hash}: Error during direct download fallback.", directDownload.Hash);
+                    ClearDownload();
+                    return;
+                }
+            }
+            finally
+            {
+                if (slotAcquired)
+                {
+                    _orchestrator.ReleaseDownloadSlot();
+                }
+
+                try
+                {
+                    File.Delete(tempFilename);
+                }
+                catch
+                {
+                    // ignore
+                }
+            }
+        });
+
+        await Task.WhenAll(batchDownloadsTask, directDownloadsTask).ConfigureAwait(false);
+
+        Logger.LogDebug("Download end: {id}", objectName);
 
         ClearDownload();
     }
@@ -554,4 +890,24 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
             _orchestrator.ClearDownloadRequest(requestId);
         }
     }
-}
+
+    private static IProgress<long> CreateInlineProgress(Action<long> callback)
+    {
+        return new InlineProgress(callback);
+    }
+
+    private sealed class InlineProgress : IProgress<long>
+    {
+        private readonly Action<long> _callback;
+
+        public InlineProgress(Action<long> callback)
+        {
+            _callback = callback ?? throw new ArgumentNullException(nameof(callback));
+        }
+
+        public void Report(long value)
+        {
+            _callback(value);
+        }
+    }
+}