Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 52 Wiki Activity

changes

Browse Source
This commit is contained in:
Essie
2025-10-06 03:47:24 +09:00
parent 39784a1fea
commit c35650438c
22 changed files with 1480 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
LightlessSync/WebAPI/Files/FileDownloadManager.cs
View File

@@ -1,4 +1,4 @@
using Dalamud.Utility;
using Dalamud.Utility;
using K4os.Compression.LZ4.Legacy;
using LightlessSync.API.Data;
using LightlessSync.API.Dto.Files;
@@ -8,6 +8,7 @@ using LightlessSync.PlayerData.Handlers;
using LightlessSync.Services.Mediator;
using LightlessSync.WebAPI.Files.Models;
using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
using System.Net;
using System.Net.Http.Json;
@@ -19,7 +20,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
private readonly FileCompactor _fileCompactor;
private readonly FileCacheManager _fileDbManager;
private readonly FileTransferOrchestrator _orchestrator;
private readonly List<ThrottledStream> _activeDownloadStreams;
private readonly ConcurrentDictionary<ThrottledStream, byte> _activeDownloadStreams;
public FileDownloadManager(ILogger<FileDownloadManager> logger, LightlessMediator mediator,
FileTransferOrchestrator orchestrator,
@@ -29,14 +30,14 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
_orchestrator = orchestrator;
_fileDbManager = fileCacheManager;
_fileCompactor = fileCompactor;
_activeDownloadStreams = [];
_activeDownloadStreams = new();
Mediator.Subscribe<DownloadLimitChangedMessage>(this, (msg) =>
{
if (!_activeDownloadStreams.Any()) return;
if (_activeDownloadStreams.IsEmpty) return;
var newLimit = _orchestrator.DownloadLimitPerSlot();
Logger.LogTrace("Setting new Download Speed Limit to {newLimit}", newLimit);
foreach (var stream in _activeDownloadStreams)
foreach (var stream in _activeDownloadStreams.Keys)
{
stream.BandwidthLimit = newLimit;
}
@@ -47,7 +48,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
public List<FileTransfer> ForbiddenTransfers => _orchestrator.ForbiddenTransfers;
public bool IsDownloading => !CurrentDownloads.Any();
public bool IsDownloading => CurrentDownloads.Any();
public static void MungeBuffer(Span<byte> buffer)
{
@@ -84,7 +85,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
protected override void Dispose(bool disposing)
{
ClearDownload();
foreach (var stream in _activeDownloadStreams.ToList())
foreach (var stream in _activeDownloadStreams.Keys.ToList())
{
try
{
@@ -95,6 +96,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
// do nothing
//
}
finally
{
_activeDownloadStreams.TryRemove(stream, out _);
}
}
base.Dispose(disposing);
}
@@ -142,7 +147,14 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
await WaitForDownloadReady(fileTransfer, requestId, ct).ConfigureAwait(false);
_downloadStatus[downloadGroup].DownloadStatus = DownloadStatus.Downloading;
if (_downloadStatus.TryGetValue(downloadGroup, out var downloadStatus))
{
downloadStatus.DownloadStatus = DownloadStatus.Downloading;
}
else
{
Logger.LogWarning("Download status missing for {group} when starting download", downloadGroup);
}
const int maxRetries = 3;
int retryCount = 0;
@@ -204,7 +216,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
stream = new(await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false), limit);
_activeDownloadStreams.Add(stream);
_activeDownloadStreams.TryAdd(stream, 0);
while ((bytesRead = await stream.ReadAsync(buffer, ct).ConfigureAwait(false)) > 0)
{
@@ -245,7 +257,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
{
if (stream != null)
{
_activeDownloadStreams.Remove(stream);
_activeDownloadStreams.TryRemove(stream, out _);
await stream.DisposeAsync().ConfigureAwait(false);
}
}
@@ -253,11 +265,28 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
public async Task<List<DownloadFileTransfer>> InitiateDownloadList(GameObjectHandler gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct)
{
Logger.LogDebug("Download start: {id}", gameObjectHandler.Name);
var objectName = gameObjectHandler?.Name ?? "Unknown";
Logger.LogDebug("Download start: {id}", objectName);
if (fileReplacement == null || fileReplacement.Count == 0)
{
Logger.LogDebug("{dlName}: No file replacements provided", objectName);
CurrentDownloads = [];
return CurrentDownloads;
}
var hashes = fileReplacement.Where(f => f != null && !string.IsNullOrWhiteSpace(f.Hash)).Select(f => f.Hash).Distinct(StringComparer.Ordinal).ToList();
if (hashes.Count == 0)
{
Logger.LogDebug("{dlName}: No valid hashes to download", objectName);
CurrentDownloads = [];
return CurrentDownloads;
}
List<DownloadFileDto> downloadFileInfoFromService =
[
.. await FilesGetSizes(fileReplacement.Select(f => f.Hash).Distinct(StringComparer.Ordinal).ToList(), ct).ConfigureAwait(false),
.. await FilesGetSizes(hashes, ct).ConfigureAwait(false),
];
Logger.LogDebug("Files with size 0 or less: {files}", string.Join(", ", downloadFileInfoFromService.Where(f => f.Size <= 0).Select(f => f.Hash)));
@@ -315,15 +344,23 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
FileInfo fi = new(blockFile);
try
{
_downloadStatus[fileGroup.Key].DownloadStatus = DownloadStatus.WaitingForSlot;
if (!_downloadStatus.TryGetValue(fileGroup.Key, out var downloadStatus))
{
Logger.LogWarning("Download status missing for {group}, aborting", fileGroup.Key);
return;
}
downloadStatus.DownloadStatus = DownloadStatus.WaitingForSlot;
await _orchestrator.WaitForDownloadSlotAsync(token).ConfigureAwait(false);
_downloadStatus[fileGroup.Key].DownloadStatus = DownloadStatus.WaitingForQueue;
downloadStatus.DownloadStatus = DownloadStatus.WaitingForQueue;
Progress<long> progress = new((bytesDownloaded) =>
{
try
{
if (!_downloadStatus.TryGetValue(fileGroup.Key, out FileDownloadStatus? value)) return;
value.TransferredBytes += bytesDownloaded;
if (_downloadStatus.TryGetValue(fileGroup.Key, out FileDownloadStatus? value))
{
value.TransferredBytes += bytesDownloaded;
}
}
catch (Exception ex)
{
@@ -353,6 +390,12 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
status.TransferredFiles = 1;
status.DownloadStatus = DownloadStatus.Decompressing;
}
if (!File.Exists(blockFile))
{
Logger.LogWarning("{dlName}: Block file missing before extraction, skipping", fi.Name);
return;
}
fileBlockStream = File.OpenRead(blockFile);
while (fileBlockStream.Position < fileBlockStream.Length)
{
@@ -511,4 +554,6 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
_orchestrator.ClearDownloadRequest(requestId);
}
}
}
}