Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 51 Wiki Activity

Added decompression gate to decompress files

Browse Source
This commit is contained in:
cake
2025-12-29 02:50:49 +01:00
parent cf495dc826
commit 6d20995dbf
1 changed files with 40 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
LightlessSync/WebAPI/Files/FileDownloadManager.cs
View File

@@ -28,6 +28,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
private readonly TextureMetadataHelper _textureMetadataHelper;
private readonly ConcurrentDictionary<ThrottledStream, byte> _activeDownloadStreams;
private readonly SemaphoreSlim _decompressGate =
new(Math.Max(1, Environment.ProcessorCount / 2), Math.Max(1, Environment.ProcessorCount / 2));
private volatile bool _disableDirectDownloads;
private int _consecutiveDirectDownloadFailures;
@@ -522,32 +524,57 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
try
{
// sanity check length
if (fileLengthBytes < 0 || fileLengthBytes > int.MaxValue)
throw new InvalidDataException($"Invalid block entry length: {fileLengthBytes}");
// safe cast after check
var len = checked((int)fileLengthBytes);
if (!replacementLookup.TryGetValue(fileHash, out var repl))
{
Logger.LogWarning("{dlName}: No replacement mapping for {fileHash}", downloadLabel, fileHash);
// still need to skip bytes:
var skip = checked((int)fileLengthBytes);
fileBlockStream.Position += skip;
fileBlockStream.Seek(len, SeekOrigin.Current);
continue;
}
// decompress
var filePath = _fileDbManager.GetCacheFilePath(fileHash, repl.Extension);
Logger.LogTrace("{dlName}: Decompressing {file}:{len} => {dest}", downloadLabel, fileHash, fileLengthBytes, filePath);
Logger.LogDebug("{dlName}: Decompressing {file}:{len} => {dest}", downloadLabel, fileHash, fileLengthBytes, filePath);
var len = checked((int)fileLengthBytes);
// read compressed data
var compressed = new byte[len];
await ReadExactlyAsync(fileBlockStream, compressed.AsMemory(0, len), ct).ConfigureAwait(false);
MungeBuffer(compressed);
var decompressed = LZ4Wrapper.Unwrap(compressed);
if (len == 0)
{
await _fileCompactor.WriteAllBytesAsync(filePath, Array.Empty<byte>(), ct).ConfigureAwait(false);
PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale);
continue;
}
await _fileCompactor.WriteAllBytesAsync(filePath, decompressed, ct).ConfigureAwait(false);
PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale);
MungeBuffer(compressed);
// limit concurrent decompressions
await _decompressGate.WaitAsync(ct).ConfigureAwait(false);
try
{
var sw = System.Diagnostics.Stopwatch.StartNew();
// decompress
var decompressed = LZ4Wrapper.Unwrap(compressed);
Logger.LogTrace("{dlName}: Unwrap {fileHash} took {ms}ms (compressed {c} bytes, decompressed {d} bytes)",
downloadLabel, fileHash, sw.ElapsedMilliseconds, compressed.Length, decompressed?.Length ?? -1);
// write to file
await _fileCompactor.WriteAllBytesAsync(filePath, decompressed, ct).ConfigureAwait(false);
PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale);
}
finally
{
_decompressGate.Release();
}
}
catch (EndOfStreamException)
{
@@ -605,20 +632,16 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
.. await FilesGetSizes(hashes, ct).ConfigureAwait(false),
];
Logger.LogDebug("Files with size 0 or less: {files}",
string.Join(", ", downloadFileInfoFromService.Where(f => f.Size <= 0).Select(f => f.Hash)));
foreach (var dto in downloadFileInfoFromService.Where(c => c.IsForbidden))
{
if (!_orchestrator.ForbiddenTransfers.Exists(f => string.Equals(f.Hash, dto.Hash, StringComparison.Ordinal)))
_orchestrator.ForbiddenTransfers.Add(new DownloadFileTransfer(dto));
}
CurrentDownloads = downloadFileInfoFromService
CurrentDownloads = [.. downloadFileInfoFromService
.Distinct()
.Select(d => new DownloadFileTransfer(d))
.Where(d => d.CanBeTransferred)
.ToList();
.Where(d => d.CanBeTransferred)];
return CurrentDownloads;
}