2.0.0 (#92)
All checks were successful
Tag and Release Lightless / tag-and-release (push) Successful in 2m27s

2.0.0 Changes:

- Reworked shell finder UI with compact or list view with profile tags showing with the listing, allowing moderators to broadcast the syncshell as well to have it be used more.
- Reworked user list in syncshell admin screen to have filter visible and moved away from table to its own thing, allowing to copy uid/note/alias when clicking on the name.
- Reworked download bars and download box to make it look more modern, removed the jitter around, so it shouldn't vibrate around much.
- Chat has been added to the top menu, working in Zone or in Syncshells to be used there.
- Paired system has been revamped to make pausing and unpausing faster, and loading people should be faster as well.
- Moved to the internal object table to have faster load times for users; people should load in faster
- Compactor is running on a multi-threaded level instead of single-threaded; this should increase the speed of compacting files
- Nameplate Service has been reworked so it wouldn't use the nameplate handler anymore.
- Files can be resized when downloading to reduce load on users if they aren't compressed. (can be toggled to resize all).
- Penumbra Collections are now only made when people are visible, reducing the load on boot-up when having many syncshells in your list.
- Lightfinder plates have been moved away from using Nameplates, but will use an overlay.
- Main UI has been changed a bit with a gradient, and on hover will glow up now.
- Reworked Profile UI for Syncshell and Users to be more user-facing with more customizable items.
- Reworked Settings UI to look more modern.
- Performance should be better due to new systems that would dispose of the collections and better caching of items.

Co-authored-by: defnotken <itsdefnotken@gmail.com>
Co-authored-by: azyges <aaaaaa@aaa.aaa>
Co-authored-by: choco <choco@patat.nl>
Co-authored-by: cake <admin@cakeandbanana.nl>
Co-authored-by: Minmoose <KennethBohr@outlook.com>
Reviewed-on: #92
This commit was merged in pull request #92.
This commit is contained in:
2025-12-21 17:19:34 +00:00
parent 906f401940
commit 835a0a637d
191 changed files with 32636 additions and 8841 deletions

View File

@@ -1,21 +1,16 @@
using Dalamud.Utility;
using K4os.Compression.LZ4.Legacy;
using LightlessSync.API.Data;
using LightlessSync.API.Dto.Files;
using LightlessSync.API.Routes;
using LightlessSync.FileCache;
using LightlessSync.PlayerData.Handlers;
using LightlessSync.Services;
using LightlessSync.Services.Mediator;
using LightlessSync.Services.TextureCompression;
using LightlessSync.WebAPI.Files.Models;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Concurrent;
using System.IO;
using System.Net;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
using LightlessSync.LightlessConfiguration;
namespace LightlessSync.WebAPI.Files;
@@ -26,25 +21,30 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
private readonly FileCompactor _fileCompactor;
private readonly FileCacheManager _fileDbManager;
private readonly FileTransferOrchestrator _orchestrator;
private readonly PairProcessingLimiter _pairProcessingLimiter;
private readonly LightlessConfigService _configService;
private readonly TextureDownscaleService _textureDownscaleService;
private readonly TextureMetadataHelper _textureMetadataHelper;
private readonly ConcurrentDictionary<ThrottledStream, byte> _activeDownloadStreams;
private static readonly TimeSpan DownloadStallTimeout = TimeSpan.FromSeconds(30);
private volatile bool _disableDirectDownloads;
private int _consecutiveDirectDownloadFailures;
private bool _lastConfigDirectDownloadsState;
public FileDownloadManager(ILogger<FileDownloadManager> logger, LightlessMediator mediator,
public FileDownloadManager(
ILogger<FileDownloadManager> logger,
LightlessMediator mediator,
FileTransferOrchestrator orchestrator,
FileCacheManager fileCacheManager, FileCompactor fileCompactor,
PairProcessingLimiter pairProcessingLimiter, LightlessConfigService configService) : base(logger, mediator)
FileCacheManager fileCacheManager,
FileCompactor fileCompactor,
LightlessConfigService configService,
TextureDownscaleService textureDownscaleService, TextureMetadataHelper textureMetadataHelper) : base(logger, mediator)
{
_downloadStatus = new Dictionary<string, FileDownloadStatus>(StringComparer.Ordinal);
_orchestrator = orchestrator;
_fileDbManager = fileCacheManager;
_fileCompactor = fileCompactor;
_pairProcessingLimiter = pairProcessingLimiter;
_configService = configService;
_textureDownscaleService = textureDownscaleService;
_textureMetadataHelper = textureMetadataHelper;
_activeDownloadStreams = new();
_lastConfigDirectDownloadsState = _configService.Current.EnableDirectDownloads;
@@ -63,6 +63,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
public List<DownloadFileTransfer> CurrentDownloads { get; private set; } = [];
public List<FileTransfer> ForbiddenTransfers => _orchestrator.ForbiddenTransfers;
public Guid? CurrentOwnerToken { get; private set; }
public bool IsDownloading => CurrentDownloads.Any();
@@ -83,14 +84,15 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
{
CurrentDownloads.Clear();
_downloadStatus.Clear();
CurrentOwnerToken = null;
}
public async Task DownloadFiles(GameObjectHandler gameObject, List<FileReplacementData> fileReplacementDto, CancellationToken ct)
public async Task DownloadFiles(GameObjectHandler? gameObject, List<FileReplacementData> fileReplacementDto, CancellationToken ct, bool skipDownscale = false)
{
Mediator.Publish(new HaltScanMessage(nameof(DownloadFiles)));
try
{
await DownloadFilesInternal(gameObject, fileReplacementDto, ct).ConfigureAwait(false);
await DownloadFilesInternal(gameObject, fileReplacementDto, ct, skipDownscale).ConfigureAwait(false);
}
catch
{
@@ -98,7 +100,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
}
finally
{
Mediator.Publish(new DownloadFinishedMessage(gameObject));
if (gameObject is not null)
{
Mediator.Publish(new DownloadFinishedMessage(gameObject));
}
Mediator.Publish(new ResumeScanMessage(nameof(DownloadFiles)));
}
}
@@ -272,30 +277,11 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
int bytesRead;
try
{
var readTask = stream.ReadAsync(buffer.AsMemory(0, buffer.Length), ct).AsTask();
while (!readTask.IsCompleted)
{
var completedTask = await Task.WhenAny(readTask, Task.Delay(DownloadStallTimeout)).ConfigureAwait(false);
if (completedTask == readTask)
{
break;
}
ct.ThrowIfCancellationRequested();
var snapshot = _pairProcessingLimiter.GetSnapshot();
if (snapshot.Waiting > 0)
{
throw new TimeoutException($"No data received for {DownloadStallTimeout.TotalSeconds} seconds while downloading {requestUrl} (waiting: {snapshot.Waiting})");
}
Logger.LogTrace("Download stalled for {requestUrl} but no queued pairs, continuing to wait", requestUrl);
}
bytesRead = await readTask.ConfigureAwait(false);
bytesRead = await stream.ReadAsync(buffer.AsMemory(0, buffer.Length), ct).ConfigureAwait(false);
}
catch (OperationCanceledException)
catch (OperationCanceledException ex)
{
Logger.LogWarning(ex, "Request got cancelled : {url}", requestUrl);
throw;
}
@@ -314,11 +300,6 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
Logger.LogDebug("{requestUrl} downloaded to {destination}", requestUrl, destinationFilename);
}
}
catch (TimeoutException ex)
{
Logger.LogWarning(ex, "Detected stalled download for {requestUrl}, aborting transfer", requestUrl);
throw;
}
catch (OperationCanceledException)
{
throw;
@@ -352,7 +333,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
}
}
private async Task DecompressBlockFileAsync(string downloadStatusKey, string blockFilePath, List<FileReplacementData> fileReplacement, string downloadLabel)
private async Task DecompressBlockFileAsync(string downloadStatusKey, string blockFilePath, List<FileReplacementData> fileReplacement, string downloadLabel, bool skipDownscale)
{
if (_downloadStatus.TryGetValue(downloadStatusKey, out var status))
{
@@ -385,7 +366,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
var decompressedFile = LZ4Wrapper.Unwrap(compressedFileContent);
await _fileCompactor.WriteAllBytesAsync(filePath, decompressedFile, CancellationToken.None).ConfigureAwait(false);
PersistFileToStorage(fileHash, filePath);
var gamePath = fileReplacement.FirstOrDefault(f => string.Equals(f.Hash, fileHash, StringComparison.OrdinalIgnoreCase))?.GamePaths.FirstOrDefault() ?? string.Empty;
PersistFileToStorage(fileHash, filePath, gamePath, skipDownscale);
}
catch (EndOfStreamException)
{
@@ -413,7 +395,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
}
private async Task PerformDirectDownloadFallbackAsync(DownloadFileTransfer directDownload, List<FileReplacementData> fileReplacement,
IProgress<long> progress, CancellationToken token, bool slotAlreadyAcquired)
IProgress<long> progress, CancellationToken token, bool skipDownscale, bool slotAlreadyAcquired)
{
if (string.IsNullOrEmpty(directDownload.DirectDownloadUrl))
{
@@ -455,7 +437,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
throw new FileNotFoundException("Block file missing after direct download fallback.", blockFile);
}
await DecompressBlockFileAsync(downloadKey, blockFile, fileReplacement, $"fallback-{directDownload.Hash}").ConfigureAwait(false);
await DecompressBlockFileAsync(downloadKey, blockFile, fileReplacement, $"fallback-{directDownload.Hash}", skipDownscale).ConfigureAwait(false);
}
finally
{
@@ -478,8 +460,9 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
}
}
public async Task<List<DownloadFileTransfer>> InitiateDownloadList(GameObjectHandler gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct)
public async Task<List<DownloadFileTransfer>> InitiateDownloadList(GameObjectHandler? gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct, Guid? ownerToken = null)
{
CurrentOwnerToken = ownerToken;
var objectName = gameObjectHandler?.Name ?? "Unknown";
Logger.LogDebug("Download start: {id}", objectName);
@@ -520,7 +503,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
return CurrentDownloads;
}
private async Task DownloadFilesInternal(GameObjectHandler gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct)
private async Task DownloadFilesInternal(GameObjectHandler? gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct, bool skipDownscale)
{
var objectName = gameObjectHandler?.Name ?? "Unknown";
@@ -583,7 +566,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
Logger.LogWarning("Downloading {direct} files directly, and {batchtotal} in {batches} batches.", directDownloads.Count, batchDownloads.Count, downloadBatches.Length);
}
Mediator.Publish(new DownloadStartedMessage(gameObjectHandler, _downloadStatus));
if (gameObjectHandler is not null)
{
Mediator.Publish(new DownloadStartedMessage(gameObjectHandler, _downloadStatus));
}
Task batchDownloadsTask = downloadBatches.Length == 0 ? Task.CompletedTask : Parallel.ForEachAsync(downloadBatches, new ParallelOptions()
{
@@ -651,7 +637,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
return;
}
await DecompressBlockFileAsync(fileGroup.Key, blockFile, fileReplacement, fi.Name).ConfigureAwait(false);
await DecompressBlockFileAsync(fileGroup.Key, blockFile, fileReplacement, fi.Name, skipDownscale).ConfigureAwait(false);
}
finally
{
@@ -690,14 +676,13 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
if (!ShouldUseDirectDownloads())
{
await PerformDirectDownloadFallbackAsync(directDownload, fileReplacement, progress, token, slotAlreadyAcquired: false).ConfigureAwait(false);
await PerformDirectDownloadFallbackAsync(directDownload, fileReplacement, progress, token, skipDownscale, slotAlreadyAcquired: false).ConfigureAwait(false);
return;
}
var tempFilename = _fileDbManager.GetCacheFilePath(directDownload.Hash, "bin");
var slotAcquired = false;
try
{
downloadTracker.DownloadStatus = DownloadStatus.WaitingForSlot;
@@ -727,7 +712,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
byte[] compressedBytes = await File.ReadAllBytesAsync(tempFilename).ConfigureAwait(false);
var decompressedBytes = LZ4Wrapper.Unwrap(compressedBytes);
await _fileCompactor.WriteAllBytesAsync(finalFilename, decompressedBytes, CancellationToken.None).ConfigureAwait(false);
PersistFileToStorage(directDownload.Hash, finalFilename);
PersistFileToStorage(directDownload.Hash, finalFilename, replacement.GamePaths[0], skipDownscale);
downloadTracker.TransferredFiles = 1;
Logger.LogDebug("Finished direct download of {hash}.", directDownload.Hash);
@@ -739,8 +724,15 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
}
catch (OperationCanceledException ex)
{
Logger.LogDebug("{hash}: Detected cancellation of direct download, discarding file.", directDownload.Hash);
Logger.LogError(ex, "{hash}: Error during direct download.", directDownload.Hash);
if (token.IsCancellationRequested)
{
Logger.LogDebug("{hash}: Direct download cancelled by caller, discarding file.", directDownload.Hash);
}
else
{
Logger.LogWarning(ex, "{hash}: Direct download cancelled unexpectedly.", directDownload.Hash);
}
ClearDownload();
return;
}
@@ -762,7 +754,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
try
{
downloadTracker.DownloadStatus = DownloadStatus.WaitingForQueue;
await PerformDirectDownloadFallbackAsync(directDownload, fileReplacement, progress, token, slotAcquired).ConfigureAwait(false);
await PerformDirectDownloadFallbackAsync(directDownload, fileReplacement, progress, token, skipDownscale, slotAcquired).ConfigureAwait(false);
if (!expectedDirectDownloadFailure && failureCount >= 3 && !_disableDirectDownloads)
{
@@ -815,7 +807,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
return await response.Content.ReadFromJsonAsync<List<DownloadFileDto>>(cancellationToken: ct).ConfigureAwait(false) ?? [];
}
private void PersistFileToStorage(string fileHash, string filePath)
private void PersistFileToStorage(string fileHash, string filePath, string gamePath, bool skipDownscale)
{
var fi = new FileInfo(filePath);
Func<DateTime> RandomDayInThePast()
@@ -832,6 +824,11 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
try
{
var entry = _fileDbManager.CreateCacheEntry(filePath);
var mapKind = _textureMetadataHelper.DetermineMapKind(gamePath, filePath);
if (!skipDownscale)
{
_textureDownscaleService.ScheduleDownscale(fileHash, filePath, mapKind);
}
if (entry != null && !string.Equals(entry.Hash, fileHash, StringComparison.OrdinalIgnoreCase))
{
Logger.LogError("Hash mismatch after extracting, got {hash}, expected {expectedHash}, deleting file", entry.Hash, fileHash);

View File

@@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Net.Sockets;
using System.Reflection;
namespace LightlessSync.WebAPI.Files;
@@ -84,27 +85,46 @@ public class FileTransferOrchestrator : DisposableMediatorSubscriberBase
CancellationToken? ct = null, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead,
bool withToken = true)
{
using var requestMessage = new HttpRequestMessage(method, uri);
return await SendRequestInternalAsync(requestMessage, ct, httpCompletionOption, withToken).ConfigureAwait(false);
return await SendRequestInternalAsync(() => new HttpRequestMessage(method, uri),
ct, httpCompletionOption, withToken, allowRetry: true).ConfigureAwait(false);
}
public async Task<HttpResponseMessage> SendRequestAsync<T>(HttpMethod method, Uri uri, T content, CancellationToken ct,
bool withToken = true) where T : class
{
using var requestMessage = new HttpRequestMessage(method, uri);
if (content is not ByteArrayContent)
requestMessage.Content = JsonContent.Create(content);
else
requestMessage.Content = content as ByteArrayContent;
return await SendRequestInternalAsync(requestMessage, ct, withToken: withToken).ConfigureAwait(false);
return await SendRequestInternalAsync(() =>
{
var requestMessage = new HttpRequestMessage(method, uri);
if (content is not ByteArrayContent byteArrayContent)
{
requestMessage.Content = JsonContent.Create(content);
}
else
{
var clonedContent = new ByteArrayContent(byteArrayContent.ReadAsByteArrayAsync().GetAwaiter().GetResult());
foreach (var header in byteArrayContent.Headers)
{
clonedContent.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
requestMessage.Content = clonedContent;
}
return requestMessage;
}, ct, HttpCompletionOption.ResponseContentRead, withToken,
allowRetry: content is not HttpContent || content is ByteArrayContent).ConfigureAwait(false);
}
public async Task<HttpResponseMessage> SendRequestStreamAsync(HttpMethod method, Uri uri, ProgressableStreamContent content,
CancellationToken ct, bool withToken = true)
{
using var requestMessage = new HttpRequestMessage(method, uri);
requestMessage.Content = content;
return await SendRequestInternalAsync(requestMessage, ct, withToken: withToken).ConfigureAwait(false);
return await SendRequestInternalAsync(() =>
{
var requestMessage = new HttpRequestMessage(method, uri)
{
Content = content
};
return requestMessage;
}, ct, HttpCompletionOption.ResponseContentRead, withToken, allowRetry: false).ConfigureAwait(false);
}
public async Task WaitForDownloadSlotAsync(CancellationToken token)
@@ -146,39 +166,78 @@ public class FileTransferOrchestrator : DisposableMediatorSubscriberBase
return Math.Clamp(dividedLimit, 1, long.MaxValue);
}
private async Task<HttpResponseMessage> SendRequestInternalAsync(HttpRequestMessage requestMessage,
CancellationToken? ct = null, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead, bool withToken = true)
private async Task<HttpResponseMessage> SendRequestInternalAsync(Func<HttpRequestMessage> requestFactory,
CancellationToken? ct = null, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead,
bool withToken = true, bool allowRetry = true)
{
if (withToken)
{
var token = await _tokenProvider.GetToken().ConfigureAwait(false);
requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token);
}
const int maxAttempts = 2;
var attempt = 0;
if (requestMessage.Content != null && requestMessage.Content is not StreamContent && requestMessage.Content is not ByteArrayContent)
while (true)
{
var content = await ((JsonContent)requestMessage.Content).ReadAsStringAsync().ConfigureAwait(false);
Logger.LogDebug("Sending {method} to {uri} (Content: {content})", requestMessage.Method, requestMessage.RequestUri, content);
}
else
{
Logger.LogDebug("Sending {method} to {uri}", requestMessage.Method, requestMessage.RequestUri);
}
attempt++;
using var requestMessage = requestFactory();
try
{
if (ct != null)
return await _httpClient.SendAsync(requestMessage, httpCompletionOption, ct.Value).ConfigureAwait(false);
return await _httpClient.SendAsync(requestMessage, httpCompletionOption).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
throw;
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Error during SendRequestInternal for {uri}", requestMessage.RequestUri);
throw;
if (withToken)
{
var token = await _tokenProvider.GetToken().ConfigureAwait(false);
requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token);
}
if (requestMessage.Content != null && requestMessage.Content is not StreamContent && requestMessage.Content is not ByteArrayContent)
{
var content = await ((JsonContent)requestMessage.Content).ReadAsStringAsync().ConfigureAwait(false);
Logger.LogDebug("Sending {method} to {uri} (Content: {content})", requestMessage.Method, requestMessage.RequestUri, content);
}
else
{
Logger.LogDebug("Sending {method} to {uri}", requestMessage.Method, requestMessage.RequestUri);
}
try
{
if (ct != null)
return await _httpClient.SendAsync(requestMessage, httpCompletionOption, ct.Value).ConfigureAwait(false);
return await _httpClient.SendAsync(requestMessage, httpCompletionOption).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
throw;
}
catch (Exception ex) when (allowRetry && attempt < maxAttempts && IsTransientNetworkException(ex))
{
Logger.LogWarning(ex, "Transient error during SendRequestInternal for {uri}, retrying attempt {attempt}/{maxAttempts}",
requestMessage.RequestUri, attempt, maxAttempts);
if (ct.HasValue)
{
await Task.Delay(TimeSpan.FromMilliseconds(200), ct.Value).ConfigureAwait(false);
}
else
{
await Task.Delay(TimeSpan.FromMilliseconds(200)).ConfigureAwait(false);
}
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Error during SendRequestInternal for {uri}", requestMessage.RequestUri);
throw;
}
}
}
private static bool IsTransientNetworkException(Exception ex)
{
var current = ex;
while (current != null)
{
if (current is SocketException socketEx)
{
return socketEx.SocketErrorCode is SocketError.ConnectionReset or SocketError.ConnectionAborted or SocketError.TimedOut;
}
current = current.InnerException;
}
return false;
}
}

View File

@@ -11,7 +11,6 @@ using Microsoft.Extensions.Logging;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Collections.Concurrent;
using System.Threading;
namespace LightlessSync.WebAPI.Files;
@@ -44,6 +43,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
}
public List<FileTransfer> CurrentUploads { get; } = [];
public bool IsReady => _orchestrator.IsInitialized;
public bool IsUploading
{
get