Merge pull request 'changes' (#49) from file-system-improvements into 1.12.1

Reviewed-on: #49
Reviewed-by: defnotken <defnotken@noreply.git.lightless-sync.org>
This commit was merged in pull request #49.
This commit is contained in:
2025-10-06 00:13:19 +02:00
21 changed files with 1343 additions and 174 deletions

View File

@@ -1,4 +1,4 @@
using K4os.Compression.LZ4.Legacy;
using K4os.Compression.LZ4.Legacy;
using LightlessSync.Interop.Ipc;
using LightlessSync.LightlessConfiguration;
using LightlessSync.Services.Mediator;
@@ -19,7 +19,8 @@ public sealed class FileCacheManager : IHostedService
private readonly LightlessConfigService _configService;
private readonly LightlessMediator _lightlessMediator;
private readonly string _csvPath;
private readonly ConcurrentDictionary<string, List<FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, FileCacheEntity> _fileCachesByPrefixedPath = new(StringComparer.OrdinalIgnoreCase);
private readonly SemaphoreSlim _getCachesByPathsSemaphore = new(1, 1);
private readonly Lock _fileWriteLock = new();
private readonly IpcManager _ipcManager;
@@ -37,6 +38,57 @@ public sealed class FileCacheManager : IHostedService
private string CsvBakPath => _csvPath + ".bak";
private static string NormalizeSeparators(string path)
{
return path.Replace("/", "\\", StringComparison.Ordinal)
.Replace("\\\\", "\\", StringComparison.Ordinal);
}
private static string NormalizePrefixedPathKey(string prefixedPath)
{
if (string.IsNullOrEmpty(prefixedPath))
{
return string.Empty;
}
return NormalizeSeparators(prefixedPath).ToLowerInvariant();
}
private string NormalizeToPrefixedPath(string path)
{
if (string.IsNullOrEmpty(path)) return string.Empty;
var normalized = NormalizeSeparators(path);
if (normalized.StartsWith(CachePrefix, StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith(PenumbraPrefix, StringComparison.OrdinalIgnoreCase))
{
return NormalizePrefixedPathKey(normalized);
}
var penumbraDir = _ipcManager.Penumbra.ModDirectory;
if (!string.IsNullOrEmpty(penumbraDir))
{
var normalizedPenumbra = NormalizeSeparators(penumbraDir);
var replacement = normalizedPenumbra.EndsWith("\\", StringComparison.Ordinal)
? PenumbraPrefix + "\\"
: PenumbraPrefix;
normalized = normalized.Replace(normalizedPenumbra, replacement, StringComparison.OrdinalIgnoreCase);
}
var cacheFolder = _configService.Current.CacheFolder;
if (!string.IsNullOrEmpty(cacheFolder))
{
var normalizedCache = NormalizeSeparators(cacheFolder);
var replacement = normalizedCache.EndsWith("\\", StringComparison.Ordinal)
? CachePrefix + "\\"
: CachePrefix;
normalized = normalized.Replace(normalizedCache, replacement, StringComparison.OrdinalIgnoreCase);
}
return NormalizePrefixedPathKey(normalized);
}
public FileCacheEntity? CreateCacheEntry(string path)
{
FileInfo fi = new(path);
@@ -61,20 +113,26 @@ public sealed class FileCacheManager : IHostedService
return CreateFileCacheEntity(fi, prefixedPath);
}
public List<FileCacheEntity> GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v).ToList();
public List<FileCacheEntity> GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v.Values.Where(e => e != null)).ToList();
public List<FileCacheEntity> GetAllFileCachesByHash(string hash, bool ignoreCacheEntries = false, bool validate = true)
{
List<FileCacheEntity> output = [];
if (_fileCaches.TryGetValue(hash, out var fileCacheEntities))
{
foreach (var fileCache in fileCacheEntities.Where(c => !ignoreCacheEntries || !c.IsCacheEntry).ToList())
foreach (var fileCache in fileCacheEntities.Values.Where(c => !ignoreCacheEntries || !c.IsCacheEntry).ToList())
{
if (!validate) output.Add(fileCache);
if (!validate)
{
output.Add(fileCache);
}
else
{
var validated = GetValidatedFileCache(fileCache);
if (validated != null) output.Add(validated);
if (validated != null)
{
output.Add(validated);
}
}
}
}
@@ -86,7 +144,7 @@ public sealed class FileCacheManager : IHostedService
{
_lightlessMediator.Publish(new HaltScanMessage(nameof(ValidateLocalIntegrity)));
_logger.LogInformation("Validating local storage");
var cacheEntries = _fileCaches.SelectMany(v => v.Value).Where(v => v.IsCacheEntry).ToList();
var cacheEntries = _fileCaches.Values.SelectMany(v => v.Values.Where(e => e != null)).Where(v => v.IsCacheEntry).ToList();
List<FileCacheEntity> brokenEntities = [];
int i = 0;
foreach (var fileCache in cacheEntries)
@@ -151,29 +209,40 @@ public sealed class FileCacheManager : IHostedService
public FileCacheEntity? GetFileCacheByHash(string hash)
{
if (_fileCaches.TryGetValue(hash, out var hashes))
if (_fileCaches.TryGetValue(hash, out var entries))
{
var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix, StringComparison.Ordinal) ? 0 : 1).FirstOrDefault();
if (item != null) return GetValidatedFileCache(item);
var item = entries.Values
.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix, StringComparison.Ordinal) ? 0 : 1)
.FirstOrDefault();
if (item != null)
{
return GetValidatedFileCache(item);
}
}
return null;
}
private FileCacheEntity? GetFileCacheByPath(string path)
{
var cleanedPath = path.Replace("/", "\\", StringComparison.OrdinalIgnoreCase).ToLowerInvariant()
.Replace(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), "", StringComparison.OrdinalIgnoreCase);
var entry = _fileCaches.SelectMany(v => v.Value).FirstOrDefault(f => f.ResolvedFilepath.EndsWith(cleanedPath, StringComparison.OrdinalIgnoreCase));
if (entry == null)
var normalizedPrefixedPath = NormalizeToPrefixedPath(path);
if (string.IsNullOrEmpty(normalizedPrefixedPath))
{
_logger.LogDebug("Found no entries for {path}", cleanedPath);
return CreateFileEntry(path);
return null;
}
var validatedCacheEntry = GetValidatedFileCache(entry);
if (_fileCachesByPrefixedPath.TryGetValue(normalizedPrefixedPath, out var entry))
{
return GetValidatedFileCache(entry);
}
return validatedCacheEntry;
_logger.LogDebug("Found no entries for {path}", normalizedPrefixedPath);
if (normalizedPrefixedPath.Contains(CachePrefix, StringComparison.Ordinal))
{
return CreateCacheEntry(path);
}
return CreateFileEntry(path) ?? CreateCacheEntry(path);
}
public Dictionary<string, FileCacheEntity?> GetFileCachesByPaths(string[] paths)
@@ -182,73 +251,55 @@ public sealed class FileCacheManager : IHostedService
try
{
var allEntities = _fileCaches.SelectMany(f => f.Value).ToArray();
var result = new Dictionary<string, FileCacheEntity?>(StringComparer.OrdinalIgnoreCase);
var seenNormalized = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var cacheDict = new ConcurrentDictionary<string, FileCacheEntity>(
StringComparer.OrdinalIgnoreCase);
Parallel.ForEach(allEntities, entity =>
foreach (var originalPath in paths)
{
if (entity != null && entity.PrefixedFilePath != null)
if (string.IsNullOrEmpty(originalPath))
{
cacheDict[entity.PrefixedFilePath] = entity;
result[originalPath] = null;
continue;
}
var normalized = NormalizeToPrefixedPath(originalPath);
if (seenNormalized.Add(normalized))
{
if (!string.IsNullOrEmpty(normalized))
{
_logger.LogDebug("Normalized path {cleaned}", normalized);
}
}
else if (!string.IsNullOrEmpty(normalized))
{
_logger.LogWarning("Duplicate normalized path detected: {cleaned}", normalized);
}
if (_fileCachesByPrefixedPath.TryGetValue(normalized, out var entity))
{
result[originalPath] = GetValidatedFileCache(entity);
continue;
}
FileCacheEntity? created = null;
if (normalized.Contains(CachePrefix, StringComparison.Ordinal))
{
created = CreateCacheEntry(originalPath);
}
else if (normalized.Contains(PenumbraPrefix, StringComparison.Ordinal))
{
created = CreateFileEntry(originalPath);
}
else
{
_logger.LogWarning("Null FileCacheEntity or PrefixedFilePath encountered in cache population: {entity}", entity);
created = CreateFileEntry(originalPath) ?? CreateCacheEntry(originalPath);
}
});
var cleanedPaths = new ConcurrentDictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var seenCleaned = new ConcurrentDictionary<string, byte>(StringComparer.OrdinalIgnoreCase);
result[originalPath] = created;
}
Parallel.ForEach(paths, p =>
{
var cleaned = p.Replace("/", "\\", StringComparison.OrdinalIgnoreCase)
.Replace(
_ipcManager.Penumbra.ModDirectory!,
_ipcManager.Penumbra.ModDirectory!.EndsWith('\\')
? PenumbraPrefix + '\\' : PenumbraPrefix,
StringComparison.OrdinalIgnoreCase)
.Replace(
_configService.Current.CacheFolder,
_configService.Current.CacheFolder.EndsWith('\\')
? CachePrefix + '\\' : CachePrefix,
StringComparison.OrdinalIgnoreCase)
.Replace("\\\\", "\\", StringComparison.Ordinal);
if (seenCleaned.TryAdd(cleaned, 0))
{
_logger.LogDebug("Adding to cleanedPaths: {cleaned}", cleaned);
cleanedPaths[p] = cleaned;
}
else
{
_logger.LogWarning("Duplicate found: {cleaned}", cleaned);
}
});
var result = new ConcurrentDictionary<string, FileCacheEntity?>(StringComparer.OrdinalIgnoreCase);
Parallel.ForEach(cleanedPaths, entry =>
{
_logger.LogDebug("Checking if in cache: {path}", entry.Value);
if (cacheDict.TryGetValue(entry.Value, out var entity))
{
var validatedCache = GetValidatedFileCache(entity);
result[entry.Key] = validatedCache;
}
else
{
if (!entry.Value.Contains(CachePrefix, StringComparison.Ordinal))
result[entry.Key] = CreateFileEntry(entry.Key);
else
result[entry.Key] = CreateCacheEntry(entry.Key);
}
});
return new Dictionary<string, FileCacheEntity?>(result, StringComparer.OrdinalIgnoreCase);
return result;
}
finally
{
@@ -258,17 +309,24 @@ public sealed class FileCacheManager : IHostedService
public void RemoveHashedFile(string hash, string prefixedFilePath)
{
var normalizedPath = NormalizePrefixedPathKey(prefixedFilePath);
if (_fileCaches.TryGetValue(hash, out var caches))
{
_logger.LogTrace("Removing from DB: {hash} => {path}", hash, prefixedFilePath);
var removedCount = caches?.RemoveAll(c => string.Equals(c.PrefixedFilePath, prefixedFilePath, StringComparison.Ordinal));
_logger.LogTrace("Removed from DB: {count} file(s) with hash {hash} and file cache {path}", removedCount, hash, prefixedFilePath);
if (caches?.Count == 0)
if (caches.TryRemove(normalizedPath, out var removedEntity))
{
_fileCaches.Remove(hash, out var entity);
_logger.LogTrace("Removed from DB: {hash} => {path}", hash, removedEntity.PrefixedFilePath);
}
if (caches.IsEmpty)
{
_fileCaches.TryRemove(hash, out _);
}
}
_fileCachesByPrefixedPath.TryRemove(normalizedPath, out _);
}
public void UpdateHashedFile(FileCacheEntity fileCache, bool computeProperties = true)
@@ -309,7 +367,7 @@ public sealed class FileCacheManager : IHostedService
lock (_fileWriteLock)
{
StringBuilder sb = new();
foreach (var entry in _fileCaches.SelectMany(k => k.Value).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase))
foreach (var entry in _fileCaches.Values.SelectMany(k => k.Values).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase))
{
sb.AppendLine(entry.CsvEntry);
}
@@ -354,16 +412,11 @@ public sealed class FileCacheManager : IHostedService
private void AddHashedFile(FileCacheEntity fileCache)
{
if (!_fileCaches.TryGetValue(fileCache.Hash, out var entries) || entries is null)
{
_fileCaches[fileCache.Hash] = entries = [];
}
var normalizedPath = NormalizePrefixedPathKey(fileCache.PrefixedFilePath);
var entries = _fileCaches.GetOrAdd(fileCache.Hash, _ => new ConcurrentDictionary<string, FileCacheEntity>(StringComparer.OrdinalIgnoreCase));
if (!entries.Exists(u => string.Equals(u.PrefixedFilePath, fileCache.PrefixedFilePath, StringComparison.OrdinalIgnoreCase)))
{
//_logger.LogTrace("Adding to DB: {hash} => {path}", fileCache.Hash, fileCache.PrefixedFilePath);
entries.Add(fileCache);
}
entries[normalizedPath] = fileCache;
_fileCachesByPrefixedPath[normalizedPath] = fileCache;
}
private FileCacheEntity? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath, string? hash = null)

View File

@@ -1,3 +1,5 @@
using Dalamud.Game.Text;
using LightlessSync.UtilsEnum.Enum;
using LightlessSync.LightlessConfiguration.Models;
using LightlessSync.UI;
using Microsoft.Extensions.Logging;
@@ -33,6 +35,9 @@ public class LightlessConfig : ILightlessConfiguration
public bool OpenGposeImportOnGposeStart { get; set; } = false;
public bool OpenPopupOnAdd { get; set; } = true;
public int ParallelDownloads { get; set; } = 10;
public int ParallelUploads { get; set; } = 8;
public bool EnablePairProcessingLimiter { get; set; } = true;
public int MaxConcurrentPairApplications { get; set; } = 3;
public int DownloadSpeedLimitInBytes { get; set; } = 0;
public DownloadSpeeds DownloadSpeedType { get; set; } = DownloadSpeeds.MBps;
public bool PreferNotesOverNamesForVisible { get; set; } = false;
@@ -70,7 +75,14 @@ public class LightlessConfig : ILightlessConfiguration
public bool overrideFcTagColor { get; set; } = false;
public bool useColoredUIDs { get; set; } = true;
public bool BroadcastEnabled { get; set; } = false;
public short LightfinderLabelOffsetX { get; set; } = 0;
public short LightfinderLabelOffsetY { get; set; } = 0;
public bool LightfinderLabelUseIcon { get; set; } = false;
public string LightfinderLabelIconGlyph { get; set; } = SeIconCharExtensions.ToIconString(SeIconChar.LinkMarker);
public float LightfinderLabelScale { get; set; } = 1.0f;
public bool LightfinderAutoAlign { get; set; } = true;
public LabelAlignment LabelAlignment { get; set; } = LabelAlignment.Left;
public DateTime BroadcastTtl { get; set; } = DateTime.MinValue;
public bool SyncshellFinderEnabled { get; set; } = false;
public string? SelectedFinderSyncshell { get; set; } = null;
}
}

View File

@@ -0,0 +1,9 @@
using LightlessSync.LightlessConfiguration.Models;
namespace LightlessSync.LightlessConfiguration.Configurations;
public class ServerTagConfig : ILightlessConfiguration
{
public Dictionary<string, ServerTagStorage> ServerTagStorage { get; set; } = new(StringComparer.OrdinalIgnoreCase);
public int Version { get; set; } = 0;
}

View File

@@ -0,0 +1,9 @@
namespace LightlessSync.LightlessConfiguration.Models;
[Serializable]
public class ServerTagStorage
{
public HashSet<string> OpenPairTags { get; set; } = new(StringComparer.Ordinal);
public HashSet<string> ServerAvailablePairTags { get; set; } = new(StringComparer.Ordinal);
public Dictionary<string, List<string>> UidServerPairedUserTags { get; set; } = new(StringComparer.Ordinal);
}

View File

@@ -0,0 +1,14 @@
using LightlessSync.LightlessConfiguration.Configurations;
namespace LightlessSync.LightlessConfiguration;
public class ServerTagConfigService : ConfigurationServiceBase<ServerTagConfig>
{
public const string ConfigName = "servertags.json";
public ServerTagConfigService(string configDir) : base(configDir)
{
}
public override string ConfigurationName => ConfigName;
}

View File

@@ -1,4 +1,4 @@
using LightlessSync.FileCache;
using LightlessSync.FileCache;
using LightlessSync.Interop.Ipc;
using LightlessSync.PlayerData.Handlers;
using LightlessSync.PlayerData.Pairs;
@@ -21,6 +21,7 @@ public class PairHandlerFactory
private readonly ILoggerFactory _loggerFactory;
private readonly LightlessMediator _lightlessMediator;
private readonly PlayerPerformanceService _playerPerformanceService;
private readonly PairProcessingLimiter _pairProcessingLimiter;
private readonly ServerConfigurationManager _serverConfigManager;
private readonly PluginWarningNotificationService _pluginWarningNotificationManager;
@@ -28,6 +29,7 @@ public class PairHandlerFactory
FileDownloadManagerFactory fileDownloadManagerFactory, DalamudUtilService dalamudUtilService,
PluginWarningNotificationService pluginWarningNotificationManager, IHostApplicationLifetime hostApplicationLifetime,
FileCacheManager fileCacheManager, LightlessMediator lightlessMediator, PlayerPerformanceService playerPerformanceService,
PairProcessingLimiter pairProcessingLimiter,
ServerConfigurationManager serverConfigManager)
{
_loggerFactory = loggerFactory;
@@ -40,6 +42,7 @@ public class PairHandlerFactory
_fileCacheManager = fileCacheManager;
_lightlessMediator = lightlessMediator;
_playerPerformanceService = playerPerformanceService;
_pairProcessingLimiter = pairProcessingLimiter;
_serverConfigManager = serverConfigManager;
}
@@ -47,6 +50,6 @@ public class PairHandlerFactory
{
return new PairHandler(_loggerFactory.CreateLogger<PairHandler>(), pair, _gameObjectHandlerFactory,
_ipcManager, _fileDownloadManagerFactory.Create(), _pluginWarningNotificationManager, _dalamudUtilService, _hostApplicationLifetime,
_fileCacheManager, _lightlessMediator, _playerPerformanceService, _serverConfigManager);
_fileCacheManager, _lightlessMediator, _playerPerformanceService, _pairProcessingLimiter, _serverConfigManager);
}
}

View File

@@ -1,4 +1,4 @@
using LightlessSync.API.Data;
using LightlessSync.API.Data;
using LightlessSync.FileCache;
using LightlessSync.Interop.Ipc;
using LightlessSync.PlayerData.Factories;
@@ -28,6 +28,7 @@ public sealed class PairHandler : DisposableMediatorSubscriberBase
private readonly IpcManager _ipcManager;
private readonly IHostApplicationLifetime _lifetime;
private readonly PlayerPerformanceService _playerPerformanceService;
private readonly PairProcessingLimiter _pairProcessingLimiter;
private readonly ServerConfigurationManager _serverConfigManager;
private readonly PluginWarningNotificationService _pluginWarningNotificationManager;
private CancellationTokenSource? _applicationCancellationTokenSource = new();
@@ -50,6 +51,7 @@ public sealed class PairHandler : DisposableMediatorSubscriberBase
DalamudUtilService dalamudUtil, IHostApplicationLifetime lifetime,
FileCacheManager fileDbManager, LightlessMediator mediator,
PlayerPerformanceService playerPerformanceService,
PairProcessingLimiter pairProcessingLimiter,
ServerConfigurationManager serverConfigManager) : base(logger, mediator)
{
Pair = pair;
@@ -61,6 +63,7 @@ public sealed class PairHandler : DisposableMediatorSubscriberBase
_lifetime = lifetime;
_fileDbManager = fileDbManager;
_playerPerformanceService = playerPerformanceService;
_pairProcessingLimiter = pairProcessingLimiter;
_serverConfigManager = serverConfigManager;
_penumbraCollection = _ipcManager.Penumbra.CreateTemporaryCollectionAsync(logger, Pair.UserData.UID).ConfigureAwait(false).GetAwaiter().GetResult();
@@ -420,6 +423,7 @@ public sealed class PairHandler : DisposableMediatorSubscriberBase
private async Task DownloadAndApplyCharacterAsync(Guid applicationBase, CharacterData charaData, Dictionary<ObjectKind, HashSet<PlayerChanges>> updatedData,
bool updateModdedPaths, bool updateManip, CancellationToken downloadToken)
{
await using var concurrencyLease = await _pairProcessingLimiter.AcquireAsync(downloadToken).ConfigureAwait(false);
Dictionary<(string GamePath, string? Hash), string> moddedPaths = [];
if (updateModdedPaths)

View File

@@ -1,4 +1,4 @@
using Dalamud.Plugin.Services;
using Dalamud.Plugin.Services;
using LightlessSync.API.Data;
using LightlessSync.API.Data.Comparer;
using LightlessSync.API.Data.Extensions;
@@ -7,10 +7,14 @@ using LightlessSync.API.Dto.User;
using LightlessSync.LightlessConfiguration;
using LightlessSync.LightlessConfiguration.Models;
using LightlessSync.PlayerData.Factories;
using LightlessSync.Services;
using LightlessSync.Services.Events;
using LightlessSync.Services.Mediator;
using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
using System.Threading;
using System.Threading.Tasks;
namespace LightlessSync.PlayerData.Pairs;
@@ -24,14 +28,19 @@ public sealed class PairManager : DisposableMediatorSubscriberBase
private Lazy<List<Pair>> _directPairsInternal;
private Lazy<Dictionary<GroupFullInfoDto, List<Pair>>> _groupPairsInternal;
private Lazy<Dictionary<Pair, List<GroupFullInfoDto>>> _pairsWithGroupsInternal;
private readonly PairProcessingLimiter _pairProcessingLimiter;
private readonly ConcurrentQueue<(Pair Pair, OnlineUserIdentDto? Ident)> _pairCreationQueue = new();
private CancellationTokenSource _pairCreationCts = new();
private int _pairCreationProcessorRunning;
public PairManager(ILogger<PairManager> logger, PairFactory pairFactory,
LightlessConfigService configurationService, LightlessMediator mediator,
IContextMenu dalamudContextMenu) : base(logger, mediator)
IContextMenu dalamudContextMenu, PairProcessingLimiter pairProcessingLimiter) : base(logger, mediator)
{
_pairFactory = pairFactory;
_configurationService = configurationService;
_dalamudContextMenu = dalamudContextMenu;
_pairProcessingLimiter = pairProcessingLimiter;
Mediator.Subscribe<DisconnectedMessage>(this, (_) => ClearPairs());
Mediator.Subscribe<CutsceneEndMessage>(this, (_) => ReapplyPairData());
_directPairsInternal = DirectPairsLazy();
@@ -112,6 +121,7 @@ public sealed class PairManager : DisposableMediatorSubscriberBase
public void ClearPairs()
{
Logger.LogDebug("Clearing all Pairs");
ResetPairCreationQueue();
DisposePairs();
_allClientPairs.Clear();
_allGroups.Clear();
@@ -161,7 +171,7 @@ public sealed class PairManager : DisposableMediatorSubscriberBase
Mediator.Publish(new NotificationMessage("User online", msg, NotificationType.Info, TimeSpan.FromSeconds(5)));
}
pair.CreateCachedPlayer(dto);
QueuePairCreation(pair, dto);
RecreateLazy();
}
@@ -332,6 +342,7 @@ public sealed class PairManager : DisposableMediatorSubscriberBase
{
base.Dispose(disposing);
ResetPairCreationQueue();
_dalamudContextMenu.OnMenuOpened -= DalamudContextMenuOnOnOpenGameObjectContextMenu;
DisposePairs();
@@ -390,6 +401,84 @@ public sealed class PairManager : DisposableMediatorSubscriberBase
});
}
private void QueuePairCreation(Pair pair, OnlineUserIdentDto? dto)
{
if (pair.HasCachedPlayer)
{
RecreateLazy();
return;
}
_pairCreationQueue.Enqueue((pair, dto));
StartPairCreationProcessor();
}
private void StartPairCreationProcessor()
{
if (_pairCreationCts.IsCancellationRequested)
{
return;
}
if (Interlocked.CompareExchange(ref _pairCreationProcessorRunning, 1, 0) == 0)
{
_ = Task.Run(ProcessPairCreationQueueAsync);
}
}
private async Task ProcessPairCreationQueueAsync()
{
try
{
while (!_pairCreationCts.IsCancellationRequested)
{
if (!_pairCreationQueue.TryDequeue(out var work))
{
break;
}
try
{
await using var lease = await _pairProcessingLimiter.AcquireAsync(_pairCreationCts.Token).ConfigureAwait(false);
if (!work.Pair.HasCachedPlayer)
{
work.Pair.CreateCachedPlayer(work.Ident);
}
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
Logger.LogError(ex, "Error creating cached player for {uid}", work.Pair.UserData.UID);
}
RecreateLazy();
await Task.Yield();
}
}
finally
{
Interlocked.Exchange(ref _pairCreationProcessorRunning, 0);
if (!_pairCreationQueue.IsEmpty && !_pairCreationCts.IsCancellationRequested)
{
StartPairCreationProcessor();
}
}
}
private void ResetPairCreationQueue()
{
_pairCreationCts.Cancel();
while (_pairCreationQueue.TryDequeue(out _))
{
}
_pairCreationCts.Dispose();
_pairCreationCts = new CancellationTokenSource();
Interlocked.Exchange(ref _pairCreationProcessorRunning, 0);
}
private void ReapplyPairData()
{
foreach (var pair in _allClientPairs.Select(k => k.Value))

View File

@@ -1,4 +1,4 @@
using LightlessSync.API.Data;
using LightlessSync.API.Data;
using LightlessSync.Services;
using LightlessSync.Services.Mediator;
using LightlessSync.Utils;
@@ -101,6 +101,8 @@ public class VisibleUserDataDistributor : DisposableMediatorSubscriberBase
_ = Task.Run(async () =>
{
try
{
forced |= _uploadingCharacterData?.DataHash != _lastCreatedData.DataHash;
if (_fileUploadTask == null || (_fileUploadTask?.IsCompleted ?? false) || forced)
@@ -127,6 +129,15 @@ public class VisibleUserDataDistributor : DisposableMediatorSubscriberBase
_pushDataSemaphore.Release();
}
}
}
catch (OperationCanceledException) when (_runtimeCts.IsCancellationRequested)
{
Logger.LogDebug("PushCharacterData cancelled");
}
catch (Exception ex)
{
Logger.LogError(ex, "Failed to push character data");
}
});
}
}

View File

@@ -106,6 +106,7 @@ public sealed class Plugin : IDalamudPlugin
collection.AddSingleton<GameObjectHandlerFactory>();
collection.AddSingleton<FileDownloadManagerFactory>();
collection.AddSingleton<PairHandlerFactory>();
collection.AddSingleton<PairProcessingLimiter>();
collection.AddSingleton<PairFactory>();
collection.AddSingleton<XivDataAnalyzer>();
collection.AddSingleton<CharacterAnalyzer>();
@@ -144,7 +145,7 @@ public sealed class Plugin : IDalamudPlugin
collection.AddSingleton((s) => new DtrEntry(s.GetRequiredService<ILogger<DtrEntry>>(), dtrBar, s.GetRequiredService<LightlessConfigService>(),
s.GetRequiredService<LightlessMediator>(), s.GetRequiredService<PairManager>(), s.GetRequiredService<ApiController>(), s.GetRequiredService<ServerConfigurationManager>()));
collection.AddSingleton(s => new PairManager(s.GetRequiredService<ILogger<PairManager>>(), s.GetRequiredService<PairFactory>(),
s.GetRequiredService<LightlessConfigService>(), s.GetRequiredService<LightlessMediator>(), contextMenu));
s.GetRequiredService<LightlessConfigService>(), s.GetRequiredService<LightlessMediator>(), contextMenu, s.GetRequiredService<PairProcessingLimiter>()));
collection.AddSingleton<RedrawManager>();
collection.AddSingleton<BroadcastService>();
collection.AddSingleton(addonLifecycle);
@@ -277,4 +278,4 @@ public sealed class Plugin : IDalamudPlugin
_host.StopAsync().GetAwaiter().GetResult();
_host.Dispose();
}
}
}

View File

@@ -1,4 +1,4 @@
using Dalamud.Game.ClientState.Objects.Types;
using Dalamud.Game.ClientState.Objects.Types;
using LightlessSync.API.Data;
using LightlessSync.API.Dto;
using LightlessSync.API.Dto.CharaData;
@@ -77,6 +77,7 @@ public record OpenCensusPopupMessage() : MessageBase;
public record OpenSyncshellAdminPanel(GroupFullInfoDto GroupInfo) : MessageBase;
public record OpenPermissionWindow(Pair Pair) : MessageBase;
public record DownloadLimitChangedMessage() : SameThreadMessage;
public record PairProcessingLimitChangedMessage : SameThreadMessage;
public record CensusUpdateMessage(byte Gender, byte RaceId, byte TribeId) : MessageBase;
public record TargetPairMessage(Pair Pair) : MessageBase;
public record CombatStartMessage : MessageBase;

View File

@@ -1,15 +1,22 @@
using Dalamud.Game.Addon.Lifecycle;
using Dalamud.Game.Addon.Lifecycle;
using Dalamud.Game.Addon.Lifecycle.AddonArgTypes;
using Dalamud.Game.Text;
using Dalamud.Plugin.Services;
using FFXIVClientStructs.FFXIV.Client.System.Framework;
using FFXIVClientStructs.FFXIV.Client.UI;
using FFXIVClientStructs.FFXIV.Component.GUI;
using LightlessSync.LightlessConfiguration;
using LightlessSync.LightlessConfiguration.Configurations;
using LightlessSync.Services.Mediator;
using LightlessSync.UI;
using LightlessSync.Utils;
using LightlessSync.UtilsEnum.Enum;
// Created using https://github.com/PunishedPineapple/Distance as a reference, thank you!
using Microsoft.Extensions.Logging;
using System.Globalization;
using System.Text;
namespace LightlessSync.Services;
@@ -19,6 +26,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber
private readonly IAddonLifecycle _addonLifecycle;
private readonly IGameGui _gameGui;
private readonly DalamudUtilService _dalamudUtil;
private readonly LightlessConfigService _configService;
private readonly LightlessMediator _mediator;
public LightlessMediator Mediator => _mediator;
@@ -26,18 +34,28 @@ public unsafe class NameplateHandler : IMediatorSubscriber
private bool _needsLabelRefresh = false;
private AddonNamePlate* mpNameplateAddon = null;
private readonly AtkTextNode*[] mTextNodes = new AtkTextNode*[AddonNamePlate.NumNamePlateObjects];
private readonly int[] _cachedNameplateTextWidths = new int[AddonNamePlate.NumNamePlateObjects];
private readonly int[] _cachedNameplateTextHeights = new int[AddonNamePlate.NumNamePlateObjects];
private readonly int[] _cachedNameplateContainerHeights = new int[AddonNamePlate.NumNamePlateObjects];
private readonly int[] _cachedNameplateTextOffsets = new int[AddonNamePlate.NumNamePlateObjects];
internal const uint mNameplateNodeIDBase = 0x7D99D500;
private const string DefaultLabelText = "Lightfinder";
private const SeIconChar DefaultIcon = SeIconChar.LinkMarker;
private static readonly string DefaultIconGlyph = SeIconCharExtensions.ToIconString(DefaultIcon);
private volatile HashSet<string> _activeBroadcastingCids = new();
public NameplateHandler(ILogger<NameplateHandler> logger, IAddonLifecycle addonLifecycle, IGameGui gameGui, DalamudUtilService dalamudUtil, LightlessMediator mediator)
public NameplateHandler(ILogger<NameplateHandler> logger, IAddonLifecycle addonLifecycle, IGameGui gameGui, DalamudUtilService dalamudUtil, LightlessConfigService configService, LightlessMediator mediator)
{
_logger = logger;
_addonLifecycle = addonLifecycle;
_gameGui = gameGui;
_dalamudUtil = dalamudUtil;
_configService = configService;
_mediator = mediator;
System.Array.Fill(_cachedNameplateTextOffsets, int.MinValue);
}
internal void Init()
@@ -96,6 +114,10 @@ public unsafe class NameplateHandler : IMediatorSubscriber
if (mpNameplateAddon != pNameplateAddon)
{
for (int i = 0; i < mTextNodes.Length; ++i) mTextNodes[i] = null;
System.Array.Clear(_cachedNameplateTextWidths, 0, _cachedNameplateTextWidths.Length);
System.Array.Clear(_cachedNameplateTextHeights, 0, _cachedNameplateTextHeights.Length);
System.Array.Clear(_cachedNameplateContainerHeights, 0, _cachedNameplateContainerHeights.Length);
System.Array.Fill(_cachedNameplateTextOffsets, int.MinValue);
mpNameplateAddon = pNameplateAddon;
if (mpNameplateAddon != null) CreateNameplateNodes();
}
@@ -156,6 +178,11 @@ public unsafe class NameplateHandler : IMediatorSubscriber
}
}
}
System.Array.Clear(_cachedNameplateTextWidths, 0, _cachedNameplateTextWidths.Length);
System.Array.Clear(_cachedNameplateTextHeights, 0, _cachedNameplateTextHeights.Length);
System.Array.Clear(_cachedNameplateContainerHeights, 0, _cachedNameplateContainerHeights.Length);
System.Array.Fill(_cachedNameplateTextOffsets, int.MinValue);
}
private void HideAllNameplateNodes()
@@ -214,12 +241,143 @@ public unsafe class NameplateHandler : IMediatorSubscriber
var labelColor = UIColors.Get("LightlessPurple");
var edgeColor = UIColors.Get("FullBlack");
if (nameContainer == null || nameText == null)
{
pNode->AtkResNode.ToggleVisibility(false);
continue;
}
var labelColor = UIColors.Get("LightlessPurple");
var edgeColor = UIColors.Get("FullBlack");
var config = _configService.Current;
var scaleMultiplier = System.Math.Clamp(config.LightfinderLabelScale, 0.5f, 2.0f);
var baseScale = config.LightfinderLabelUseIcon ? 1.0f : 0.5f;
var effectiveScale = baseScale * scaleMultiplier;
var nodeWidth = (int)System.Math.Round(AtkNodeHelpers.DefaultTextNodeWidth * effectiveScale);
var nodeHeight = (int)System.Math.Round(AtkNodeHelpers.DefaultTextNodeHeight * effectiveScale);
int positionX = 58;
AlignmentType alignment = AlignmentType.Bottom;
var textScaleY = nameText->AtkResNode.ScaleY;
if (textScaleY <= 0f)
textScaleY = 1f;
var blockHeight = System.Math.Abs((int)nameplateObject.TextH);
if (blockHeight > 0)
{
_cachedNameplateTextHeights[nameplateIndex] = blockHeight;
}
else
{
blockHeight = _cachedNameplateTextHeights[nameplateIndex];
}
if (blockHeight <= 0)
{
blockHeight = GetScaledTextHeight(nameText);
if (blockHeight <= 0)
blockHeight = nodeHeight;
_cachedNameplateTextHeights[nameplateIndex] = blockHeight;
}
var containerHeight = (int)nameContainer->Height;
if (containerHeight > 0)
{
_cachedNameplateContainerHeights[nameplateIndex] = containerHeight;
}
else
{
containerHeight = _cachedNameplateContainerHeights[nameplateIndex];
}
if (containerHeight <= 0)
{
containerHeight = blockHeight + (int)System.Math.Round(8 * textScaleY);
if (containerHeight <= blockHeight)
containerHeight = blockHeight + 1;
_cachedNameplateContainerHeights[nameplateIndex] = containerHeight;
}
var labelY = nameContainer->Height - nameplateObject.TextH - (int)(24 * nameText->AtkResNode.ScaleY);
var blockTop = containerHeight - blockHeight;
if (blockTop < 0)
blockTop = 0;
var verticalPadding = (int)System.Math.Round(4 * effectiveScale);
var positionY = blockTop - verticalPadding - nodeHeight;
var textWidth = System.Math.Abs((int)nameplateObject.TextW);
if (textWidth > 0)
{
_cachedNameplateTextWidths[nameplateIndex] = textWidth;
}
else
{
textWidth = _cachedNameplateTextWidths[nameplateIndex];
}
if (textWidth <= 0)
{
textWidth = GetScaledTextWidth(nameText);
if (textWidth <= 0)
textWidth = nodeWidth;
_cachedNameplateTextWidths[nameplateIndex] = textWidth;
}
var textOffset = (int)System.Math.Round(nameText->AtkResNode.X);
var hasValidOffset = true;
if (System.Math.Abs((int)nameplateObject.TextW) > 0 || textOffset != 0)
{
_cachedNameplateTextOffsets[nameplateIndex] = textOffset;
}
else if (_cachedNameplateTextOffsets[nameplateIndex] != int.MinValue)
{
textOffset = _cachedNameplateTextOffsets[nameplateIndex];
}
else
{
hasValidOffset = false;
}
pNode->AtkResNode.SetPositionShort(58, (short)labelY);
pNode->AtkResNode.SetUseDepthBasedPriority(true);
pNode->AtkResNode.SetScale(0.5f, 0.5f);
if (config.LightfinderAutoAlign && nameContainer != null && hasValidOffset)
{
switch (config.LabelAlignment)
{
case LabelAlignment.Left:
positionX = textOffset;
alignment = AlignmentType.BottomLeft;
break;
case LabelAlignment.Right:
positionX = textOffset + textWidth - nodeWidth;
alignment = AlignmentType.BottomRight;
break;
default:
positionX = textOffset + textWidth / 2 - nodeWidth / 2;
alignment = AlignmentType.Bottom;
break;
}
}
else
{
alignment = AlignmentType.Bottom;
}
positionX += config.LightfinderLabelOffsetX;
positionY += config.LightfinderLabelOffsetY;
alignment = (AlignmentType)System.Math.Clamp((int)alignment, 0, 8);
pNode->AtkResNode.SetPositionShort((short)System.Math.Clamp(positionX, short.MinValue, short.MaxValue), (short)System.Math.Clamp(positionY, short.MinValue, short.MaxValue));
pNode->AtkResNode.SetUseDepthBasedPriority(true);
pNode->AtkResNode.SetScale(effectiveScale, effectiveScale);
pNode->AtkResNode.Color.A = 255;
@@ -238,14 +396,102 @@ public unsafe class NameplateHandler : IMediatorSubscriber
pNode->FontType = FontType.MiedingerMed;
pNode->LineSpacing = 24;
pNode->CharSpacing = 1;
var baseFontSize = config.LightfinderLabelUseIcon ? 36f : 24f;
var computedFontSize = (int)System.Math.Round(baseFontSize * scaleMultiplier);
pNode->FontSize = (byte)System.Math.Clamp(computedFontSize, 1, 255);
pNode->AlignmentType = alignment;
var computedLineSpacing = (int)System.Math.Round(24 * scaleMultiplier);
pNode->LineSpacing = (byte)System.Math.Clamp(computedLineSpacing, 0, byte.MaxValue);
pNode->CharSpacing = 1;
pNode->TextFlags = TextFlags.Edge | TextFlags.Glare;
pNode->TextFlags = config.LightfinderLabelUseIcon
? TextFlags.Edge | TextFlags.Glare | TextFlags.AutoAdjustNodeSize
: TextFlags.Edge | TextFlags.Glare;
var labelContent = config.LightfinderLabelUseIcon
? NormalizeIconGlyph(config.LightfinderLabelIconGlyph)
: DefaultLabelText;
pNode->FontType = config.LightfinderLabelUseIcon ? FontType.Axis : FontType.MiedingerMed;
pNode->SetText(labelContent);
}
}
private static unsafe int GetScaledTextHeight(AtkTextNode* node)
{
if (node == null)
return 0;
var resNode = &node->AtkResNode;
var rawHeight = (int)resNode->GetHeight();
if (rawHeight <= 0 && node->LineSpacing > 0)
rawHeight = node->LineSpacing;
if (rawHeight <= 0)
rawHeight = AtkNodeHelpers.DefaultTextNodeHeight;
var scale = resNode->ScaleY;
if (scale <= 0f)
scale = 1f;
var computed = (int)System.Math.Round(rawHeight * scale);
return System.Math.Max(1, computed);
}
private static unsafe int GetScaledTextWidth(AtkTextNode* node)
{
if (node == null)
return 0;
var resNode = &node->AtkResNode;
var rawWidth = (int)resNode->GetWidth();
if (rawWidth <= 0)
rawWidth = AtkNodeHelpers.DefaultTextNodeWidth;
var scale = resNode->ScaleX;
if (scale <= 0f)
scale = 1f;
pNode->SetText("Lightfinder");
}
}
}
var computed = (int)System.Math.Round(rawWidth * scale);
return System.Math.Max(1, computed);
}
internal static string NormalizeIconGlyph(string? rawInput)
{
if (string.IsNullOrWhiteSpace(rawInput))
return DefaultIconGlyph;
var trimmed = rawInput.Trim();
if (Enum.TryParse<SeIconChar>(trimmed, true, out var iconEnum))
return SeIconCharExtensions.ToIconString(iconEnum);
var hexCandidate = trimmed.StartsWith("0x", StringComparison.OrdinalIgnoreCase)
? trimmed[2..]
: trimmed;
if (ushort.TryParse(hexCandidate, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var hexValue))
return char.ConvertFromUtf32(hexValue);
var enumerator = trimmed.EnumerateRunes();
if (enumerator.MoveNext())
return enumerator.Current.ToString();
return DefaultIconGlyph;
}
internal static string ToIconEditorString(string? rawInput)
{
var normalized = NormalizeIconGlyph(rawInput);
var runeEnumerator = normalized.EnumerateRunes();
return runeEnumerator.MoveNext()
? runeEnumerator.Current.Value.ToString("X4", CultureInfo.InvariantCulture)
: DefaultIconGlyph;
}
private void HideNameplateTextNode(int i)
{
var pNode = mTextNodes[i];

View File

@@ -0,0 +1,220 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using LightlessSync.LightlessConfiguration;
using LightlessSync.Services.Mediator;
using Microsoft.Extensions.Logging;
namespace LightlessSync.Services;
public sealed class PairProcessingLimiter : DisposableMediatorSubscriberBase
{
private const int HardLimit = 32;
private readonly LightlessConfigService _configService;
private readonly object _limitLock = new();
private readonly SemaphoreSlim _semaphore;
private int _currentLimit;
private int _pendingReductions;
private int _waiting;
private int _inFlight;
public PairProcessingLimiter(ILogger<PairProcessingLimiter> logger, LightlessMediator mediator, LightlessConfigService configService)
: base(logger, mediator)
{
_configService = configService;
_currentLimit = CalculateLimit();
var initialCount = _configService.Current.EnablePairProcessingLimiter ? _currentLimit : HardLimit;
_semaphore = new SemaphoreSlim(initialCount, HardLimit);
Mediator.Subscribe<PairProcessingLimitChangedMessage>(this, _ => UpdateSemaphoreLimit());
}
public ValueTask<IAsyncDisposable> AcquireAsync(CancellationToken cancellationToken)
{
return WaitInternalAsync(cancellationToken);
}
public PairProcessingLimiterSnapshot GetSnapshot()
{
lock (_limitLock)
{
var enabled = IsEnabled;
var limit = enabled ? _currentLimit : CalculateLimit();
var waiting = Math.Max(0, Volatile.Read(ref _waiting));
var inFlight = Math.Max(0, Volatile.Read(ref _inFlight));
return new PairProcessingLimiterSnapshot(enabled, limit, inFlight, waiting);
}
}
private bool IsEnabled => _configService.Current.EnablePairProcessingLimiter;
private async ValueTask<IAsyncDisposable> WaitInternalAsync(CancellationToken token)
{
if (!IsEnabled)
{
return NoopReleaser.Instance;
}
Interlocked.Increment(ref _waiting);
try
{
await _semaphore.WaitAsync(token).ConfigureAwait(false);
}
catch
{
Interlocked.Decrement(ref _waiting);
throw;
}
Interlocked.Decrement(ref _waiting);
if (!IsEnabled)
{
_semaphore.Release();
return NoopReleaser.Instance;
}
Interlocked.Increment(ref _inFlight);
return new Releaser(this);
}
private void UpdateSemaphoreLimit()
{
lock (_limitLock)
{
var enabled = IsEnabled;
var desiredLimit = CalculateLimit();
if (!enabled)
{
var releaseAmount = HardLimit - _semaphore.CurrentCount;
if (releaseAmount > 0)
{
try
{
_semaphore.Release(releaseAmount);
}
catch (SemaphoreFullException)
{
// ignore, already at max
}
}
_currentLimit = desiredLimit;
_pendingReductions = 0;
return;
}
if (desiredLimit == _currentLimit)
{
return;
}
if (desiredLimit > _currentLimit)
{
var increment = desiredLimit - _currentLimit;
var allowed = Math.Min(increment, HardLimit - _semaphore.CurrentCount);
if (allowed > 0)
{
_semaphore.Release(allowed);
}
}
else
{
var decrement = _currentLimit - desiredLimit;
var removed = 0;
while (removed < decrement && _semaphore.Wait(0))
{
removed++;
}
var remaining = decrement - removed;
if (remaining > 0)
{
_pendingReductions += remaining;
}
}
_currentLimit = desiredLimit;
Logger.LogDebug("Pair processing concurrency updated to {limit} (pending reductions: {pending})", _currentLimit, _pendingReductions);
}
}
private int CalculateLimit()
{
var configured = _configService.Current.MaxConcurrentPairApplications;
return Math.Clamp(configured, 1, HardLimit);
}
private void ReleaseOne()
{
var inFlight = Interlocked.Decrement(ref _inFlight);
if (inFlight < 0)
{
Interlocked.Exchange(ref _inFlight, 0);
}
if (!IsEnabled)
{
return;
}
lock (_limitLock)
{
if (_pendingReductions > 0)
{
_pendingReductions--;
return;
}
}
_semaphore.Release();
}
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
if (!disposing)
{
return;
}
_semaphore.Dispose();
}
private sealed class Releaser : IAsyncDisposable
{
private PairProcessingLimiter? _owner;
public Releaser(PairProcessingLimiter owner)
{
_owner = owner;
}
public ValueTask DisposeAsync()
{
var owner = Interlocked.Exchange(ref _owner, null);
owner?.ReleaseOne();
return ValueTask.CompletedTask;
}
}
private sealed class NoopReleaser : IAsyncDisposable
{
public static readonly NoopReleaser Instance = new();
private NoopReleaser()
{
}
public ValueTask DisposeAsync()
{
return ValueTask.CompletedTask;
}
}
}
public readonly record struct PairProcessingLimiterSnapshot(bool IsEnabled, int Limit, int InFlight, int Waiting)
{
public int Remaining => Math.Max(0, Limit - InFlight);
}

View File

@@ -1,4 +1,5 @@
using Dalamud.Bindings.ImGui;
using System;
using Dalamud.Bindings.ImGui;
using Dalamud.Interface;
using Dalamud.Interface.Utility;
using Dalamud.Interface.Utility.Raii;
@@ -376,7 +377,7 @@ public class CompactUi : WindowMediatorSubscriberBase
private void DrawTransfers()
{
var currentUploads = _fileTransferManager.CurrentUploads.ToList();
var currentUploads = _fileTransferManager.GetCurrentUploadsSnapshot();
ImGui.AlignTextToFramePadding();
_uiSharedService.IconText(FontAwesomeIcon.Upload);
ImGui.SameLine(35 * ImGuiHelpers.GlobalScale);
@@ -386,10 +387,12 @@ public class CompactUi : WindowMediatorSubscriberBase
var totalUploads = currentUploads.Count;
var doneUploads = currentUploads.Count(c => c.IsTransferred);
var activeUploads = currentUploads.Count(c => !c.IsTransferred);
var uploadSlotLimit = Math.Clamp(_configService.Current.ParallelUploads, 1, 8);
var totalUploaded = currentUploads.Sum(c => c.Transferred);
var totalToUpload = currentUploads.Sum(c => c.Total);
ImGui.TextUnformatted($"{doneUploads}/{totalUploads}");
ImGui.TextUnformatted($"{doneUploads}/{totalUploads} (slots {activeUploads}/{uploadSlotLimit})");
var uploadText = $"({UiSharedService.ByteToString(totalUploaded)}/{UiSharedService.ByteToString(totalToUpload)})";
var textSize = ImGui.CalcTextSize(uploadText);
ImGui.SameLine(_windowContentWidth - textSize.X);
@@ -488,7 +491,7 @@ public class CompactUi : WindowMediatorSubscriberBase
float contentWidth = ImGui.GetWindowContentRegionMax().X - ImGui.GetWindowContentRegionMin().X;
float uidStartX = (contentWidth - uidTextSize.X) / 2f;
float cursorY = ImGui.GetCursorPosY();
float cursorY = ImGui.GetCursorPosY();
if (_configService.Current.BroadcastEnabled && _apiController.IsConnected)
{
@@ -619,7 +622,7 @@ public class CompactUi : WindowMediatorSubscriberBase
ImGui.SameLine();
ImGui.SetCursorPosY(cursorY + 15f);
_uiSharedService.IconText(FontAwesomeIcon.ExclamationTriangle, UIColors.Get("LightlessYellow"));
string warningMessage = "";
if (isOverTriHold)
{
@@ -825,7 +828,7 @@ public class CompactUi : WindowMediatorSubscriberBase
.Where(u => FilterGroupUsers(u.Value, group)));
filteredGroupPairs = filteredPairs
.Where(u => FilterGroupUsers( u.Value, group) && FilterOnlineOrPausedSelf(u.Key))
.Where(u => FilterGroupUsers(u.Value, group) && FilterOnlineOrPausedSelf(u.Key))
.OrderByDescending(u => u.Key.IsOnline)
.ThenBy(u =>
{

View File

@@ -1,4 +1,5 @@
using Dalamud.Bindings.ImGui;
using System;
using Dalamud.Bindings.ImGui;
using Dalamud.Interface.Colors;
using LightlessSync.LightlessConfiguration;
using LightlessSync.PlayerData.Handlers;
@@ -19,14 +20,16 @@ public class DownloadUi : WindowMediatorSubscriberBase
private readonly DalamudUtilService _dalamudUtilService;
private readonly FileUploadManager _fileTransferManager;
private readonly UiSharedService _uiShared;
private readonly PairProcessingLimiter _pairProcessingLimiter;
private readonly ConcurrentDictionary<GameObjectHandler, bool> _uploadingPlayers = new();
public DownloadUi(ILogger<DownloadUi> logger, DalamudUtilService dalamudUtilService, LightlessConfigService configService,
FileUploadManager fileTransferManager, LightlessMediator mediator, UiSharedService uiShared, PerformanceCollectorService performanceCollectorService)
PairProcessingLimiter pairProcessingLimiter, FileUploadManager fileTransferManager, LightlessMediator mediator, UiSharedService uiShared, PerformanceCollectorService performanceCollectorService)
: base(logger, mediator, "Lightless Sync Downloads", performanceCollectorService)
{
_dalamudUtilService = dalamudUtilService;
_configService = configService;
_pairProcessingLimiter = pairProcessingLimiter;
_fileTransferManager = fileTransferManager;
_uiShared = uiShared;
@@ -73,11 +76,25 @@ public class DownloadUi : WindowMediatorSubscriberBase
{
if (_configService.Current.ShowTransferWindow)
{
var limiterSnapshot = _pairProcessingLimiter.GetSnapshot();
if (limiterSnapshot.IsEnabled)
{
var queueColor = limiterSnapshot.Waiting > 0 ? ImGuiColors.DalamudYellow : ImGuiColors.DalamudGrey;
var queueText = $"Pair queue {limiterSnapshot.InFlight}/{limiterSnapshot.Limit}";
queueText += limiterSnapshot.Waiting > 0 ? $" ({limiterSnapshot.Waiting} waiting, {limiterSnapshot.Remaining} free)" : $" ({limiterSnapshot.Remaining} free)";
UiSharedService.DrawOutlinedFont(queueText, queueColor, new Vector4(0, 0, 0, 255), 1);
ImGui.NewLine();
}
else
{
UiSharedService.DrawOutlinedFont("Pair apply limiter disabled", ImGuiColors.DalamudGrey, new Vector4(0, 0, 0, 255), 1);
ImGui.NewLine();
}
try
{
if (_fileTransferManager.CurrentUploads.Any())
if (_fileTransferManager.IsUploading)
{
var currentUploads = _fileTransferManager.CurrentUploads.ToList();
var currentUploads = _fileTransferManager.GetCurrentUploadsSnapshot();
var totalUploads = currentUploads.Count;
var doneUploads = currentUploads.Count(c => c.IsTransferred);
@@ -214,7 +231,7 @@ public class DownloadUi : WindowMediatorSubscriberBase
{
if (_uiShared.EditTrackerPosition) return true;
if (!_configService.Current.ShowTransferWindow && !_configService.Current.ShowTransferBars) return false;
if (!_currentDownloads.Any() && !_fileTransferManager.CurrentUploads.Any() && !_uploadingPlayers.Any()) return false;
if (!_currentDownloads.Any() && !_fileTransferManager.IsUploading && !_uploadingPlayers.Any()) return false;
if (!IsOpen) return false;
return true;
}

View File

@@ -1,4 +1,5 @@
using Dalamud.Bindings.ImGui;
using Dalamud.Game.Text;
using Dalamud.Interface;
using Dalamud.Interface.Colors;
using Dalamud.Interface.Utility;
@@ -10,6 +11,7 @@ using LightlessSync.API.Routes;
using LightlessSync.FileCache;
using LightlessSync.Interop.Ipc;
using LightlessSync.LightlessConfiguration;
using LightlessSync.LightlessConfiguration.Configurations;
using LightlessSync.LightlessConfiguration.Models;
using LightlessSync.PlayerData.Handlers;
using LightlessSync.PlayerData.Pairs;
@@ -17,15 +19,18 @@ using LightlessSync.Services;
using LightlessSync.Services.Mediator;
using LightlessSync.Services.ServerConfiguration;
using LightlessSync.Utils;
using LightlessSync.UtilsEnum.Enum;
using LightlessSync.WebAPI;
using LightlessSync.WebAPI.Files;
using LightlessSync.WebAPI.Files.Models;
using LightlessSync.WebAPI.SignalR.Utils;
using Microsoft.AspNetCore.Http.Connections;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Numerics;
@@ -50,10 +55,12 @@ public class SettingsUi : WindowMediatorSubscriberBase
private readonly PairManager _pairManager;
private readonly PerformanceCollectorService _performanceCollector;
private readonly PlayerPerformanceConfigService _playerPerformanceConfigService;
private readonly PairProcessingLimiter _pairProcessingLimiter;
private readonly ServerConfigurationManager _serverConfigurationManager;
private readonly UiSharedService _uiShared;
private readonly IProgress<(int, int, FileCacheEntity)> _validationProgress;
private readonly NameplateService _nameplateService;
private readonly NameplateHandler _nameplateHandler;
private (int, int, FileCacheEntity) _currentProgress;
private bool _deleteAccountPopupModalShown = false;
private bool _deleteFilesPopupModalShown = false;
@@ -63,6 +70,23 @@ public class SettingsUi : WindowMediatorSubscriberBase
private bool _readClearCache = false;
private int _selectedEntry = -1;
private string _uidToAddForIgnore = string.Empty;
private string _lightfinderIconInput = string.Empty;
private bool _lightfinderIconInputInitialized = false;
private int _lightfinderIconPresetIndex = -1;
private static readonly (string Label, SeIconChar Icon)[] LightfinderIconPresets = new[]
{
("Link Marker", SeIconChar.LinkMarker),
("Hyadelyn", SeIconChar.Hyadelyn),
("Gil", SeIconChar.Gil),
("Quest Sync", SeIconChar.QuestSync),
("Glamoured", SeIconChar.Glamoured),
("Glamoured (Dyed)", SeIconChar.GlamouredDyed),
("Auto-Translate Open", SeIconChar.AutoTranslateOpen),
("Auto-Translate Close", SeIconChar.AutoTranslateClose),
("Boxed Star", SeIconChar.BoxedStar),
("Boxed Plus", SeIconChar.BoxedPlus)
};
private CancellationTokenSource? _validationCts;
private Task<List<FileCacheEntity>>? _validationTask;
private bool _wasOpen = false;
@@ -72,6 +96,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
PairManager pairManager,
ServerConfigurationManager serverConfigurationManager,
PlayerPerformanceConfigService playerPerformanceConfigService,
PairProcessingLimiter pairProcessingLimiter,
LightlessMediator mediator, PerformanceCollectorService performanceCollector,
FileUploadManager fileTransferManager,
FileTransferOrchestrator fileTransferOrchestrator,
@@ -79,12 +104,14 @@ public class SettingsUi : WindowMediatorSubscriberBase
FileCompactor fileCompactor, ApiController apiController,
IpcManager ipcManager, CacheMonitor cacheMonitor,
DalamudUtilService dalamudUtilService, HttpClient httpClient,
NameplateService nameplateService) : base(logger, mediator, "Lightless Sync Settings", performanceCollector)
NameplateService nameplateService,
NameplateHandler nameplateHandler) : base(logger, mediator, "Lightless Sync Settings", performanceCollector)
{
_configService = configService;
_pairManager = pairManager;
_serverConfigurationManager = serverConfigurationManager;
_playerPerformanceConfigService = playerPerformanceConfigService;
_pairProcessingLimiter = pairProcessingLimiter;
_performanceCollector = performanceCollector;
_fileTransferManager = fileTransferManager;
_fileTransferOrchestrator = fileTransferOrchestrator;
@@ -97,6 +124,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
_fileCompactor = fileCompactor;
_uiShared = uiShared;
_nameplateService = nameplateService;
_nameplateHandler = nameplateHandler;
AllowClickthrough = false;
AllowPinning = true;
_validationProgress = new Progress<(int, int, FileCacheEntity)>(v => _currentProgress = v);
@@ -218,6 +246,9 @@ public class SettingsUi : WindowMediatorSubscriberBase
ImGuiHelpers.ScaledDummy(5);
int maxParallelDownloads = _configService.Current.ParallelDownloads;
int maxParallelUploads = _configService.Current.ParallelUploads;
int maxPairApplications = _configService.Current.MaxConcurrentPairApplications;
bool limitPairApplications = _configService.Current.EnablePairProcessingLimiter;
bool useAlternativeUpload = _configService.Current.UseAlternativeFileUpload;
int downloadSpeedLimit = _configService.Current.DownloadSpeedLimitInBytes;
@@ -254,7 +285,60 @@ public class SettingsUi : WindowMediatorSubscriberBase
{
_configService.Current.ParallelDownloads = maxParallelDownloads;
_configService.Save();
Mediator.Publish(new DownloadLimitChangedMessage());
}
_uiShared.DrawHelpText("Controls how many download slots can be active at once.");
if (ImGui.SliderInt("Maximum Parallel Uploads", ref maxParallelUploads, 1, 8))
{
_configService.Current.ParallelUploads = maxParallelUploads;
_configService.Save();
}
_uiShared.DrawHelpText("Controls how many uploads can run at once.");
_uiShared.ColoredSeparator(UIColors.Get("LightlessPurple"), 2f);
if (ImGui.Checkbox("Enable Pair Download Limiter", ref limitPairApplications))
{
_configService.Current.EnablePairProcessingLimiter = limitPairApplications;
_configService.Save();
Mediator.Publish(new PairProcessingLimitChangedMessage());
}
_uiShared.DrawHelpText("When enabled we stagger pair downloads to avoid large network and game lag caused by attempting to download everyone at once.");
var limiterDisabledScope = !limitPairApplications;
if (limiterDisabledScope)
{
ImGui.BeginDisabled();
}
if (ImGui.SliderInt("Maximum Concurrent Pair Downloads", ref maxPairApplications, 1, 6))
{
_configService.Current.MaxConcurrentPairApplications = maxPairApplications;
_configService.Save();
Mediator.Publish(new PairProcessingLimitChangedMessage());
}
_uiShared.DrawHelpText("How many pair downloads/applications can run simultaneously when the limit is on.");
if (limiterDisabledScope)
{
ImGui.EndDisabled();
}
var limiterSnapshot = _pairProcessingLimiter.GetSnapshot();
if (limiterSnapshot.IsEnabled)
{
var queueColor = limiterSnapshot.Waiting > 0 ? ImGuiColors.DalamudYellow : ImGuiColors.DalamudGrey;
var queueText = $"Pair queue {limiterSnapshot.InFlight}/{limiterSnapshot.Limit}";
queueText += limiterSnapshot.Waiting > 0 ? $" ({limiterSnapshot.Waiting} waiting, {limiterSnapshot.Remaining} free)" : $" ({limiterSnapshot.Remaining} free)";
ImGui.TextColored(queueColor, queueText);
}
else
{
ImGui.TextColored(ImGuiColors.DalamudGrey, "Pair apply limiter is disabled.");
}
_uiShared.ColoredSeparator(UIColors.Get("LightlessPurple"), 2f);
if (ImGui.Checkbox("Use Alternative Upload Method", ref useAlternativeUpload))
{
@@ -409,25 +493,33 @@ public class SettingsUi : WindowMediatorSubscriberBase
{
if (ApiController.ServerState is ServerState.Connected && ImGui.BeginTabItem("Transfers"))
{
ImGui.TextUnformatted("Uploads");
var uploadsSnapshot = _fileTransferManager.GetCurrentUploadsSnapshot();
var activeUploads = uploadsSnapshot.Count(c => !c.IsTransferred);
var uploadSlotLimit = Math.Clamp(_configService.Current.ParallelUploads, 1, 8);
ImGui.TextUnformatted($"Uploads (slots {activeUploads}/{uploadSlotLimit})");
if (ImGui.BeginTable("UploadsTable", 3))
{
ImGui.TableSetupColumn("File");
ImGui.TableSetupColumn("Uploaded");
ImGui.TableSetupColumn("Size");
ImGui.TableHeadersRow();
foreach (var transfer in _fileTransferManager.CurrentUploads.ToArray())
foreach (var transfer in uploadsSnapshot)
{
var color = UiSharedService.UploadColor((transfer.Transferred, transfer.Total));
var col = ImRaii.PushColor(ImGuiCol.Text, color);
using var col = ImRaii.PushColor(ImGuiCol.Text, color);
ImGui.TableNextColumn();
ImGui.TextUnformatted(transfer.Hash);
if (transfer is UploadFileTransfer uploadTransfer)
{
ImGui.TextUnformatted(uploadTransfer.LocalFile);
}
else
{
ImGui.TextUnformatted(transfer.Hash);
}
ImGui.TableNextColumn();
ImGui.TextUnformatted(UiSharedService.ByteToString(transfer.Transferred));
ImGui.TableNextColumn();
ImGui.TextUnformatted(UiSharedService.ByteToString(transfer.Total));
col.Dispose();
ImGui.TableNextRow();
}
ImGui.EndTable();
@@ -972,6 +1064,191 @@ public class SettingsUi : WindowMediatorSubscriberBase
ImGui.Separator();
if (_uiShared.MediumTreeNode("Lightfinder", UIColors.Get("LightlessPurple")))
{
var offsetX = (int)_configService.Current.LightfinderLabelOffsetX;
if (ImGui.SliderInt("Label Offset X", ref offsetX, -200, 200))
{
_configService.Current.LightfinderLabelOffsetX = (short)offsetX;
_configService.Save();
_nameplateHandler.FlagRefresh();
_nameplateService.RequestRedraw();
}
_uiShared.DrawHelpText("Moves the Lightfinder label horizontally on player nameplates.");
var offsetY = (int)_configService.Current.LightfinderLabelOffsetY;
if (ImGui.SliderInt("Label Offset Y", ref offsetY, -200, 200))
{
_configService.Current.LightfinderLabelOffsetY = (short)offsetY;
_configService.Save();
_nameplateHandler.FlagRefresh();
_nameplateService.RequestRedraw();
}
_uiShared.DrawHelpText("Moves the Lightfinder label vertically on player nameplates.");
var labelScale = _configService.Current.LightfinderLabelScale;
if (ImGui.SliderFloat("Label Size", ref labelScale, 0.5f, 2.0f, "%.2fx"))
{
_configService.Current.LightfinderLabelScale = labelScale;
_configService.Save();
_nameplateHandler.FlagRefresh();
_nameplateService.RequestRedraw();
}
_uiShared.DrawHelpText("Adjusts the Lightfinder label size for both text and icon modes.");
var autoAlign = _configService.Current.LightfinderAutoAlign;
if (ImGui.Checkbox("Automatically align with nameplate", ref autoAlign))
{
_configService.Current.LightfinderAutoAlign = autoAlign;
_configService.Save();
_nameplateHandler.FlagRefresh();
_nameplateService.RequestRedraw();
}
_uiShared.DrawHelpText("Automatically position the label relative to the in-game nameplate. Turn off to rely entirely on manual offsets.");
if (autoAlign)
{
var alignmentOption = _configService.Current.LabelAlignment;
var alignmentLabel = alignmentOption switch
{
LabelAlignment.Left => "Left",
LabelAlignment.Right => "Right",
_ => "Center",
};
if (ImGui.BeginCombo("Horizontal Alignment", alignmentLabel))
{
foreach (LabelAlignment option in Enum.GetValues<LabelAlignment>())
{
var optionLabel = option switch
{
LabelAlignment.Left => "Left",
LabelAlignment.Right => "Right",
_ => "Center",
};
var selected = option == alignmentOption;
if (ImGui.Selectable(optionLabel, selected))
{
_configService.Current.LabelAlignment = option;
_configService.Save();
_nameplateHandler.FlagRefresh();
_nameplateService.RequestRedraw();
}
if (selected)
ImGui.SetItemDefaultFocus();
}
ImGui.EndCombo();
}
}
var useIcon = _configService.Current.LightfinderLabelUseIcon;
if (ImGui.Checkbox("Show icon instead of text", ref useIcon))
{
_configService.Current.LightfinderLabelUseIcon = useIcon;
_configService.Save();
_nameplateHandler.FlagRefresh();
_nameplateService.RequestRedraw();
if (useIcon)
{
RefreshLightfinderIconState();
}
else
{
_lightfinderIconInputInitialized = false;
_lightfinderIconPresetIndex = -1;
}
}
_uiShared.DrawHelpText("Switch between the Lightfinder text label and an icon on nameplates.");
if (useIcon)
{
if (!_lightfinderIconInputInitialized)
{
RefreshLightfinderIconState();
}
var currentPresetLabel = _lightfinderIconPresetIndex >= 0
? $"{GetLightfinderPresetGlyph(_lightfinderIconPresetIndex)} {LightfinderIconPresets[_lightfinderIconPresetIndex].Label}"
: "Custom";
if (ImGui.BeginCombo("Preset Icon", currentPresetLabel))
{
for (int i = 0; i < LightfinderIconPresets.Length; i++)
{
var optionGlyph = GetLightfinderPresetGlyph(i);
var preview = $"{optionGlyph} {LightfinderIconPresets[i].Label}";
var selected = i == _lightfinderIconPresetIndex;
if (ImGui.Selectable(preview, selected))
{
ApplyLightfinderIcon(optionGlyph, i);
}
}
if (ImGui.Selectable("Custom", _lightfinderIconPresetIndex == -1))
{
_lightfinderIconPresetIndex = -1;
}
ImGui.EndCombo();
}
var editorBuffer = _lightfinderIconInput;
if (ImGui.InputText("Icon Glyph", ref editorBuffer, 16))
{
_lightfinderIconInput = editorBuffer;
_lightfinderIconPresetIndex = -1;
}
if (ImGui.Button("Apply Icon"))
{
var normalized = NameplateHandler.NormalizeIconGlyph(_lightfinderIconInput);
ApplyLightfinderIcon(normalized, _lightfinderIconPresetIndex);
}
ImGui.SameLine();
if (ImGui.Button("Reset Icon"))
{
var defaultGlyph = NameplateHandler.NormalizeIconGlyph(null);
var defaultIndex = -1;
for (int i = 0; i < LightfinderIconPresets.Length; i++)
{
if (string.Equals(GetLightfinderPresetGlyph(i), defaultGlyph, StringComparison.Ordinal))
{
defaultIndex = i;
break;
}
}
if (defaultIndex < 0)
{
defaultIndex = 0;
}
ApplyLightfinderIcon(GetLightfinderPresetGlyph(defaultIndex), defaultIndex);
}
var previewGlyph = NameplateHandler.NormalizeIconGlyph(_lightfinderIconInput);
ImGui.SameLine();
ImGui.AlignTextToFramePadding();
ImGui.Text($"Preview: {previewGlyph}");
_uiShared.DrawHelpText("Enter a hex code (e.g. E0BB), pick a preset, or paste an icon character directly.");
}
else
{
_lightfinderIconInputInitialized = false;
_lightfinderIconPresetIndex = -1;
}
_uiShared.ColoredSeparator(UIColors.Get("LightlessPurple"), 1.5f);
ImGui.TreePop();
}
ImGui.Separator();
if (_uiShared.MediumTreeNode("Colors", UIColors.Get("LightlessPurple")))
{
ImGui.TextUnformatted("UI Theme Colors");
@@ -2216,6 +2493,39 @@ public class SettingsUi : WindowMediatorSubscriberBase
return (true, failedConversions.Count != 0, sb.ToString());
}
private static string GetLightfinderPresetGlyph(int index)
{
return NameplateHandler.NormalizeIconGlyph(SeIconCharExtensions.ToIconString(LightfinderIconPresets[index].Icon));
}
private void RefreshLightfinderIconState()
{
var normalized = NameplateHandler.NormalizeIconGlyph(_configService.Current.LightfinderLabelIconGlyph);
_lightfinderIconInput = NameplateHandler.ToIconEditorString(normalized);
_lightfinderIconInputInitialized = true;
_lightfinderIconPresetIndex = -1;
for (int i = 0; i < LightfinderIconPresets.Length; i++)
{
if (string.Equals(GetLightfinderPresetGlyph(i), normalized, StringComparison.Ordinal))
{
_lightfinderIconPresetIndex = i;
break;
}
}
}
private void ApplyLightfinderIcon(string normalizedGlyph, int presetIndex)
{
_configService.Current.LightfinderLabelIconGlyph = normalizedGlyph;
_configService.Save();
_nameplateHandler.FlagRefresh();
_nameplateService.RequestRedraw();
_lightfinderIconInput = NameplateHandler.ToIconEditorString(normalizedGlyph);
_lightfinderIconPresetIndex = presetIndex;
_lightfinderIconInputInitialized = true;
}
private void DrawSettingsContent()
{
if (_apiController.ServerState is ServerState.Connected)

View File

@@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace LightlessSync.UtilsEnum.Enum
{
public enum LabelAlignment
{
Left,
Center,
Right,
}
}

View File

@@ -1,4 +1,4 @@
using Dalamud.Utility;
using Dalamud.Utility;
using K4os.Compression.LZ4.Legacy;
using LightlessSync.API.Data;
using LightlessSync.API.Dto.Files;
@@ -8,6 +8,7 @@ using LightlessSync.PlayerData.Handlers;
using LightlessSync.Services.Mediator;
using LightlessSync.WebAPI.Files.Models;
using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
using System.Net;
using System.Net.Http.Json;
@@ -19,7 +20,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
private readonly FileCompactor _fileCompactor;
private readonly FileCacheManager _fileDbManager;
private readonly FileTransferOrchestrator _orchestrator;
private readonly List<ThrottledStream> _activeDownloadStreams;
private readonly ConcurrentDictionary<ThrottledStream, byte> _activeDownloadStreams;
public FileDownloadManager(ILogger<FileDownloadManager> logger, LightlessMediator mediator,
FileTransferOrchestrator orchestrator,
@@ -29,14 +30,14 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
_orchestrator = orchestrator;
_fileDbManager = fileCacheManager;
_fileCompactor = fileCompactor;
_activeDownloadStreams = [];
_activeDownloadStreams = new();
Mediator.Subscribe<DownloadLimitChangedMessage>(this, (msg) =>
{
if (!_activeDownloadStreams.Any()) return;
if (_activeDownloadStreams.IsEmpty) return;
var newLimit = _orchestrator.DownloadLimitPerSlot();
Logger.LogTrace("Setting new Download Speed Limit to {newLimit}", newLimit);
foreach (var stream in _activeDownloadStreams)
foreach (var stream in _activeDownloadStreams.Keys)
{
stream.BandwidthLimit = newLimit;
}
@@ -47,7 +48,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
public List<FileTransfer> ForbiddenTransfers => _orchestrator.ForbiddenTransfers;
public bool IsDownloading => !CurrentDownloads.Any();
public bool IsDownloading => CurrentDownloads.Any();
public static void MungeBuffer(Span<byte> buffer)
{
@@ -84,7 +85,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
protected override void Dispose(bool disposing)
{
ClearDownload();
foreach (var stream in _activeDownloadStreams.ToList())
foreach (var stream in _activeDownloadStreams.Keys.ToList())
{
try
{
@@ -95,6 +96,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
// do nothing
//
}
finally
{
_activeDownloadStreams.TryRemove(stream, out _);
}
}
base.Dispose(disposing);
}
@@ -142,7 +147,14 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
await WaitForDownloadReady(fileTransfer, requestId, ct).ConfigureAwait(false);
_downloadStatus[downloadGroup].DownloadStatus = DownloadStatus.Downloading;
if (_downloadStatus.TryGetValue(downloadGroup, out var downloadStatus))
{
downloadStatus.DownloadStatus = DownloadStatus.Downloading;
}
else
{
Logger.LogWarning("Download status missing for {group} when starting download", downloadGroup);
}
const int maxRetries = 3;
int retryCount = 0;
@@ -204,7 +216,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
stream = new(await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false), limit);
_activeDownloadStreams.Add(stream);
_activeDownloadStreams.TryAdd(stream, 0);
while ((bytesRead = await stream.ReadAsync(buffer, ct).ConfigureAwait(false)) > 0)
{
@@ -245,7 +257,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
{
if (stream != null)
{
_activeDownloadStreams.Remove(stream);
_activeDownloadStreams.TryRemove(stream, out _);
await stream.DisposeAsync().ConfigureAwait(false);
}
}
@@ -253,11 +265,28 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
public async Task<List<DownloadFileTransfer>> InitiateDownloadList(GameObjectHandler gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct)
{
Logger.LogDebug("Download start: {id}", gameObjectHandler.Name);
var objectName = gameObjectHandler?.Name ?? "Unknown";
Logger.LogDebug("Download start: {id}", objectName);
if (fileReplacement == null || fileReplacement.Count == 0)
{
Logger.LogDebug("{dlName}: No file replacements provided", objectName);
CurrentDownloads = [];
return CurrentDownloads;
}
var hashes = fileReplacement.Where(f => f != null && !string.IsNullOrWhiteSpace(f.Hash)).Select(f => f.Hash).Distinct(StringComparer.Ordinal).ToList();
if (hashes.Count == 0)
{
Logger.LogDebug("{dlName}: No valid hashes to download", objectName);
CurrentDownloads = [];
return CurrentDownloads;
}
List<DownloadFileDto> downloadFileInfoFromService =
[
.. await FilesGetSizes(fileReplacement.Select(f => f.Hash).Distinct(StringComparer.Ordinal).ToList(), ct).ConfigureAwait(false),
.. await FilesGetSizes(hashes, ct).ConfigureAwait(false),
];
Logger.LogDebug("Files with size 0 or less: {files}", string.Join(", ", downloadFileInfoFromService.Where(f => f.Size <= 0).Select(f => f.Hash)));
@@ -315,15 +344,23 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
FileInfo fi = new(blockFile);
try
{
_downloadStatus[fileGroup.Key].DownloadStatus = DownloadStatus.WaitingForSlot;
if (!_downloadStatus.TryGetValue(fileGroup.Key, out var downloadStatus))
{
Logger.LogWarning("Download status missing for {group}, aborting", fileGroup.Key);
return;
}
downloadStatus.DownloadStatus = DownloadStatus.WaitingForSlot;
await _orchestrator.WaitForDownloadSlotAsync(token).ConfigureAwait(false);
_downloadStatus[fileGroup.Key].DownloadStatus = DownloadStatus.WaitingForQueue;
downloadStatus.DownloadStatus = DownloadStatus.WaitingForQueue;
Progress<long> progress = new((bytesDownloaded) =>
{
try
{
if (!_downloadStatus.TryGetValue(fileGroup.Key, out FileDownloadStatus? value)) return;
value.TransferredBytes += bytesDownloaded;
if (_downloadStatus.TryGetValue(fileGroup.Key, out FileDownloadStatus? value))
{
value.TransferredBytes += bytesDownloaded;
}
}
catch (Exception ex)
{
@@ -353,6 +390,12 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
status.TransferredFiles = 1;
status.DownloadStatus = DownloadStatus.Decompressing;
}
if (!File.Exists(blockFile))
{
Logger.LogWarning("{dlName}: Block file missing before extraction, skipping", fi.Name);
return;
}
fileBlockStream = File.OpenRead(blockFile);
while (fileBlockStream.Position < fileBlockStream.Length)
{

View File

@@ -1,4 +1,4 @@
using LightlessSync.API.Data;
using LightlessSync.API.Data;
using LightlessSync.API.Dto.Files;
using LightlessSync.API.Routes;
using LightlessSync.FileCache;
@@ -10,6 +10,8 @@ using LightlessSync.WebAPI.Files.Models;
using Microsoft.Extensions.Logging;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Collections.Concurrent;
using System.Threading;
namespace LightlessSync.WebAPI.Files;
@@ -19,7 +21,9 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
private readonly LightlessConfigService _lightlessConfigService;
private readonly FileTransferOrchestrator _orchestrator;
private readonly ServerConfigurationManager _serverManager;
private readonly Dictionary<string, DateTime> _verifiedUploadedHashes = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, DateTime> _verifiedUploadedHashes = new(StringComparer.Ordinal);
private readonly object _currentUploadsLock = new();
private readonly Dictionary<string, FileTransfer> _currentUploadsByHash = new(StringComparer.Ordinal);
private CancellationTokenSource? _uploadCancellationTokenSource = new();
public FileUploadManager(ILogger<FileUploadManager> logger, LightlessMediator mediator,
@@ -40,17 +44,38 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
}
public List<FileTransfer> CurrentUploads { get; } = [];
public bool IsUploading => CurrentUploads.Count > 0;
public bool IsUploading
{
get
{
lock (_currentUploadsLock)
{
return CurrentUploads.Count > 0;
}
}
}
public List<FileTransfer> GetCurrentUploadsSnapshot()
{
lock (_currentUploadsLock)
{
return CurrentUploads.ToList();
}
}
public bool CancelUpload()
{
if (CurrentUploads.Any())
if (IsUploading)
{
Logger.LogDebug("Cancelling current upload");
_uploadCancellationTokenSource?.Cancel();
_uploadCancellationTokenSource?.Dispose();
_uploadCancellationTokenSource = null;
CurrentUploads.Clear();
lock (_currentUploadsLock)
{
CurrentUploads.Clear();
_currentUploadsByHash.Clear();
}
return true;
}
@@ -83,22 +108,44 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
return [.. filesToUpload.Where(f => f.IsForbidden).Select(f => f.Hash)];
}
Task uploadTask = Task.CompletedTask;
var cancellationToken = ct ?? CancellationToken.None;
var parallelUploads = Math.Clamp(_lightlessConfigService.Current.ParallelUploads, 1, 8);
using SemaphoreSlim uploadSlots = new(parallelUploads, parallelUploads);
List<Task> uploadTasks = new();
int i = 1;
foreach (var file in filesToUpload)
{
progress.Report($"Uploading file {i++}/{filesToUpload.Count}. Please wait until the upload is completed.");
Logger.LogDebug("[{hash}] Compressing", file);
var data = await _fileDbManager.GetCompressedFileData(file.Hash, ct ?? CancellationToken.None).ConfigureAwait(false);
Logger.LogDebug("[{hash}] Starting upload for {filePath}", data.Item1, _fileDbManager.GetFileCacheByHash(data.Item1)!.ResolvedFilepath);
await uploadTask.ConfigureAwait(false);
uploadTask = UploadFile(data.Item2, file.Hash, postProgress: false, ct ?? CancellationToken.None);
(ct ?? CancellationToken.None).ThrowIfCancellationRequested();
uploadTasks.Add(UploadSingleFileAsync(file, uploadSlots, cancellationToken));
}
await uploadTask.ConfigureAwait(false);
await Task.WhenAll(uploadTasks).ConfigureAwait(false);
return [];
async Task UploadSingleFileAsync(UploadFileDto fileDto, SemaphoreSlim gate, CancellationToken token)
{
await gate.WaitAsync(token).ConfigureAwait(false);
try
{
token.ThrowIfCancellationRequested();
Logger.LogDebug("[{hash}] Compressing", fileDto.Hash);
var data = await _fileDbManager.GetCompressedFileData(fileDto.Hash, token).ConfigureAwait(false);
var cacheEntry = _fileDbManager.GetFileCacheByHash(data.Item1);
if (cacheEntry != null)
{
Logger.LogDebug("[{hash}] Starting upload for {filePath}", data.Item1, cacheEntry.ResolvedFilepath);
}
await UploadFile(data.Item2, fileDto.Hash, postProgress: false, token).ConfigureAwait(false);
}
finally
{
gate.Release();
}
}
}
public async Task<CharacterData> UploadFiles(CharacterData data, List<UserData> visiblePlayers)
@@ -167,7 +214,11 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
_uploadCancellationTokenSource?.Cancel();
_uploadCancellationTokenSource?.Dispose();
_uploadCancellationTokenSource = null;
CurrentUploads.Clear();
lock (_currentUploadsLock)
{
CurrentUploads.Clear();
_currentUploadsByHash.Clear();
}
_verifiedUploadedHashes.Clear();
}
@@ -211,7 +262,17 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
{
try
{
CurrentUploads.Single(f => string.Equals(f.Hash, fileHash, StringComparison.Ordinal)).Transferred = prog.Uploaded;
lock (_currentUploadsLock)
{
if (_currentUploadsByHash.TryGetValue(fileHash, out var transfer))
{
transfer.Transferred = prog.Uploaded;
}
else
{
Logger.LogDebug("[{hash}] Could not find upload transfer during progress update", fileHash);
}
}
}
catch (Exception ex)
{
@@ -240,10 +301,16 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
{
try
{
CurrentUploads.Add(new UploadFileTransfer(file)
var uploadTransfer = new UploadFileTransfer(file)
{
Total = new FileInfo(_fileDbManager.GetFileCacheByHash(file.Hash)!.ResolvedFilepath).Length,
});
};
lock (_currentUploadsLock)
{
CurrentUploads.Add(uploadTransfer);
_currentUploadsByHash[file.Hash] = uploadTransfer;
}
}
catch (Exception ex)
{
@@ -264,33 +331,75 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
_verifiedUploadedHashes[file.Hash] = DateTime.UtcNow;
}
var totalSize = CurrentUploads.Sum(c => c.Total);
long totalSize;
List<FileTransfer> pendingUploads;
lock (_currentUploadsLock)
{
totalSize = CurrentUploads.Sum(c => c.Total);
pendingUploads = CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList();
}
var parallelUploads = Math.Clamp(_lightlessConfigService.Current.ParallelUploads, 1, 8);
using SemaphoreSlim uploadSlots = new(parallelUploads, parallelUploads);
Logger.LogDebug("Compressing and uploading files");
Task uploadTask = Task.CompletedTask;
foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList())
List<Task> uploadTasks = new();
foreach (var transfer in pendingUploads)
{
Logger.LogDebug("[{hash}] Compressing", file);
var data = await _fileDbManager.GetCompressedFileData(file.Hash, uploadToken).ConfigureAwait(false);
CurrentUploads.Single(e => string.Equals(e.Hash, data.Item1, StringComparison.Ordinal)).Total = data.Item2.Length;
Logger.LogDebug("[{hash}] Starting upload for {filePath}", data.Item1, _fileDbManager.GetFileCacheByHash(data.Item1)!.ResolvedFilepath);
await uploadTask.ConfigureAwait(false);
uploadTask = UploadFile(data.Item2, file.Hash, true, uploadToken);
uploadToken.ThrowIfCancellationRequested();
uploadTasks.Add(UploadPendingFileAsync(transfer, uploadSlots, uploadToken));
}
if (CurrentUploads.Any())
{
await uploadTask.ConfigureAwait(false);
await Task.WhenAll(uploadTasks).ConfigureAwait(false);
var compressedSize = CurrentUploads.Sum(c => c.Total);
Logger.LogDebug("Upload complete, compressed {size} to {compressed}", UiSharedService.ByteToString(totalSize), UiSharedService.ByteToString(compressedSize));
long compressedSize;
HashSet<string> uploadedHashes;
lock (_currentUploadsLock)
{
compressedSize = CurrentUploads.Sum(c => c.Total);
uploadedHashes = CurrentUploads.Select(u => u.Hash).ToHashSet(StringComparer.Ordinal);
}
foreach (var file in unverifiedUploadHashes.Where(c => !CurrentUploads.Exists(u => string.Equals(u.Hash, c, StringComparison.Ordinal))))
Logger.LogDebug("Upload complete, compressed {size} to {compressed}", UiSharedService.ByteToString(totalSize), UiSharedService.ByteToString(compressedSize));
foreach (var file in unverifiedUploadHashes.Where(c => !uploadedHashes.Contains(c)))
{
_verifiedUploadedHashes[file] = DateTime.UtcNow;
}
CurrentUploads.Clear();
lock (_currentUploadsLock)
{
CurrentUploads.Clear();
_currentUploadsByHash.Clear();
}
async Task UploadPendingFileAsync(FileTransfer transfer, SemaphoreSlim gate, CancellationToken token)
{
await gate.WaitAsync(token).ConfigureAwait(false);
try
{
token.ThrowIfCancellationRequested();
Logger.LogDebug("[{hash}] Compressing", transfer.Hash);
var data = await _fileDbManager.GetCompressedFileData(transfer.Hash, token).ConfigureAwait(false);
lock (_currentUploadsLock)
{
if (_currentUploadsByHash.TryGetValue(data.Item1, out var trackedUpload))
{
trackedUpload.Total = data.Item2.Length;
}
}
var cacheEntry = _fileDbManager.GetFileCacheByHash(data.Item1);
if (cacheEntry != null)
{
Logger.LogDebug("[{hash}] Starting upload for {filePath}", data.Item1, cacheEntry.ResolvedFilepath);
}
await UploadFile(data.Item2, transfer.Hash, true, token).ConfigureAwait(false);
}
finally
{
gate.Release();
}
}
}
}