diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 165a58c..808c257 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -21,6 +21,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase private long _currentFileProgress = 0; private CancellationTokenSource _scanCancellationTokenSource = new(); private readonly CancellationTokenSource _periodicCalculationTokenSource = new(); + private readonly SemaphoreSlim _dbGate = new(1, 1); public static readonly IImmutableList AllowedFileExtensions = [".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".pbd", ".scd", ".skp", ".shpk", ".kdb"]; private static readonly HashSet AllowedFileExtensionSet = new(AllowedFileExtensions, StringComparer.OrdinalIgnoreCase); @@ -68,6 +69,9 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase { Logger.LogInformation("Starting Periodic Storage Directory Calculation Task"); var token = _periodicCalculationTokenSource.Token; + while (IsHalted() && !token.IsCancellationRequested) + await Task.Delay(TimeSpan.FromSeconds(5), token).ConfigureAwait(false); + while (!token.IsCancellationRequested) { try @@ -91,6 +95,9 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase public long CurrentFileProgress => _currentFileProgress; public long FileCacheSize { get; set; } public long FileCacheDriveFree { get; set; } + + private int _haltCount; + private bool IsHalted() => Volatile.Read(ref _haltCount) > 0; public ConcurrentDictionary HaltScanLocks { get; set; } = new(StringComparer.Ordinal); public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0; public long TotalFiles { get; private set; } @@ -98,14 +105,36 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase public void HaltScan(string source) { - if (!HaltScanLocks.ContainsKey(source)) HaltScanLocks[source] = 0; - HaltScanLocks[source]++; + HaltScanLocks.AddOrUpdate(source, 1, (_, v) => v + 1); + Interlocked.Increment(ref _haltCount); } record WatcherChange(WatcherChangeTypes ChangeType, string? OldPath = null); private readonly record struct CacheEvictionCandidate(string FullPath, long Size, DateTime LastAccessTime); - private readonly Dictionary _watcherChanges = new(StringComparer.OrdinalIgnoreCase); - private readonly Dictionary _lightlessChanges = new(StringComparer.OrdinalIgnoreCase); + private readonly object _penumbraGate = new(); + private Dictionary _watcherChanges = new(StringComparer.OrdinalIgnoreCase); + + private readonly object _lightlessGate = new(); + private Dictionary _lightlessChanges = new(StringComparer.OrdinalIgnoreCase); + private Dictionary DrainPenumbraChanges() + { + lock (_penumbraGate) + { + var snapshot = _watcherChanges; + _watcherChanges = new(StringComparer.OrdinalIgnoreCase); + return snapshot; + } + } + + private Dictionary DrainLightlessChanges() + { + lock (_lightlessGate) + { + var snapshot = _lightlessChanges; + _lightlessChanges = new(StringComparer.OrdinalIgnoreCase); + return snapshot; + } + } public void StopMonitoring() { @@ -168,7 +197,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase if (!HasAllowedExtension(e.FullPath)) return; - lock (_watcherChanges) + lock (_lightlessChanges) { _lightlessChanges[e.FullPath] = new(e.ChangeType); } @@ -279,67 +308,58 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase private async Task LightlessWatcherExecution() { - _lightlessFswCts = _lightlessFswCts.CancelRecreate(); + _lightlessFswCts = _lightlessFswCts.CancelRecreate(); var token = _lightlessFswCts.Token; - var delay = TimeSpan.FromSeconds(5); - Dictionary changes; - lock (_lightlessChanges) - changes = _lightlessChanges.ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal); + try { - do - { - await Task.Delay(delay, token).ConfigureAwait(false); - } while (HaltScanLocks.Any(f => f.Value > 0)); - } - catch (TaskCanceledException) - { - return; + await Task.Delay(TimeSpan.FromSeconds(10), token).ConfigureAwait(false); + while (IsHalted() && !token.IsCancellationRequested) + await Task.Delay(250, token).ConfigureAwait(false); } + catch (TaskCanceledException) { return; } - lock (_lightlessChanges) - { - foreach (var key in changes.Keys) - { - _lightlessChanges.Remove(key); - } - } - - HandleChanges(changes); + var changes = DrainLightlessChanges(); + if (changes.Count > 0) + _ = HandleChangesAsync(changes, token); } - - private void HandleChanges(Dictionary changes) + private async Task HandleChangesAsync(Dictionary changes, CancellationToken token) { - lock (_fileDbManager) + await _dbGate.WaitAsync(token).ConfigureAwait(false); + try { - var deletedEntries = changes.Where(c => c.Value.ChangeType == WatcherChangeTypes.Deleted).Select(c => c.Key); - var renamedEntries = changes.Where(c => c.Value.ChangeType == WatcherChangeTypes.Renamed); - var remainingEntries = changes.Where(c => c.Value.ChangeType != WatcherChangeTypes.Deleted).Select(c => c.Key); + var deleted = changes.Where(c => c.Value.ChangeType == WatcherChangeTypes.Deleted).Select(c => c.Key); + var renamed = changes.Where(c => c.Value.ChangeType == WatcherChangeTypes.Renamed); + var remaining = changes.Where(c => c.Value.ChangeType != WatcherChangeTypes.Deleted).Select(c => c.Key); - foreach (var entry in deletedEntries) + foreach (var entry in deleted) { Logger.LogDebug("FSW Change: Deletion - {val}", entry); } - foreach (var entry in renamedEntries) + foreach (var entry in renamed) { Logger.LogDebug("FSW Change: Renamed - {oldVal} => {val}", entry.Value.OldPath, entry.Key); } - foreach (var entry in remainingEntries) + foreach (var entry in remaining) { Logger.LogDebug("FSW Change: Creation or Change - {val}", entry); } - var allChanges = deletedEntries - .Concat(renamedEntries.Select(c => c.Value.OldPath!)) - .Concat(renamedEntries.Select(c => c.Key)) - .Concat(remainingEntries) + var allChanges = deleted + .Concat(renamed.Select(c => c.Value.OldPath!)) + .Concat(renamed.Select(c => c.Key)) + .Concat(remaining) + .Distinct(StringComparer.OrdinalIgnoreCase) .ToArray(); _ = _fileDbManager.GetFileCachesByPaths(allChanges); - - _fileDbManager.WriteOutFullCsv(); + await _fileDbManager.WriteOutFullCsvAsync(token).ConfigureAwait(false); + } + finally + { + _dbGate.Release(); } } @@ -347,77 +367,97 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase { _penumbraFswCts = _penumbraFswCts.CancelRecreate(); var token = _penumbraFswCts.Token; - Dictionary changes; - lock (_watcherChanges) - changes = _watcherChanges.ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal); - var delay = TimeSpan.FromSeconds(10); + try { - do - { - await Task.Delay(delay, token).ConfigureAwait(false); - } while (HaltScanLocks.Any(f => f.Value > 0)); - } - catch (TaskCanceledException) - { - return; + await Task.Delay(TimeSpan.FromSeconds(10), token).ConfigureAwait(false); + while (IsHalted() && !token.IsCancellationRequested) + await Task.Delay(250, token).ConfigureAwait(false); } + catch (TaskCanceledException) { return; } - lock (_watcherChanges) - { - foreach (var key in changes.Keys) - { - _watcherChanges.Remove(key); - } - } - - HandleChanges(changes); + var changes = DrainPenumbraChanges(); + if (changes.Count > 0) + _ = HandleChangesAsync(changes, token); } public void InvokeScan() { TotalFiles = 0; - _currentFileProgress = 0; + TotalFilesStorage = 0; + Interlocked.Exchange(ref _currentFileProgress, 0); + _scanCancellationTokenSource = _scanCancellationTokenSource?.CancelRecreate() ?? new CancellationTokenSource(); var token = _scanCancellationTokenSource.Token; + _ = Task.Run(async () => { - Logger.LogDebug("Starting Full File Scan"); - TotalFiles = 0; - _currentFileProgress = 0; - while (_dalamudUtil.IsOnFrameworkThread) - { - Logger.LogWarning("Scanner is on framework, waiting for leaving thread before continuing"); - await Task.Delay(250, token).ConfigureAwait(false); - } + TaskCompletionSource scanTcs = new(TaskCreationOptions.RunContinuationsAsynchronously); - Thread scanThread = new(() => + try { - try + Logger.LogDebug("Starting Full File Scan"); + + while (IsHalted() && !token.IsCancellationRequested) { - _performanceCollector.LogPerformance(this, $"FullFileScan", () => FullFileScan(token)); + Logger.LogDebug("Scan is halted, waiting..."); + await Task.Delay(250, token).ConfigureAwait(false); } - catch (Exception ex) + + var scanThread = new Thread(() => { - Logger.LogError(ex, "Error during Full File Scan"); - } - }) - { - Priority = ThreadPriority.Lowest, - IsBackground = true - }; - scanThread.Start(); - while (scanThread.IsAlive) - { - await Task.Delay(250, token).ConfigureAwait(false); + try + { + token.ThrowIfCancellationRequested(); + + _performanceCollector.LogPerformance(this, $"FullFileScan", + () => FullFileScan(token)); + + scanTcs.TrySetResult(); + } + catch (OperationCanceledException) + { + scanTcs.TrySetCanceled(token); + } + catch (Exception ex) + { + Logger.LogError(ex, "Error during Full File Scan"); + scanTcs.TrySetException(ex); + } + }) + { + Priority = ThreadPriority.Lowest, + IsBackground = true, + Name = "LightlessSync.FullFileScan" + }; + + scanThread.Start(); + + using var _ = token.Register(() => scanTcs.TrySetCanceled(token)); + + await scanTcs.Task.ConfigureAwait(false); + } + catch (TaskCanceledException) + { + Logger.LogInformation("Full File Scan was canceled."); + } + catch (Exception ex) + { + Logger.LogError(ex, "Unexpected error in InvokeScan task"); + } + finally + { + TotalFiles = 0; + TotalFilesStorage = 0; + Interlocked.Exchange(ref _currentFileProgress, 0); } - TotalFiles = 0; - _currentFileProgress = 0; }, token); } public void RecalculateFileCacheSize(CancellationToken token) { + if (IsHalted()) return; + if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder)) { @@ -594,10 +634,20 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase public void ResumeScan(string source) { - if (!HaltScanLocks.ContainsKey(source)) HaltScanLocks[source] = 0; + int delta = 0; - HaltScanLocks[source]--; - if (HaltScanLocks[source] < 0) HaltScanLocks[source] = 0; + HaltScanLocks.AddOrUpdate(source, + addValueFactory: _ => 0, + updateValueFactory: (_, v) => + { + ArgumentException.ThrowIfNullOrEmpty(_); + if (v <= 0) return 0; + delta = 1; + return v - 1; + }); + + if (delta == 1) + Interlocked.Decrement(ref _haltCount); } protected override void Dispose(bool disposing) @@ -621,201 +671,81 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase private void FullFileScan(CancellationToken ct) { TotalFiles = 1; + _currentFileProgress = 0; + var penumbraDir = _ipcManager.Penumbra.ModDirectory; - bool penDirExists = true; - bool cacheDirExists = true; + var cacheFolder = _configService.Current.CacheFolder; + if (string.IsNullOrEmpty(penumbraDir) || !Directory.Exists(penumbraDir)) { - penDirExists = false; Logger.LogWarning("Penumbra directory is not set or does not exist."); + return; } - if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder)) + + if (string.IsNullOrEmpty(cacheFolder) || !Directory.Exists(cacheFolder)) { - cacheDirExists = false; Logger.LogWarning("Lightless Cache directory is not set or does not exist."); - } - if (!penDirExists || !cacheDirExists) - { return; } - var previousThreadPriority = Thread.CurrentThread.Priority; + var prevPriority = Thread.CurrentThread.Priority; Thread.CurrentThread.Priority = ThreadPriority.Lowest; - Logger.LogDebug("Getting files from {penumbra} and {storage}", penumbraDir, _configService.Current.CacheFolder); - Dictionary penumbraFiles = new(StringComparer.Ordinal); - foreach (var folder in Directory.EnumerateDirectories(penumbraDir!)) + try { - try + Logger.LogDebug("Getting files from {penumbra} and {storage}", penumbraDir, cacheFolder); + + var onDiskPaths = new HashSet(StringComparer.OrdinalIgnoreCase); + + static bool IsExcludedPenumbraPath(string path) + => path.Contains(@"\bg\", StringComparison.OrdinalIgnoreCase) + || path.Contains(@"\bgcommon\", StringComparison.OrdinalIgnoreCase) + || path.Contains(@"\ui\", StringComparison.OrdinalIgnoreCase); + + foreach (var folder in Directory.EnumerateDirectories(penumbraDir)) { - penumbraFiles[folder] = - [ - .. Directory.GetFiles(folder, "*.*", SearchOption.AllDirectories) - .AsParallel() - .Where(f => HasAllowedExtension(f) - && !f.Contains(@"\bg\", StringComparison.OrdinalIgnoreCase) - && !f.Contains(@"\bgcommon\", StringComparison.OrdinalIgnoreCase) - && !f.Contains(@"\ui\", StringComparison.OrdinalIgnoreCase)), - ]; - } - catch (Exception ex) - { - Logger.LogWarning(ex, "Could not enumerate path {path}", folder); - } - Thread.Sleep(50); - if (ct.IsCancellationRequested) return; - } + ct.ThrowIfCancellationRequested(); - var allCacheFiles = Directory.GetFiles(_configService.Current.CacheFolder, "*.*", SearchOption.TopDirectoryOnly) - .AsParallel() - .Where(f => - { - var val = f.Split('\\')[^1]; - return val.Length == 40 || (val.Split('.').FirstOrDefault()?.Length ?? 0) == 40; - }); - - if (ct.IsCancellationRequested) return; - - var allScannedFiles = (penumbraFiles.SelectMany(k => k.Value)) - .Concat(allCacheFiles) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToDictionary(t => t.ToLowerInvariant(), t => false, StringComparer.OrdinalIgnoreCase); - - TotalFiles = allScannedFiles.Count; - Thread.CurrentThread.Priority = previousThreadPriority; - - Thread.Sleep(TimeSpan.FromSeconds(2)); - - if (ct.IsCancellationRequested) return; - - // scan files from database - var threadCount = Math.Clamp((int)(Environment.ProcessorCount / 2.0f), 2, 8); - - List entitiesToRemove = []; - List entitiesToUpdate = []; - Lock sync = new(); - Thread[] workerThreads = new Thread[threadCount]; - - ConcurrentQueue fileCaches = new(_fileDbManager.GetAllFileCaches()); - - TotalFilesStorage = fileCaches.Count; - - for (int i = 0; i < threadCount; i++) - { - Logger.LogTrace("Creating Thread {i}", i); - workerThreads[i] = new((tcounter) => - { - var threadNr = (int)tcounter!; - Logger.LogTrace("Spawning Worker Thread {i}", threadNr); - while (!ct.IsCancellationRequested && fileCaches.TryDequeue(out var workload)) + try { - try + foreach (var file in Directory.EnumerateFiles(folder, "*.*", SearchOption.AllDirectories)) { - if (ct.IsCancellationRequested) return; + ct.ThrowIfCancellationRequested(); - if (!_ipcManager.Penumbra.APIAvailable) - { - Logger.LogWarning("Penumbra not available"); - return; - } + if (!HasAllowedExtension(file)) continue; + if (IsExcludedPenumbraPath(file)) continue; - var validatedCacheResult = _fileDbManager.ValidateFileCacheEntity(workload); - if (validatedCacheResult.State != FileState.RequireDeletion) - { - lock (sync) { allScannedFiles[validatedCacheResult.FileCache.ResolvedFilepath] = true; } - } - if (validatedCacheResult.State == FileState.RequireUpdate) - { - Logger.LogTrace("To update: {path}", validatedCacheResult.FileCache.ResolvedFilepath); - lock (sync) { entitiesToUpdate.Add(validatedCacheResult.FileCache); } - } - else if (validatedCacheResult.State == FileState.RequireDeletion) - { - Logger.LogTrace("To delete: {path}", validatedCacheResult.FileCache.ResolvedFilepath); - lock (sync) { entitiesToRemove.Add(validatedCacheResult.FileCache); } - } + onDiskPaths.Add(file); } - catch (Exception ex) - { - if (workload != null) - { - Logger.LogWarning(ex, "Failed validating {path}", workload.ResolvedFilepath); - } - else - { - Logger.LogWarning(ex, "Failed validating unknown workload"); - } - } - Interlocked.Increment(ref _currentFileProgress); } - - Logger.LogTrace("Ending Worker Thread {i}", threadNr); - }) - { - Priority = ThreadPriority.Lowest, - IsBackground = true - }; - workerThreads[i].Start(i); - } - - while (!ct.IsCancellationRequested && workerThreads.Any(u => u.IsAlive)) - { - Thread.Sleep(1000); - } - - if (ct.IsCancellationRequested) return; - - Logger.LogTrace("Threads exited"); - - if (!_ipcManager.Penumbra.APIAvailable) - { - Logger.LogWarning("Penumbra not available"); - return; - } - - if (entitiesToUpdate.Count != 0 || entitiesToRemove.Count != 0) - { - foreach (var entity in entitiesToUpdate) - { - _fileDbManager.UpdateHashedFile(entity); + catch (Exception ex) + { + Logger.LogWarning(ex, "Could not enumerate path {path}", folder); + } } - foreach (var entity in entitiesToRemove) + foreach (var file in Directory.EnumerateFiles(cacheFolder, "*.*", SearchOption.TopDirectoryOnly)) { - _fileDbManager.RemoveHashedFile(entity.Hash, entity.PrefixedFilePath); + ct.ThrowIfCancellationRequested(); + + var name = Path.GetFileName(file); + var stem = Path.GetFileNameWithoutExtension(file); + + if (name.Length == 40 || stem.Length == 40) + onDiskPaths.Add(file); } - _fileDbManager.WriteOutFullCsv(); - } + var threadCount = Math.Clamp((int)(Environment.ProcessorCount / 2.0f), 2, 8); - Logger.LogTrace("Scanner validated existing db files"); + var fileCacheList = _fileDbManager.GetAllFileCaches(); + var fileCaches = new ConcurrentQueue(fileCacheList); - if (!_ipcManager.Penumbra.APIAvailable) - { - Logger.LogWarning("Penumbra not available"); - return; - } + TotalFilesStorage = fileCaches.Count; + TotalFiles = onDiskPaths.Count + TotalFilesStorage; - if (ct.IsCancellationRequested) return; - - var newFiles = allScannedFiles.Where(c => !c.Value).Select(c => c.Key).ToList(); - foreach (var cachePath in newFiles) - { - if (ct.IsCancellationRequested) break; - ProcessOne(cachePath); - Interlocked.Increment(ref _currentFileProgress); - } - - Logger.LogTrace("Scanner added {count} new files to db", newFiles.Count); - - void ProcessOne(string? cachePath) - { - if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null) - { - Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}", - _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null); - return; - } + var validOrPresentInDb = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + var entitiesToUpdate = new ConcurrentBag(); + var entitiesToRemove = new ConcurrentBag(); if (!_ipcManager.Penumbra.APIAvailable) { @@ -823,33 +753,161 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase return; } - try + Thread[] workerThreads = new Thread[threadCount]; + for (int i = 0; i < threadCount; i++) { - var entry = _fileDbManager.CreateFileEntry(cachePath); - if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath); + workerThreads[i] = new Thread(tcounter => + { + var threadNr = (int)tcounter!; + Logger.LogTrace("Spawning Worker Thread {i}", threadNr); + + while (!ct.IsCancellationRequested && fileCaches.TryDequeue(out var workload)) + { + try + { + if (ct.IsCancellationRequested) break; + + if (!_ipcManager.Penumbra.APIAvailable) + break; + + var validated = _fileDbManager.ValidateFileCacheEntity(workload); + + if (validated.State != FileState.RequireDeletion) + { + validOrPresentInDb.TryAdd(validated.FileCache.ResolvedFilepath, 0); + } + + if (validated.State == FileState.RequireUpdate) + { + Logger.LogTrace("To update: {path}", validated.FileCache.ResolvedFilepath); + entitiesToUpdate.Add(validated.FileCache); + } + else if (validated.State == FileState.RequireDeletion) + { + Logger.LogTrace("To delete: {path}", validated.FileCache.ResolvedFilepath); + entitiesToRemove.Add(validated.FileCache); + } + } + catch (Exception ex) + { + if (workload != null) + Logger.LogWarning(ex, "Failed validating {path}", workload.ResolvedFilepath); + else + Logger.LogWarning(ex, "Failed validating unknown workload"); + } + finally + { + Interlocked.Increment(ref _currentFileProgress); + } + } + + Logger.LogTrace("Ending Worker Thread {i}", threadNr); + }) + { + Priority = ThreadPriority.Lowest, + IsBackground = true + }; + + workerThreads[i].Start(i); } - catch (IOException ioex) + + while (!ct.IsCancellationRequested && workerThreads.Any(t => t.IsAlive)) { - Logger.LogDebug(ioex, "File busy or locked: {file}", cachePath); + ct.WaitHandle.WaitOne(250); } - catch (Exception ex) + + if (ct.IsCancellationRequested) return; + + Logger.LogTrace("Scanner validated existing db files"); + + if (!_ipcManager.Penumbra.APIAvailable) { - Logger.LogWarning(ex, "Failed adding {file}", cachePath); + Logger.LogWarning("Penumbra not available"); + return; + } + + var didMutateDb = false; + + foreach (var entity in entitiesToUpdate) + { + didMutateDb = true; + _fileDbManager.UpdateHashedFile(entity); + } + + foreach (var entity in entitiesToRemove) + { + didMutateDb = true; + _fileDbManager.RemoveHashedFile(entity.Hash, entity.PrefixedFilePath); + } + + if (didMutateDb) + _fileDbManager.WriteOutFullCsv(); + + if (ct.IsCancellationRequested) return; + + var newFiles = onDiskPaths.Where(p => !validOrPresentInDb.ContainsKey(p)).ToList(); + + foreach (var path in newFiles) + { + if (ct.IsCancellationRequested) break; + ProcessOne(path); + Interlocked.Increment(ref _currentFileProgress); + } + + Logger.LogTrace("Scanner added {count} new files to db", newFiles.Count); + + void ProcessOne(string? filePath) + { + if (filePath == null) + return; + + if (!_ipcManager.Penumbra.APIAvailable) + { + Logger.LogWarning("Penumbra not available"); + return; + } + + try + { + var entry = _fileDbManager.CreateFileEntry(filePath); + if (entry == null) + _ = _fileDbManager.CreateCacheEntry(filePath); + } + catch (IOException ioex) + { + Logger.LogDebug(ioex, "File busy or locked: {file}", filePath); + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Failed adding {file}", filePath); + } + } + + Logger.LogDebug("Scan complete"); + + TotalFiles = 0; + _currentFileProgress = 0; + + if (!_configService.Current.InitialScanComplete) + { + _configService.Current.InitialScanComplete = true; + _configService.Save(); + + StartLightlessWatcher(cacheFolder); + StartPenumbraWatcher(penumbraDir); } } - - Logger.LogDebug("Scan complete"); - TotalFiles = 0; - _currentFileProgress = 0; - entitiesToRemove.Clear(); - allScannedFiles.Clear(); - - if (!_configService.Current.InitialScanComplete) + catch (OperationCanceledException) { - _configService.Current.InitialScanComplete = true; - _configService.Save(); - StartLightlessWatcher(_configService.Current.CacheFolder); - StartPenumbraWatcher(penumbraDir); + // normal cancellation + } + catch (Exception ex) + { + Logger.LogError(ex, "Error during Full File Scan"); + } + finally + { + Thread.CurrentThread.Priority = prevPriority; } } } \ No newline at end of file diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 771f558..31bbd4d 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -213,7 +213,7 @@ public sealed partial class FileCompactor : IDisposable /// Bytes that have to be written /// Cancellation Token for interupts /// Writing Task - public async Task WriteAllBytesAsync(string filePath, byte[] bytes, CancellationToken token) + public async Task WriteAllBytesAsync(string filePath, byte[] bytes, CancellationToken token, bool enqueueCompaction = true) { var dir = Path.GetDirectoryName(filePath); if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) @@ -221,6 +221,12 @@ public sealed partial class FileCompactor : IDisposable await File.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false); + if (enqueueCompaction && _lightlessConfigService.Current.UseCompactor) + EnqueueCompaction(filePath); + } + + public void RequestCompaction(string filePath) + { if (_lightlessConfigService.Current.UseCompactor) EnqueueCompaction(filePath); } diff --git a/LightlessSync/FileCache/TransientResourceManager.cs b/LightlessSync/FileCache/TransientResourceManager.cs index 11073dc..59b9c4d 100644 --- a/LightlessSync/FileCache/TransientResourceManager.cs +++ b/LightlessSync/FileCache/TransientResourceManager.cs @@ -321,7 +321,7 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase { foreach (var handler in _playerRelatedPointers) { - var address = (nint)handler.Address; + var address = handler.Address; if (address != nint.Zero) { tempMap[address] = handler; diff --git a/LightlessSync/Interop/Ipc/IpcCallerPenumbra.cs b/LightlessSync/Interop/Ipc/IpcCallerPenumbra.cs index e077eab..14d439b 100644 --- a/LightlessSync/Interop/Ipc/IpcCallerPenumbra.cs +++ b/LightlessSync/Interop/Ipc/IpcCallerPenumbra.cs @@ -81,7 +81,10 @@ public sealed class IpcCallerPenumbra : IpcServiceBase => _collections.RemoveTemporaryCollectionAsync(logger, applicationId, collectionId); public Task SetTemporaryModsAsync(ILogger logger, Guid applicationId, Guid collectionId, Dictionary modPaths) - => _collections.SetTemporaryModsAsync(logger, applicationId, collectionId, modPaths); + => _collections.SetTemporaryModsAsync(logger, applicationId, collectionId, modPaths, "Player"); + + public Task SetTemporaryModsAsync(ILogger logger, Guid applicationId, Guid collectionId, Dictionary modPaths, string scope) + => _collections.SetTemporaryModsAsync(logger, applicationId, collectionId, modPaths, scope); public Task SetManipulationDataAsync(ILogger logger, Guid applicationId, Guid collectionId, string manipulationData) => _collections.SetManipulationDataAsync(logger, applicationId, collectionId, manipulationData); diff --git a/LightlessSync/Interop/Ipc/Penumbra/PenumbraCollections.cs b/LightlessSync/Interop/Ipc/Penumbra/PenumbraCollections.cs index c095471..b91162a 100644 --- a/LightlessSync/Interop/Ipc/Penumbra/PenumbraCollections.cs +++ b/LightlessSync/Interop/Ipc/Penumbra/PenumbraCollections.cs @@ -92,25 +92,43 @@ public sealed class PenumbraCollections : PenumbraBase _activeTemporaryCollections.TryRemove(collectionId, out _); } - public async Task SetTemporaryModsAsync(ILogger logger, Guid applicationId, Guid collectionId, Dictionary modPaths) + public async Task SetTemporaryModsAsync( + ILogger logger, + Guid applicationId, + Guid collectionId, + Dictionary modPaths, + string scope) { if (!IsAvailable || collectionId == Guid.Empty) - { return; + + var modName = $"LightlessChara_Files_{applicationId:N}_{scope}"; + + var normalized = new Dictionary(modPaths.Count, StringComparer.OrdinalIgnoreCase); + foreach (var kvp in modPaths) + { + if (string.IsNullOrWhiteSpace(kvp.Key) || string.IsNullOrWhiteSpace(kvp.Value)) + continue; + + var gamePath = kvp.Key.Replace('\\', '/').ToLowerInvariant(); + normalized[gamePath] = kvp.Value; } await DalamudUtil.RunOnFrameworkThread(() => { - foreach (var mod in modPaths) - { - logger.LogTrace("[{ApplicationId}] Change: {From} => {To}", applicationId, mod.Key, mod.Value); - } + foreach (var mod in normalized) + logger.LogTrace("[{ApplicationId}] {ModName}: {From} => {To}", applicationId, modName, mod.Key, mod.Value); - var removeResult = _removeTemporaryMod.Invoke("LightlessChara_Files", collectionId, 0); - logger.LogTrace("[{ApplicationId}] Removing temp files mod for {CollectionId}, Success: {Result}", applicationId, collectionId, removeResult); + var removeResult = _removeTemporaryMod.Invoke(modName, collectionId, 0); + logger.LogTrace("[{ApplicationId}] Removing temp mod {ModName} for {CollectionId}, Success: {Result}", + applicationId, modName, collectionId, removeResult); - var addResult = _addTemporaryMod.Invoke("LightlessChara_Files", collectionId, modPaths, string.Empty, 0); - logger.LogTrace("[{ApplicationId}] Setting temp files mod for {CollectionId}, Success: {Result}", applicationId, collectionId, addResult); + if (normalized.Count == 0) + return; + + var addResult = _addTemporaryMod.Invoke(modName, collectionId, normalized, string.Empty, 0); + logger.LogTrace("[{ApplicationId}] Setting temp mod {ModName} for {CollectionId}, Success: {Result}", + applicationId, modName, collectionId, addResult); }).ConfigureAwait(false); } @@ -171,7 +189,7 @@ public sealed class PenumbraCollections : PenumbraBase Logger.LogDebug("Cleaning up stale temporary collection {CollectionName} ({CollectionId})", name, collectionId); var deleteResult = await DalamudUtil.RunOnFrameworkThread(() => { - var result = (PenumbraApiEc)_removeTemporaryCollection.Invoke(collectionId); + var result = _removeTemporaryCollection.Invoke(collectionId); Logger.LogTrace("Cleanup RemoveTemporaryCollection result for {CollectionName} ({CollectionId}): {Result}", name, collectionId, result); return result; }).ConfigureAwait(false); diff --git a/LightlessSync/LightlessConfiguration/Configurations/LightlessConfig.cs b/LightlessSync/LightlessConfiguration/Configurations/LightlessConfig.cs index 9e92b63..22e3c94 100644 --- a/LightlessSync/LightlessConfiguration/Configurations/LightlessConfig.cs +++ b/LightlessSync/LightlessConfiguration/Configurations/LightlessConfig.cs @@ -158,7 +158,6 @@ public class LightlessConfig : ILightlessConfiguration public string? SelectedFinderSyncshell { get; set; } = null; public string LastSeenVersion { get; set; } = string.Empty; public bool EnableParticleEffects { get; set; } = true; - public HashSet OrphanableTempCollections { get; set; } = []; public AnimationValidationMode AnimationValidationMode { get; set; } = AnimationValidationMode.Unsafe; public bool AnimationAllowOneBasedShift { get; set; } = false; diff --git a/LightlessSync/LightlessConfiguration/Configurations/PenumbraJanitorConfig.cs b/LightlessSync/LightlessConfiguration/Configurations/PenumbraJanitorConfig.cs new file mode 100644 index 0000000..135f62d --- /dev/null +++ b/LightlessSync/LightlessConfiguration/Configurations/PenumbraJanitorConfig.cs @@ -0,0 +1,8 @@ +namespace LightlessSync.LightlessConfiguration.Configurations; + +public class PenumbraJanitorConfig : ILightlessConfiguration +{ + public int Version { get; set; } = 0; + + public HashSet OrphanableTempCollections { get; set; } = []; +} diff --git a/LightlessSync/LightlessConfiguration/PenumbraJanitorConfigService.cs b/LightlessSync/LightlessConfiguration/PenumbraJanitorConfigService.cs new file mode 100644 index 0000000..e7bd647 --- /dev/null +++ b/LightlessSync/LightlessConfiguration/PenumbraJanitorConfigService.cs @@ -0,0 +1,14 @@ +using LightlessSync.LightlessConfiguration.Configurations; + +namespace LightlessSync.LightlessConfiguration; + +public class PenumbraJanitorConfigService : ConfigurationServiceBase +{ + public const string ConfigName = "penumbra-collections.json"; + + public PenumbraJanitorConfigService(string configDir) : base(configDir) + { + } + + public override string ConfigurationName => ConfigName; +} \ No newline at end of file diff --git a/LightlessSync/PlayerData/Factories/PairFactory.cs b/LightlessSync/PlayerData/Factories/PairFactory.cs index a7ffd6e..f9c2fb5 100644 --- a/LightlessSync/PlayerData/Factories/PairFactory.cs +++ b/LightlessSync/PlayerData/Factories/PairFactory.cs @@ -1,4 +1,4 @@ -using LightlessSync.API.Data.Enum; + using LightlessSync.API.Data.Enum; using LightlessSync.API.Dto.User; using LightlessSync.PlayerData.Pairs; using LightlessSync.Services.Mediator; diff --git a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs index e8f3459..6e32a91 100644 --- a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs +++ b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs @@ -1,6 +1,6 @@ using Dalamud.Utility; using FFXIVClientStructs.FFXIV.Client.Game.Character; -using LightlessSync.API.Data.Enum; +using FFXIVClientStructs.FFXIV.Client.Game.Object; using LightlessSync.FileCache; using LightlessSync.Interop.Ipc; using LightlessSync.LightlessConfiguration; @@ -9,11 +9,12 @@ using LightlessSync.PlayerData.Data; using LightlessSync.PlayerData.Handlers; using LightlessSync.Services; using LightlessSync.Services.Mediator; +using LightlessSync.Utils; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Diagnostics; -using System.Runtime.ExceptionServices; using System.Runtime.InteropServices; +using ObjectKind = LightlessSync.API.Data.Enum.ObjectKind; namespace LightlessSync.PlayerData.Factories; @@ -119,45 +120,28 @@ public class PlayerDataFactory return null; } + private static readonly int _characterGameObjectOffset = + (int)Marshal.OffsetOf(nameof(Character.GameObject)); + + private static readonly int _gameObjectDrawObjectOffset = + (int)Marshal.OffsetOf(nameof(GameObject.DrawObject)); + private async Task CheckForNullDrawObject(IntPtr playerPointer) - => await _dalamudUtil.RunOnFrameworkThread(() => CheckForNullDrawObjectUnsafe(playerPointer)).ConfigureAwait(false); + => await _dalamudUtil.RunOnFrameworkThread(() => CheckForNullDrawObjectSafe(playerPointer)) + .ConfigureAwait(false); - private unsafe static bool CheckForNullDrawObjectUnsafe(IntPtr playerPointer) + private static bool CheckForNullDrawObjectSafe(nint playerPointer) { - if (playerPointer == IntPtr.Zero) + if (playerPointer == nint.Zero) return true; - if (!IsPointerValid(playerPointer)) + var drawObjPtrAddress = playerPointer + _characterGameObjectOffset + _gameObjectDrawObjectOffset; + + // Read the DrawObject pointer from memory + if (!MemoryProcessProbe.TryReadIntPtr(drawObjPtrAddress, out var drawObj)) return true; - var character = (Character*)playerPointer; - if (character == null) - return true; - - var gameObject = &character->GameObject; - if (gameObject == null) - return true; - - if (!IsPointerValid((IntPtr)gameObject)) - return true; - - return gameObject->DrawObject == null; - } - - private static bool IsPointerValid(IntPtr ptr) - { - if (ptr == IntPtr.Zero) - return false; - - try - { - _ = Marshal.ReadByte(ptr); - return true; - } - catch - { - return false; - } + return drawObj == nint.Zero; } private static bool IsCacheFresh(CacheEntry entry) @@ -173,7 +157,7 @@ public class PlayerDataFactory if (_characterBuildCache.TryGetValue(key, out var cached) && IsCacheFresh(cached) && !_characterBuildInflight.ContainsKey(key)) return cached.Fragment; - var buildTask = _characterBuildInflight.GetOrAdd(key, _ => BuildAndCacheAsync(obj, key)); + var buildTask = _characterBuildInflight.GetOrAdd(key, valueFactory: k => BuildAndCacheAsync(obj, k)); if (_characterBuildCache.TryGetValue(key, out cached)) { diff --git a/LightlessSync/PlayerData/Handlers/OwnedObjectHandler.cs b/LightlessSync/PlayerData/Handlers/OwnedObjectHandler.cs new file mode 100644 index 0000000..f335057 --- /dev/null +++ b/LightlessSync/PlayerData/Handlers/OwnedObjectHandler.cs @@ -0,0 +1,238 @@ +using Microsoft.Extensions.Logging; +using LightlessSync.API.Data; +using LightlessSync.API.Data.Enum; +using LightlessSync.PlayerData.Factories; +using LightlessSync.Services; +using LightlessSync.Services.ActorTracking; +using LightlessSync.Interop.Ipc; +using LightlessSync.PlayerData.Pairs; + +namespace LightlessSync.PlayerData.Handlers; + +internal sealed class OwnedObjectHandler +{ + private readonly ILogger _logger; + private readonly DalamudUtilService _dalamudUtil; + private readonly GameObjectHandlerFactory _handlerFactory; + private readonly IpcManager _ipc; + private readonly ActorObjectService _actorObjectService; + + private const int _fullyLoadedTimeoutMsPlayer = 30000; + private const int _fullyLoadedTimeoutMsOther = 5000; + + public OwnedObjectHandler( + ILogger logger, + DalamudUtilService dalamudUtil, + GameObjectHandlerFactory handlerFactory, + IpcManager ipc, + ActorObjectService actorObjectService) + { + _logger = logger; + _dalamudUtil = dalamudUtil; + _handlerFactory = handlerFactory; + _ipc = ipc; + _actorObjectService = actorObjectService; + } + + public async Task ApplyAsync( + Guid applicationId, + ObjectKind kind, + HashSet changes, + CharacterData data, + GameObjectHandler playerHandler, + Guid penumbraCollection, + Dictionary customizeIds, + CancellationToken token) + { + if (playerHandler.Address == nint.Zero) + return false; + + var handler = await CreateHandlerAsync(kind, playerHandler, token).ConfigureAwait(false); + if (handler is null || handler.Address == nint.Zero) + return false; + + try + { + token.ThrowIfCancellationRequested(); + + bool hasFileReplacements = + kind != ObjectKind.Player + && data.FileReplacements.TryGetValue(kind, out var repls) + && repls is { Count: > 0 }; + + bool shouldAssignCollection = + kind != ObjectKind.Player + && hasFileReplacements + && penumbraCollection != Guid.Empty + && _ipc.Penumbra.APIAvailable; + + bool isPlayerIpcOnly = + kind == ObjectKind.Player + && changes.Count > 0 + && changes.All(c => c is PlayerChanges.Honorific + or PlayerChanges.Moodles + or PlayerChanges.PetNames + or PlayerChanges.Heels); + + await handler.IsBeingDrawnRunOnFrameworkAsync().ConfigureAwait(false); + + var drawTimeoutMs = handler.ObjectKind == ObjectKind.Player ? 30000 : 5000; + var fullyLoadedTimeoutMs = handler.ObjectKind == ObjectKind.Player ? _fullyLoadedTimeoutMsPlayer : _fullyLoadedTimeoutMsOther; + + await _dalamudUtil + .WaitWhileCharacterIsDrawing(_logger, handler, applicationId, drawTimeoutMs, token) + .ConfigureAwait(false); + + if (handler.Address != nint.Zero) + { + var loaded = await _actorObjectService + .WaitForFullyLoadedAsync(handler.Address, token, fullyLoadedTimeoutMs) + .ConfigureAwait(false); + + if (!loaded) + { + _logger.LogTrace("[{appId}] {kind}: not fully loaded in time, skipping for now", applicationId, kind); + return false; + } + } + + token.ThrowIfCancellationRequested(); + + if (shouldAssignCollection) + { + var objIndex = await _dalamudUtil + .RunOnFrameworkThread(() => handler.GetGameObject()?.ObjectIndex) + .ConfigureAwait(false); + + if (!objIndex.HasValue) + { + _logger.LogTrace("[{appId}] {kind}: ObjectIndex not available yet, cannot assign collection", applicationId, kind); + return false; + } + + await _ipc.Penumbra + .AssignTemporaryCollectionAsync(_logger, penumbraCollection, objIndex.Value) + .ConfigureAwait(false); + } + + var tasks = new List(); + + foreach (var change in changes.OrderBy(c => (int)c)) + { + token.ThrowIfCancellationRequested(); + + switch (change) + { + case PlayerChanges.Customize: + if (data.CustomizePlusData.TryGetValue(kind, out var customizeData) && !string.IsNullOrEmpty(customizeData)) + tasks.Add(ApplyCustomizeAsync(handler.Address, customizeData, kind, customizeIds)); + else if (customizeIds.TryGetValue(kind, out var existingId)) + tasks.Add(RevertCustomizeAsync(existingId, kind, customizeIds)); + break; + + case PlayerChanges.Glamourer: + if (data.GlamourerData.TryGetValue(kind, out var glamourerData) && !string.IsNullOrEmpty(glamourerData)) + tasks.Add(_ipc.Glamourer.ApplyAllAsync(_logger, handler, glamourerData, applicationId, token)); + break; + + case PlayerChanges.Heels: + if (kind == ObjectKind.Player && !string.IsNullOrEmpty(data.HeelsData)) + tasks.Add(_ipc.Heels.SetOffsetForPlayerAsync(handler.Address, data.HeelsData)); + break; + + case PlayerChanges.Honorific: + if (kind == ObjectKind.Player && !string.IsNullOrEmpty(data.HonorificData)) + tasks.Add(_ipc.Honorific.SetTitleAsync(handler.Address, data.HonorificData)); + break; + + case PlayerChanges.Moodles: + if (kind == ObjectKind.Player && !string.IsNullOrEmpty(data.MoodlesData)) + tasks.Add(_ipc.Moodles.SetStatusAsync(handler.Address, data.MoodlesData)); + break; + + case PlayerChanges.PetNames: + if (kind == ObjectKind.Player && !string.IsNullOrEmpty(data.PetNamesData)) + tasks.Add(_ipc.PetNames.SetPlayerData(handler.Address, data.PetNamesData)); + break; + + case PlayerChanges.ModFiles: + case PlayerChanges.ModManip: + case PlayerChanges.ForcedRedraw: + default: + break; + } + } + + if (tasks.Count > 0) + await Task.WhenAll(tasks).ConfigureAwait(false); + + token.ThrowIfCancellationRequested(); + + bool needsRedraw = + _ipc.Penumbra.APIAvailable + && ( + shouldAssignCollection + || changes.Contains(PlayerChanges.ForcedRedraw) + || changes.Contains(PlayerChanges.ModFiles) + || changes.Contains(PlayerChanges.ModManip) + || changes.Contains(PlayerChanges.Glamourer) + || changes.Contains(PlayerChanges.Customize) + ); + + if (isPlayerIpcOnly) + needsRedraw = false; + + if (needsRedraw && _ipc.Penumbra.APIAvailable) + { + _logger.LogWarning( + "[{appId}] {kind}: Redrawing ownedTarget={isOwned} (needsRedraw={needsRedraw})", + applicationId, kind, kind != ObjectKind.Player, needsRedraw); + + await _ipc.Penumbra + .RedrawAsync(_logger, handler, applicationId, token) + .ConfigureAwait(false); + } + + return true; + } + finally + { + if (!ReferenceEquals(handler, playerHandler)) + handler.Dispose(); + } + } + + private async Task CreateHandlerAsync(ObjectKind kind, GameObjectHandler playerHandler, CancellationToken token) + { + if (kind == ObjectKind.Player) + return playerHandler; + + var playerPtr = playerHandler.Address; + nint ownedPtr = kind switch + { + ObjectKind.Companion => await _dalamudUtil.GetCompanionAsync(playerPtr).ConfigureAwait(false), + ObjectKind.MinionOrMount => await _dalamudUtil.GetMinionOrMountAsync(playerPtr).ConfigureAwait(false), + ObjectKind.Pet => await _dalamudUtil.GetPetAsync(playerPtr).ConfigureAwait(false), + _ => nint.Zero + }; + + if (ownedPtr == nint.Zero) + return null; + + return await _handlerFactory.Create(kind, () => ownedPtr, isWatched: false).ConfigureAwait(false); + } + + private async Task ApplyCustomizeAsync(nint address, string customizeData, ObjectKind kind, Dictionary customizeIds) + { + customizeIds[kind] = await _ipc.CustomizePlus.SetBodyScaleAsync(address, customizeData).ConfigureAwait(false); + } + + private async Task RevertCustomizeAsync(Guid? customizeId, ObjectKind kind, Dictionary customizeIds) + { + if (!customizeId.HasValue) + return; + + await _ipc.CustomizePlus.RevertByIdAsync(customizeId.Value).ConfigureAwait(false); + customizeIds.Remove(kind); + } +} \ No newline at end of file diff --git a/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs b/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs index eded176..b547396 100644 --- a/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs +++ b/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs @@ -1,4 +1,4 @@ -using System.Collections.Concurrent; +using System.Collections.Concurrent; using System.Diagnostics; using Dalamud.Plugin.Services; using LightlessSync.API.Data; @@ -19,7 +19,6 @@ using LightlessSync.Services.ServerConfiguration; using LightlessSync.Services.TextureCompression; using LightlessSync.Utils; using LightlessSync.WebAPI.Files; -using LightlessSync.WebAPI.Files.Models; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using DalamudObjectKind = Dalamud.Game.ClientState.Objects.Enums.ObjectKind; @@ -55,6 +54,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa private readonly PenumbraTempCollectionJanitor _tempCollectionJanitor; private readonly LightlessConfigService _configService; private readonly PairManager _pairManager; + private readonly OwnedObjectHandler _ownedObjectHandler; private readonly IFramework _framework; private CancellationTokenSource? _applicationCancellationTokenSource; private Guid _applicationId; @@ -77,6 +77,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa private string? _lastSuccessfulDataHash; private bool _isVisible; private Guid _penumbraCollection; + private Guid _penumbraOwnedCollection; private readonly object _collectionGate = new(); private bool _redrawOnNextApplication = false; private readonly object _initializationGate = new(); @@ -95,7 +96,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa private readonly Dictionary> _pendingOwnedChanges = new(); private CancellationTokenSource? _ownedRetryCts; private Task _ownedRetryTask = Task.CompletedTask; - + private string OwnedCollectionCacheKey => $"{Ident}:owned"; private static readonly TimeSpan OwnedRetryInitialDelay = TimeSpan.FromSeconds(1); @@ -246,6 +247,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa _tempCollectionJanitor = tempCollectionJanitor; _modelAnalyzer = modelAnalyzer; _configService = configService; + _ownedObjectHandler = new OwnedObjectHandler(Logger, _dalamudUtil, _gameObjectHandlerFactory, _ipcManager, _actorObjectService); } public void Initialize() @@ -475,67 +477,110 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa } } + private Guid EnsureOwnedPenumbraCollection() + { + if (!IsVisible) + return Guid.Empty; + + if (_penumbraOwnedCollection != Guid.Empty) + return _penumbraOwnedCollection; + + lock (_collectionGate) + { + if (_penumbraOwnedCollection != Guid.Empty) + return _penumbraOwnedCollection; + + var cached = _pairStateCache.TryGetTemporaryCollection(OwnedCollectionCacheKey); + if (cached.HasValue && cached.Value != Guid.Empty) + { + _penumbraOwnedCollection = cached.Value; + return _penumbraOwnedCollection; + } + + if (!_ipcManager.Penumbra.APIAvailable) + return Guid.Empty; + + var user = GetPrimaryUserDataSafe(); + var uid = !string.IsNullOrEmpty(user.UID) ? user.UID : Ident; + + var created = _ipcManager.Penumbra + .CreateTemporaryCollectionAsync(Logger, uid + "_Owned") + .ConfigureAwait(false).GetAwaiter().GetResult(); + + if (created != Guid.Empty) + { + _penumbraOwnedCollection = created; + _pairStateCache.StoreTemporaryCollection(OwnedCollectionCacheKey, created); + _tempCollectionJanitor.Register(created); + } + + return _penumbraOwnedCollection; + } + } + private void ResetPenumbraCollection(bool releaseFromPenumbra = true, string? reason = null, bool awaitIpc = true) { - Guid toRelease = Guid.Empty; - bool hadCollection = false; + Guid toReleasePlayer = Guid.Empty; + Guid toReleaseOwned = Guid.Empty; + lock (_collectionGate) { if (_penumbraCollection != Guid.Empty) { - toRelease = _penumbraCollection; + toReleasePlayer = _penumbraCollection; _penumbraCollection = Guid.Empty; - hadCollection = true; + } + + if (_penumbraOwnedCollection != Guid.Empty) + { + toReleaseOwned = _penumbraOwnedCollection; + _penumbraOwnedCollection = Guid.Empty; } } - var cached = _pairStateCache.ClearTemporaryCollection(Ident); - if (cached.HasValue && cached.Value != Guid.Empty) - { - toRelease = cached.Value; - hadCollection = true; - } + var cachedPlayer = _pairStateCache.ClearTemporaryCollection(Ident); + if (cachedPlayer is { } cp && cp != Guid.Empty) + toReleasePlayer = cp; - if (hadCollection) - { - _needsCollectionRebuild = true; - _forceFullReapply = true; - _forceApplyMods = true; - _tempCollectionJanitor.Unregister(toRelease); - } + var cachedOwned = _pairStateCache.ClearTemporaryCollection(OwnedCollectionCacheKey); + if (cachedOwned is { } co && co != Guid.Empty) + toReleaseOwned = co; - if (!releaseFromPenumbra || toRelease == Guid.Empty || !_ipcManager.Penumbra.APIAvailable) - { + if (toReleasePlayer != Guid.Empty) + _tempCollectionJanitor.Unregister(toReleasePlayer); + if (toReleaseOwned != Guid.Empty) + _tempCollectionJanitor.Unregister(toReleaseOwned); + + if (!releaseFromPenumbra || !_ipcManager.Penumbra.APIAvailable) return; + + async Task RemoveAsync(Guid id) + { + if (id == Guid.Empty) return; + try + { + var appId = Guid.NewGuid(); + Logger.LogTrace("[{applicationId}] Removing temp collection {CollectionId} for {handler} ({reason})", + appId, id, GetLogIdentifier(), reason ?? "Cleanup"); + + await _ipcManager.Penumbra.RemoveTemporaryCollectionAsync(Logger, appId, id).ConfigureAwait(false); + } + catch (Exception ex) + { + Logger.LogDebug(ex, "Failed to remove temporary Penumbra collection for {handler}", GetLogIdentifier()); + } } - var applicationId = Guid.NewGuid(); if (awaitIpc) { - try - { - Logger.LogTrace("[{applicationId}] Removing temp collection {CollectionId} for {handler} ({reason})", applicationId, toRelease, GetLogIdentifier(), reason ?? "Cleanup"); - _ipcManager.Penumbra.RemoveTemporaryCollectionAsync(Logger, applicationId, toRelease).GetAwaiter().GetResult(); - } - catch (Exception ex) - { - Logger.LogDebug(ex, "Failed to remove temporary Penumbra collection for {handler}", GetLogIdentifier()); - } - return; + RemoveAsync(toReleasePlayer).GetAwaiter().GetResult(); + RemoveAsync(toReleaseOwned).GetAwaiter().GetResult(); } - - _ = Task.Run(async () => + else { - try - { - Logger.LogTrace("[{applicationId}] Removing temp collection {CollectionId} for {handler} ({reason})", applicationId, toRelease, GetLogIdentifier(), reason ?? "Cleanup"); - await _ipcManager.Penumbra.RemoveTemporaryCollectionAsync(Logger, applicationId, toRelease).ConfigureAwait(false); - } - catch (Exception ex) - { - Logger.LogDebug(ex, "Failed to remove temporary Penumbra collection for {handler}", GetLogIdentifier()); - } - }); + _ = Task.Run(() => RemoveAsync(toReleasePlayer)); + _ = Task.Run(() => RemoveAsync(toReleaseOwned)); + } } private bool AnyPair(Func predicate) @@ -644,8 +689,6 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa var missingResolved = _lastMissingCachedFiles && !hasMissingCachedFiles; _lastMissingCachedFiles = hasMissingCachedFiles; var shouldForce = forced || missingStarted || missingResolved; - var forceApplyCustomization = forced; - if (IsPaused()) { Logger.LogTrace("Permissions paused for {Ident}, skipping reapply", Ident); @@ -663,7 +706,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa var needsApply = !dataApplied; var modFilesChanged = PlayerModFilesChanged(sanitized, _cachedData); var shouldForceMods = shouldForce || modFilesChanged; - forceApplyCustomization = forced || needsApply; + bool forceApplyCustomization = forced || _needsCollectionRebuild || _forceFullReapply; var suppressForcedModRedraw = !forced && hasMissingCachedFiles && dataApplied; if (shouldForceMods) @@ -1082,10 +1125,14 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa "Cannot apply character data: Receiving Player is in an invalid state, deferring application"))); Logger.LogDebug("[BASE-{appBase}] Received data but player was in invalid state, charaHandlerIsNull: {charaIsNull}, playerPointerIsNull: {ptrIsNull}", applicationBase, _charaHandler == null, PlayerCharacter == IntPtr.Zero); - var hasDiffMods = characterData.CheckUpdatedData(applicationBase, _cachedData, Logger, - this, forceApplyCustomization, forceApplyMods: false) - .Any(p => p.Value.Contains(PlayerChanges.ModManip) || p.Value.Contains(PlayerChanges.ModFiles)); - _forceApplyMods = hasDiffMods || _forceApplyMods || _cachedData == null; + var diffs = characterData.CheckUpdatedData(applicationBase, _cachedData, Logger, + this, forceApplyCustomization, forceApplyMods: false); + + var hasDiffPlayerMods = + diffs.TryGetValue(ObjectKind.Player, out var set) + && (set.Contains(PlayerChanges.ModManip) || set.Contains(PlayerChanges.ModFiles)); + + _forceApplyMods = hasDiffPlayerMods || _forceApplyMods || _cachedData == null; _cachedData = characterData; _forceFullReapply = true; Logger.LogDebug("[BASE-{appBase}] Setting data: {hash}, forceApplyMods: {force}", applicationBase, _cachedData.DataHash.Value, _forceApplyMods); @@ -1213,7 +1260,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa lock (_ownedRetryGate) { - _pendingOwnedChanges[kind] = new HashSet(changes); + _pendingOwnedChanges[kind] = [.. changes]; if (!_ownedRetryTask.IsCompleted) { return; @@ -1231,7 +1278,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa { if (!_pendingOwnedChanges.Remove(kind)) { - return; + // nothing to remove } } } @@ -1285,7 +1332,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa continue; } - if ((_applicationTask?.IsCompleted ?? true) == false || (_pairDownloadTask?.IsCompleted ?? true) == false) + if (!(_applicationTask?.IsCompleted ?? true) || !(_pairDownloadTask?.IsCompleted ?? true)) { await Task.Delay(delay, token).ConfigureAwait(false); delay = IncreaseRetryDelay(delay); @@ -1662,66 +1709,75 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa } token.ThrowIfCancellationRequested(); + + var kind = changes.Key; + var changeSet = changes.Value; + var tasks = new List(); - bool needsRedraw = false; - foreach (var change in changes.Value.OrderBy(p => (int)p)) + + bool needsRedraw = + changeSet.Contains(PlayerChanges.ForcedRedraw) + || changeSet.Contains(PlayerChanges.ModFiles); + + bool isIpcOnly = + !needsRedraw + && changeSet.All(c => c is PlayerChanges.Honorific + or PlayerChanges.Moodles + or PlayerChanges.PetNames + or PlayerChanges.Heels); + + foreach (var change in changeSet.OrderBy(p => (int)p)) { Logger.LogDebug("[{applicationId}] Processing {change} for {handler}", applicationId, change, handler); + switch (change) { case PlayerChanges.Customize: - if (charaData.CustomizePlusData.TryGetValue(changes.Key, out var customizePlusData)) - { - tasks.Add(ApplyCustomizeAsync(handler.Address, customizePlusData, changes.Key)); - } - else if (_customizeIds.TryGetValue(changes.Key, out var customizeId)) - { - tasks.Add(RevertCustomizeAsync(customizeId, changes.Key)); - } + if (charaData.CustomizePlusData.TryGetValue(kind, out var customizePlusData) && !string.IsNullOrEmpty(customizePlusData)) + tasks.Add(ApplyCustomizeAsync(handler.Address, customizePlusData, kind)); + else if (_customizeIds.TryGetValue(kind, out var customizeId)) + tasks.Add(RevertCustomizeAsync(customizeId, kind)); break; case PlayerChanges.Heels: - tasks.Add(_ipcManager.Heels.SetOffsetForPlayerAsync(handler.Address, charaData.HeelsData)); + if (!string.IsNullOrEmpty(charaData.HeelsData)) + tasks.Add(_ipcManager.Heels.SetOffsetForPlayerAsync(handler.Address, charaData.HeelsData)); break; case PlayerChanges.Honorific: - tasks.Add(_ipcManager.Honorific.SetTitleAsync(handler.Address, charaData.HonorificData)); + if (!string.IsNullOrEmpty(charaData.HonorificData)) + tasks.Add(_ipcManager.Honorific.SetTitleAsync(handler.Address, charaData.HonorificData)); break; case PlayerChanges.Glamourer: - if (charaData.GlamourerData.TryGetValue(changes.Key, out var glamourerData)) - { + if (charaData.GlamourerData.TryGetValue(kind, out var glamourerData) && !string.IsNullOrEmpty(glamourerData)) tasks.Add(_ipcManager.Glamourer.ApplyAllAsync(Logger, handler, glamourerData, applicationId, token)); - } - break; - - case PlayerChanges.Moodles: - tasks.Add(_ipcManager.Moodles.SetStatusAsync(handler.Address, charaData.MoodlesData)); - break; - - case PlayerChanges.PetNames: - tasks.Add(_ipcManager.PetNames.SetPlayerData(handler.Address, charaData.PetNamesData)); - break; - - case PlayerChanges.ForcedRedraw: needsRedraw = true; break; - default: + case PlayerChanges.Moodles: + if (!string.IsNullOrEmpty(charaData.MoodlesData)) + tasks.Add(_ipcManager.Moodles.SetStatusAsync(handler.Address, charaData.MoodlesData)); + break; + + case PlayerChanges.PetNames: + if (!string.IsNullOrEmpty(charaData.PetNamesData)) + tasks.Add(_ipcManager.PetNames.SetPlayerData(handler.Address, charaData.PetNamesData)); + break; + + case PlayerChanges.ModFiles: + case PlayerChanges.ForcedRedraw: break; } + token.ThrowIfCancellationRequested(); } if (tasks.Count > 0) - { await Task.WhenAll(tasks).ConfigureAwait(false); - } - if (needsRedraw) - { + if (!isIpcOnly && needsRedraw && _ipcManager.Penumbra.APIAvailable) await _ipcManager.Penumbra.RedrawAsync(Logger, handler, applicationId, token).ConfigureAwait(false); - } return true; } @@ -1873,11 +1929,18 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa .ConfigureAwait(false); } + private Task? _pairDownloadTask; private Task _visibilityGraceTask; - private async Task DownloadAndApplyCharacterAsync(Guid applicationBase, CharacterData charaData, Dictionary> updatedData, - bool updateModdedPaths, bool updateManip, Dictionary<(string GamePath, string? Hash), string>? cachedModdedPaths, CancellationToken downloadToken) + private async Task DownloadAndApplyCharacterAsync( + Guid applicationBase, + CharacterData charaData, + Dictionary> updatedData, + bool updateModdedPaths, + bool updateManip, + Dictionary<(string GamePath, string? Hash), string>? cachedModdedPaths, + CancellationToken downloadToken) { var concurrencyLease = await _pairProcessingLimiter.AcquireAsync(downloadToken).ConfigureAwait(false); try @@ -1904,15 +1967,20 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa { if (_pairDownloadTask != null && !_pairDownloadTask.IsCompleted) { - Logger.LogDebug("[BASE-{appBase}] Finishing prior running download task for player {name}, {kind}", applicationBase, PlayerName, updatedData); + Logger.LogDebug("[BASE-{appBase}] Finishing prior running download task for player {name}, {kind}", + applicationBase, PlayerName, updatedData); await _pairDownloadTask.ConfigureAwait(false); } - Logger.LogDebug("[BASE-{appBase}] Downloading missing files for player {name}, {kind}", applicationBase, PlayerName, updatedData); + Logger.LogDebug("[BASE-{appBase}] Downloading missing files for player {name}, {kind}", + applicationBase, PlayerName, updatedData); Mediator.Publish(new EventMessage(new Event(PlayerName, user, nameof(PairHandlerAdapter), EventSeverity.Informational, $"Starting download for {toDownloadReplacements.Count} files"))); - var toDownloadFiles = await _downloadManager.InitiateDownloadList(_charaHandler!, toDownloadReplacements, downloadToken).ConfigureAwait(false); + + var toDownloadFiles = await _downloadManager + .InitiateDownloadList(_charaHandler!, toDownloadReplacements, downloadToken) + .ConfigureAwait(false); if (!_playerPerformanceService.ComputeAndAutoPauseOnVRAMUsageThresholds(this, charaData, toDownloadFiles)) { @@ -1922,7 +1990,9 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa } var handlerForDownload = _charaHandler; - _pairDownloadTask = Task.Run(async () => await _downloadManager.DownloadFiles(handlerForDownload, toDownloadReplacements, downloadToken, skipDownscaleForPair, skipDecimationForPair).ConfigureAwait(false)); + await (_pairDownloadTask = Task.Run(async () => + await _downloadManager.DownloadFiles(handlerForDownload, toDownloadReplacements, downloadToken, skipDownscaleForPair, skipDecimationForPair) + .ConfigureAwait(false))).ConfigureAwait(false); await _pairDownloadTask.ConfigureAwait(false); @@ -1936,38 +2006,33 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa if (!skipDownscaleForPair) { var downloadedTextureHashes = toDownloadReplacements - .Where(static replacement => replacement.GamePaths.Any(static path => path.EndsWith(".tex", StringComparison.OrdinalIgnoreCase))) - .Select(static replacement => replacement.Hash) + .Where(static r => r.GamePaths.Any(static p => p.EndsWith(".tex", StringComparison.OrdinalIgnoreCase))) + .Select(static r => r.Hash) .Distinct(StringComparer.OrdinalIgnoreCase) .ToList(); if (downloadedTextureHashes.Count > 0) - { await _textureDownscaleService.WaitForPendingJobsAsync(downloadedTextureHashes, downloadToken).ConfigureAwait(false); - } } if (!skipDecimationForPair) { var downloadedModelHashes = toDownloadReplacements - .Where(static replacement => replacement.GamePaths.Any(static path => path.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase))) - .Select(static replacement => replacement.Hash) + .Where(static r => r.GamePaths.Any(static p => p.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase))) + .Select(static r => r.Hash) .Distinct(StringComparer.OrdinalIgnoreCase) .ToList(); if (downloadedModelHashes.Count > 0) - { await _modelDecimationService.WaitForPendingJobsAsync(downloadedModelHashes, downloadToken).ConfigureAwait(false); - } } toDownloadReplacements = TryCalculateModdedDictionary(applicationBase, charaData, out moddedPaths, downloadToken); missingReplacements = toDownloadReplacements; - if (toDownloadReplacements.TrueForAll(c => _downloadManager.ForbiddenTransfers.Exists(f => string.Equals(f.Hash, c.Hash, StringComparison.Ordinal)))) - { + if (toDownloadReplacements.TrueForAll(c => + _downloadManager.ForbiddenTransfers.Exists(f => string.Equals(f.Hash, c.Hash, StringComparison.Ordinal)))) break; - } await Task.Delay(TimeSpan.FromSeconds(2), downloadToken).ConfigureAwait(false); } @@ -1999,9 +2064,9 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa var hasCriticalMissing = missingCritical > 0; var hasNonCriticalMissing = missingNonCritical > 0; - var hasDownloadableMissing = missingReplacements.Any(replacement => !IsForbiddenHash(replacement.Hash)); + var hasDownloadableMissing = missingReplacements.Exists(r => !IsForbiddenHash(r.Hash)); var hasDownloadableCriticalMissing = hasCriticalMissing - && missingReplacements.Any(replacement => !IsForbiddenHash(replacement.Hash) && IsCriticalModReplacement(replacement)); + && missingReplacements.Exists(r => !IsForbiddenHash(r.Hash) && IsCriticalModReplacement(r)); pendingModReapply = hasDownloadableMissing; _lastModApplyDeferred = false; @@ -2039,7 +2104,9 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa var handlerForApply = _charaHandler; if (handlerForApply is null || handlerForApply.Address == nint.Zero) { - Logger.LogDebug("[BASE-{appBase}] Handler not available for {player}, cached data for later application", applicationBase, GetLogIdentifier()); + Logger.LogDebug("[BASE-{appBase}] Handler not available for {player}, cached data for later application", + applicationBase, GetLogIdentifier()); + _cachedData = charaData; _pairStateCache.Store(Ident, charaData); _forceFullReapply = true; @@ -2064,10 +2131,122 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa return; } + var needsOwnedCollectionAssign = + _ipcManager.Penumbra.APIAvailable + && updatedData.Any(kvp => + kvp.Key != ObjectKind.Player + && kvp.Value.Contains(PlayerChanges.ModFiles)); + + var wantsOwnedCollectionAssignNow = + needsOwnedCollectionAssign + && updateModdedPaths; + + Guid ownedAssignCollection = Guid.Empty; + if (wantsOwnedCollectionAssignNow) + { + ownedAssignCollection = EnsureOwnedPenumbraCollection(); + } + _applicationCancellationTokenSource = _applicationCancellationTokenSource.CancelRecreate() ?? new CancellationTokenSource(); var token = _applicationCancellationTokenSource.Token; - _applicationTask = ApplyCharacterDataAsync(applicationBase, handlerForApply, charaData, updatedData, updateModdedPaths, updateManip, moddedPaths, wantsModApply, pendingModReapply, token); + _applicationTask = ApplyCharacterDataAsync( + applicationBase, + handlerForApply, + charaData, + updatedData, + updateModdedPaths, + updateManip, + moddedPaths, + wantsModApply, + pendingModReapply, + token); + + if (wantsOwnedCollectionAssignNow && ownedAssignCollection != Guid.Empty) + { + var applyTaskSnapshot = _applicationTask; + var updatedSnapshot = updatedData.ToDictionary(k => k.Key, v => new HashSet(v.Value)); + var ownedCollectionSnapshot = ownedAssignCollection; + + _ = Task.Run(async () => + { + try + { + await applyTaskSnapshot.ConfigureAwait(false); + token.ThrowIfCancellationRequested(); + + foreach (var kvp in updatedSnapshot) + { + if (kvp.Key == ObjectKind.Player) + continue; + + if (!kvp.Value.Contains(PlayerChanges.ModFiles)) + continue; + + var delay = OwnedRetryInitialDelay; + + while (!token.IsCancellationRequested) + { + var ownedPtr = await ResolveOtherOwnedPtrAsync(kvp.Key, handlerForApply.Address).ConfigureAwait(false); + if (ownedPtr != nint.Zero) + { + using var ownedHandler = await _gameObjectHandlerFactory + .Create(kvp.Key, () => ownedPtr, isWatched: false) + .ConfigureAwait(false); + + if (ownedHandler.Address != nint.Zero) + { + var objIndex = await _dalamudUtil.RunOnFrameworkThread(() => + { + var go = ownedHandler.GetGameObject(); + return go?.ObjectIndex; + }).ConfigureAwait(false); + + if (objIndex.HasValue) + { + await _ipcManager.Penumbra + .AssignTemporaryCollectionAsync(Logger, ownedCollectionSnapshot, objIndex.Value) + .ConfigureAwait(false); + + await _ipcManager.Penumbra + .RedrawAsync(Logger, ownedHandler, Guid.NewGuid(), token) + .ConfigureAwait(false); + + Logger.LogDebug( + "Assigned OWNED temp collection {collection} to owned object {kind} for {handler} and redrew", + ownedCollectionSnapshot, kvp.Key, GetLogIdentifier()); + + break; + } + } + } + + await Task.Delay(delay, token).ConfigureAwait(false); + delay = IncreaseRetryDelay(delay); + } + } + } + catch (OperationCanceledException) + { + Logger.LogTrace("Owned object collection assignment task cancelled for {handler}", GetLogIdentifier()); + } + catch (Exception ex) + { + Logger.LogDebug(ex, "Owned object collection assignment task failed for {handler}", GetLogIdentifier()); + } + }, CancellationToken.None); + } + + async Task ResolveOtherOwnedPtrAsync(ObjectKind kind, nint playerPtr) + { + return kind switch + { + ObjectKind.MinionOrMount => await _dalamudUtil.GetMinionOrMountAsync(playerPtr).ConfigureAwait(false), + ObjectKind.Pet => await _dalamudUtil.GetPetAsync(playerPtr).ConfigureAwait(false), + ObjectKind.Companion => await _dalamudUtil.GetCompanionAsync(playerPtr).ConfigureAwait(false), + _ => nint.Zero + }; + } } finally { @@ -2075,23 +2254,38 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa } } - private async Task ApplyCharacterDataAsync(Guid applicationBase, GameObjectHandler handlerForApply, CharacterData charaData, Dictionary> updatedData, bool updateModdedPaths, bool updateManip, - Dictionary<(string GamePath, string? Hash), string> moddedPaths, bool wantsModApply, bool pendingModReapply, CancellationToken token) + private async Task ApplyCharacterDataAsync( + Guid applicationBase, + GameObjectHandler handlerForApply, + CharacterData charaData, + Dictionary> updatedData, + bool updateModdedPaths, + bool updateManip, + Dictionary<(string GamePath, string? Hash), string> moddedPaths, + bool wantsModApply, + bool pendingModReapply, + CancellationToken token) { try { _applicationId = Guid.NewGuid(); - Logger.LogDebug("[BASE-{applicationId}] Starting application task for {handler}: {appId}", applicationBase, GetLogIdentifier(), _applicationId); + Logger.LogDebug("[BASE-{applicationId}] Starting application task for {handler}: {appId}", + applicationBase, GetLogIdentifier(), _applicationId); - Logger.LogDebug("[{applicationId}] Waiting for initial draw for for {handler}", _applicationId, handlerForApply); + Logger.LogDebug("[{applicationId}] Waiting for initial draw for {handler}", _applicationId, handlerForApply); await _dalamudUtil.WaitWhileCharacterIsDrawing(Logger, handlerForApply, _applicationId, 30000, token).ConfigureAwait(false); + if (handlerForApply.Address != nint.Zero) { - var fullyLoaded = await _actorObjectService.WaitForFullyLoadedAsync(handlerForApply.Address, token, FullyLoadedTimeoutMsPlayer).ConfigureAwait(false); + var fullyLoaded = await _actorObjectService + .WaitForFullyLoadedAsync(handlerForApply.Address, token, FullyLoadedTimeoutMsPlayer) + .ConfigureAwait(false); + if (!fullyLoaded) { Logger.LogDebug("[BASE-{applicationId}] Timed out waiting for {handler} to fully load, caching data for later application", applicationBase, GetLogIdentifier()); + _cachedData = charaData; _pairStateCache.Store(Ident, charaData); _forceFullReapply = true; @@ -2102,33 +2296,82 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa token.ThrowIfCancellationRequested(); - Guid penumbraCollection = Guid.Empty; - if (updateModdedPaths || updateManip) + static bool IsPlayerIpcOnly(PlayerChanges c) => + c is PlayerChanges.Honorific or PlayerChanges.Moodles or PlayerChanges.PetNames or PlayerChanges.Heels; + + // Determine if only IPC-only changes are present + bool playerHasDelta = updatedData.TryGetValue(ObjectKind.Player, out var playerDelta) && playerDelta.Count > 0; + + bool playerIsIpcOnlyDelta = false; + + if (playerDelta != null) { - penumbraCollection = EnsurePenumbraCollection(); - if (penumbraCollection == Guid.Empty) + playerIsIpcOnlyDelta = playerHasDelta && playerDelta.All(IsPlayerIpcOnly); + } + + bool anyNonPlayerDelta = updatedData.Any(kvp => kvp.Key != ObjectKind.Player && kvp.Value.Count > 0); + + bool updatePlayerMods = updateModdedPaths || updateManip; + + bool updateOwnedMods = updatedData.Any(kvp => kvp.Key != ObjectKind.Player && kvp.Value.Contains(PlayerChanges.ModFiles)); + + bool isPureIpcOnly = playerIsIpcOnlyDelta && !anyNonPlayerDelta && !updatePlayerMods && !updateOwnedMods; + + // Short-circuit if only IPC changes + Guid playerCollection = Guid.Empty; + Guid ownedCollection = Guid.Empty; + + if (!isPureIpcOnly) + { + if (updatePlayerMods) { - Logger.LogTrace("[BASE-{applicationId}] Penumbra collection unavailable for {handler}, caching data for later application", applicationBase, GetLogIdentifier()); - _cachedData = charaData; - _pairStateCache.Store(Ident, charaData); - _forceFullReapply = true; - RecordFailure("Penumbra collection unavailable", "PenumbraUnavailable"); - return; + playerCollection = EnsurePenumbraCollection(); + if (playerCollection == Guid.Empty) + { + _cachedData = charaData; + _pairStateCache.Store(Ident, charaData); + _forceFullReapply = true; + RecordFailure("Penumbra player collection unavailable", "PenumbraUnavailablePlayer"); + return; + } + } + + if (updateOwnedMods) + { + ownedCollection = EnsureOwnedPenumbraCollection(); + if (ownedCollection == Guid.Empty) + { + _cachedData = charaData; + _pairStateCache.Store(Ident, charaData); + _forceFullReapply = true; + RecordFailure("Penumbra owned collection unavailable", "PenumbraUnavailableOwned"); + return; + } + } + + if ((updatePlayerMods || updateOwnedMods) && (moddedPaths.Count == 0)) + { + Logger.LogWarning( + "[{applicationId}] ModdedPaths missing but updatePlayerMods={up} updateOwnedMods={uo}. Rebuilding.", + _applicationId, updatePlayerMods, updateOwnedMods); + + _ = TryCalculateModdedDictionary(applicationBase, charaData, out var rebuilt, token); + moddedPaths = rebuilt; } } - if (updateModdedPaths) + if (!isPureIpcOnly && updatePlayerMods) { - // ensure collection is set + // Get object index on framework thread var objIndex = await _dalamudUtil.RunOnFrameworkThread(() => { var gameObject = handlerForApply.GetGameObject(); return gameObject?.ObjectIndex; }).ConfigureAwait(false); + // Ensure object index is available if (!objIndex.HasValue) { - Logger.LogDebug("[BASE-{applicationId}] GameObject not available for {handler}, caching data for later application", applicationBase, GetLogIdentifier()); _cachedData = charaData; _pairStateCache.Store(Ident, charaData); _forceFullReapply = true; @@ -2136,114 +2379,162 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa return; } - SplitPapMappings(moddedPaths, out var withoutPap, out var papOnly); + // Filter modded paths to player only + var playerGamePaths = charaData.FileReplacements.TryGetValue(ObjectKind.Player, out var repls) && repls?.Count > 0 + ? repls.SelectMany(r => r.GamePaths).Where(p => !string.IsNullOrEmpty(p)).ToHashSet(StringComparer.OrdinalIgnoreCase) + : new HashSet(StringComparer.OrdinalIgnoreCase); - await _ipcManager.Penumbra.AssignTemporaryCollectionAsync(Logger, penumbraCollection, objIndex.Value).ConfigureAwait(false); + // Construct player modded dictionary + var playerModded = new Dictionary<(string GamePath, string? Hash), string>(moddedPaths.Comparer); + foreach (var kv in moddedPaths) + if (playerGamePaths.Contains(kv.Key.GamePath)) + playerModded[kv.Key] = kv.Value; - await _ipcManager.Penumbra.SetTemporaryModsAsync( - Logger, _applicationId, penumbraCollection, - withoutPap.ToDictionary(k => k.Key.GamePath, k => k.Value, StringComparer.Ordinal)) - .ConfigureAwait(false); + // Handle PAP mappings separately to check compatibility + SplitPapMappings(playerModded, out var withoutPap, out var papOnly); - await _ipcManager.Penumbra.RedrawAsync(Logger, handlerForApply, _applicationId, token).ConfigureAwait(false); + // Assign collection via IPC + await _ipcManager.Penumbra.AssignTemporaryCollectionAsync(Logger, playerCollection, objIndex.Value).ConfigureAwait(false); + + // Ensure fully loaded before applying PAP mappings if (handlerForApply.Address != nint.Zero) await _actorObjectService.WaitForFullyLoadedAsync(handlerForApply.Address, token).ConfigureAwait(false); + // Strip incompatible PAP mappings before applying var removedPap = await StripIncompatiblePapAsync(handlerForApply, charaData, papOnly, token).ConfigureAwait(false); if (removedPap > 0) - { - Logger.LogTrace("[{applicationId}] Removed {removedPap} incompatible PAP mappings found for {handler}", _applicationId, removedPap, GetLogIdentifier()); - } + Logger.LogTrace("[{applicationId}] Removed {removedPap} incompatible PAP mappings for {handler}", + _applicationId, removedPap, GetLogIdentifier()); + // Merge back PAP mappings var merged = new Dictionary<(string GamePath, string? Hash), string>(withoutPap, withoutPap.Comparer); foreach (var kv in papOnly) merged[kv.Key] = kv.Value; + // Apply mods via IPC await _ipcManager.Penumbra.SetTemporaryModsAsync( - Logger, _applicationId, penumbraCollection, - merged.ToDictionary(k => k.Key.GamePath, k => k.Value, StringComparer.Ordinal)) + Logger, _applicationId, playerCollection, + merged.ToDictionary(k => k.Key.GamePath, k => k.Value, StringComparer.Ordinal), + scope: "Player") .ConfigureAwait(false); - _lastAppliedModdedPaths = new Dictionary<(string GamePath, string? Hash), string>(merged, merged.Comparer); + // Final redraw + await _ipcManager.Penumbra.RedrawAsync(Logger, handlerForApply, _applicationId, token).ConfigureAwait(false); - LastAppliedDataBytes = -1; - foreach (var path in moddedPaths.Values.Distinct(StringComparer.OrdinalIgnoreCase).Select(v => new FileInfo(v)).Where(p => p.Exists)) + // Cache last applied modded paths + _lastAppliedModdedPaths = new Dictionary<(string GamePath, string? Hash), string>(moddedPaths, moddedPaths.Comparer); + } + + if (!isPureIpcOnly && updateOwnedMods && ownedCollection != Guid.Empty) + { + // Filter modded paths to owned only + var ownedGamePaths = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var k in new[] { ObjectKind.MinionOrMount, ObjectKind.Pet, ObjectKind.Companion }) { - if (LastAppliedDataBytes == -1) LastAppliedDataBytes = 0; + if (charaData.FileReplacements.TryGetValue(k, out var repls) && repls?.Count > 0) + { + foreach (var p in repls.SelectMany(r => r.GamePaths)) + if (!string.IsNullOrEmpty(p)) + ownedGamePaths.Add(p); + } + } - LastAppliedDataBytes += path.Length; + // Construct owned modded dictionary + var ownedModded = new Dictionary<(string GamePath, string? Hash), string>(moddedPaths.Comparer); + foreach (var kv in moddedPaths) + if (ownedGamePaths.Contains(kv.Key.GamePath)) + ownedModded[kv.Key] = kv.Value; + + // Apply owned mods via IPC + if (ownedModded.Count > 0) + { + await _ipcManager.Penumbra.SetTemporaryModsAsync( + Logger, _applicationId, ownedCollection, + ownedModded.ToDictionary(k => k.Key.GamePath, k => k.Value, StringComparer.Ordinal), + scope: "Owned") + .ConfigureAwait(false); } } - if (updateManip) + // Apply manipulation data if needed + if (!isPureIpcOnly && updateManip && playerCollection != Guid.Empty) { - await _ipcManager.Penumbra.SetManipulationDataAsync(Logger, _applicationId, penumbraCollection, charaData.ManipulationData).ConfigureAwait(false); + // Apply manipulation data via IPC + await _ipcManager.Penumbra.SetManipulationDataAsync( + Logger, _applicationId, playerCollection, charaData.ManipulationData) + .ConfigureAwait(false); } token.ThrowIfCancellationRequested(); - foreach (var kind in updatedData) + // Apply changes for each object kind + foreach (var kind in updatedData.Keys.OrderBy(k => k == ObjectKind.Player ? -1 : (int)k)) { - var applied = await ApplyCustomizationDataAsync(_applicationId, kind, charaData, token).ConfigureAwait(false); - if (applied) - { - ClearOwnedObjectRetry(kind.Key); - } - else if (kind.Key != ObjectKind.Player) - { - ScheduleOwnedObjectRetry(kind.Key, kind.Value); - } token.ThrowIfCancellationRequested(); + + var changeSet = updatedData[kind]; + if (changeSet.Count == 0) + continue; + + Guid collectionToUse; + // Determine which collection to use + if (!changeSet.Contains(PlayerChanges.ModFiles)) + { + collectionToUse = kind != ObjectKind.Player ? Guid.Empty : playerCollection; + } + else + { + collectionToUse = kind == ObjectKind.Player ? playerCollection : ownedCollection; + } + + // Owned objects may fail to apply if they are not fully loaded yet. + var applied = await _ownedObjectHandler.ApplyAsync( + _applicationId, + kind, + changeSet, + charaData, + handlerForApply, + collectionToUse, + _customizeIds, + token) + .ConfigureAwait(false); + + if (applied) + ClearOwnedObjectRetry(kind); + else if (kind != ObjectKind.Player) + ScheduleOwnedObjectRetry(kind, changeSet); } _cachedData = charaData; _pairStateCache.Store(Ident, charaData); + if (wantsModApply) - { _pendingModReapply = pendingModReapply; - } + _forceFullReapply = _pendingModReapply; _needsCollectionRebuild = false; - if (LastAppliedApproximateVRAMBytes < 0 || LastAppliedApproximateEffectiveVRAMBytes < 0) - { - _playerPerformanceService.ComputeAndAutoPauseOnVRAMUsageThresholds(this, charaData, new List()); - } - - if (LastAppliedDataTris < 0 || LastAppliedApproximateEffectiveTris < 0) - { - await _playerPerformanceService.CheckTriangleUsageThresholds(this, charaData).ConfigureAwait(false); - } StorePerformanceMetrics(charaData); _lastSuccessfulDataHash = GetDataHashSafe(charaData); _lastSuccessfulApplyAt = DateTime.UtcNow; ClearFailureState(); + Logger.LogDebug("[{applicationId}] Application finished", _applicationId); - } - catch (OperationCanceledException) - { - Logger.LogDebug("[{applicationId}] Application cancelled for {handler}", _applicationId, GetLogIdentifier()); - _cachedData = charaData; - _pairStateCache.Store(Ident, charaData); - _forceFullReapply = true; - RecordFailure("Application cancelled", "Cancellation"); - } - catch (Exception ex) - { - if (ex is AggregateException aggr && aggr.InnerExceptions.Any(e => e is ArgumentNullException)) + } + catch (OperationCanceledException) { - IsVisible = false; - _forceApplyMods = true; - _cachedData = charaData; - _pairStateCache.Store(Ident, charaData); - _forceFullReapply = true; - Logger.LogDebug("[{applicationId}] Cancelled, player turned null during application", _applicationId); - } - else - { - Logger.LogWarning(ex, "[{applicationId}] Cancelled", _applicationId); - _forceFullReapply = true; - } + Logger.LogDebug("[{applicationId}] Application cancelled for {handler}", _applicationId, GetLogIdentifier()); + _cachedData = charaData; + _pairStateCache.Store(Ident, charaData); + _forceFullReapply = true; + RecordFailure("Application cancelled", "Cancellation"); + } + catch (Exception ex) + { + Logger.LogWarning(ex, "[{applicationId}] Application failed", _applicationId); + _cachedData = charaData; + _pairStateCache.Store(Ident, charaData); + _forceFullReapply = true; RecordFailure($"Application failed: {ex.Message}", "Exception"); } } @@ -2396,6 +2687,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa if (objectKind == ObjectKind.Player) { + // Players have their own object kind. using GameObjectHandler tempHandler = await _gameObjectHandlerFactory.Create(ObjectKind.Player, () => address, isWatched: false).ConfigureAwait(false); tempHandler.CompareNameAndThrow(name); Logger.LogDebug("[{applicationId}] Restoring Customization and Equipment for {alias}/{name}", applicationId, alias, name); @@ -2416,6 +2708,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa } else if (objectKind == ObjectKind.MinionOrMount) { + // Minions and mounts share the same object kind. var minionOrMount = await _dalamudUtil.GetMinionOrMountAsync(address).ConfigureAwait(false); if (minionOrMount != nint.Zero) { @@ -2427,6 +2720,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa } else if (objectKind == ObjectKind.Pet) { + // Pets share the same object kind. var pet = await _dalamudUtil.GetPetAsync(address).ConfigureAwait(false); if (pet != nint.Zero) { @@ -2438,6 +2732,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa } else if (objectKind == ObjectKind.Companion) { + // Companions share the same object kind. var companion = await _dalamudUtil.GetCompanionAsync(address).ConfigureAwait(false); if (companion != nint.Zero) { @@ -2449,13 +2744,18 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa } } - private List TryCalculateModdedDictionary(Guid applicationBase, CharacterData charaData, out Dictionary<(string GamePath, string? Hash), string> moddedDictionary, CancellationToken token) + private List TryCalculateModdedDictionary( + Guid applicationBase, + CharacterData charaData, + out Dictionary<(string GamePath, string? Hash), string> moddedDictionary, + CancellationToken token) { Stopwatch st = Stopwatch.StartNew(); ConcurrentBag missingFiles = []; moddedDictionary = []; ConcurrentDictionary<(string GamePath, string? Hash), string> outputDict = new(); - bool hasMigrationChanges = false; + int hasMigrationChanges = 0; + bool skipDownscaleForPair = ShouldSkipDownscale(); bool skipDecimationForPair = ShouldSkipDecimation(); @@ -2463,30 +2763,41 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa { RefreshPapBlockCacheIfAnimSettingsChanged(); - var replacementList = charaData.FileReplacements.SelectMany(k => k.Value.Where(v => string.IsNullOrEmpty(v.FileSwapPath))).ToList(); - Parallel.ForEach(replacementList, new ParallelOptions() - { - CancellationToken = token, - MaxDegreeOfParallelism = 4 - }, - (item) => - { - token.ThrowIfCancellationRequested(); - var fileCache = _fileDbManager.GetFileCacheByHash(item.Hash); - if (fileCache is not null && !File.Exists(fileCache.ResolvedFilepath)) - { - Logger.LogTrace("[BASE-{appBase}] Cached path {Path} missing on disk for hash {Hash}, removing cache entry", applicationBase, fileCache.ResolvedFilepath, item.Hash); - _fileDbManager.RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath); - fileCache = null; - } + var replacementList = charaData.FileReplacements + .SelectMany(k => k.Value.Where(v => string.IsNullOrEmpty(v.FileSwapPath))) + .ToList(); - if (fileCache != null) + Parallel.ForEach( + replacementList, + new ParallelOptions { - if (string.IsNullOrEmpty(new FileInfo(fileCache.ResolvedFilepath).Extension)) + CancellationToken = token, + MaxDegreeOfParallelism = 4 + }, + item => + { + token.ThrowIfCancellationRequested(); + + var fileCache = _fileDbManager.GetFileCacheByHash(item.Hash); + if (fileCache is not null && !File.Exists(fileCache.ResolvedFilepath)) { - hasMigrationChanges = true; - var anyGamePath = item.GamePaths.FirstOrDefault(); + Logger.LogTrace("[BASE-{appBase}] Cached path {Path} missing on disk for hash {Hash}, removing cache entry", + applicationBase, fileCache.ResolvedFilepath, item.Hash); + _fileDbManager.RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath); + fileCache = null; + } + + if (fileCache is null) + { + Logger.LogTrace("Missing file: {hash}", item.Hash); + missingFiles.Add(item); + return; + } + + if (string.IsNullOrEmpty(Path.GetExtension(fileCache.ResolvedFilepath))) + { + var anyGamePath = item.GamePaths.FirstOrDefault(); if (!string.IsNullOrEmpty(anyGamePath)) { var ext = Path.GetExtension(anyGamePath); @@ -2494,7 +2805,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa if (!string.IsNullOrEmpty(extNoDot)) { - hasMigrationChanges = true; + Interlocked.Exchange(ref hasMigrationChanges, 1); fileCache = _fileDbManager.MigrateFileHashToExtension(fileCache, extNoDot); } } @@ -2503,7 +2814,6 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa foreach (var gamePath in item.GamePaths) { var mode = _configService.Current.AnimationValidationMode; - if (mode != AnimationValidationMode.Unsafe && gamePath.EndsWith(".pap", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(item.Hash) @@ -2512,19 +2822,23 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa continue; } - var preferredPath = skipDownscaleForPair - ? fileCache.ResolvedFilepath - : _textureDownscaleService.GetPreferredPath(item.Hash, fileCache.ResolvedFilepath); + var preferredPath = fileCache.ResolvedFilepath; + + // Only downscale textures. + if (!skipDownscaleForPair && gamePath.EndsWith(".tex", StringComparison.OrdinalIgnoreCase)) + { + preferredPath = _textureDownscaleService.GetPreferredPath(item.Hash, preferredPath); + } + + // Only decimate models. + if (!skipDecimationForPair && gamePath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase)) + { + preferredPath = _modelDecimationService.GetPreferredPath(item.Hash, preferredPath); + } outputDict[(gamePath, item.Hash)] = preferredPath; } - } - else - { - Logger.LogTrace("Missing file: {hash}", item.Hash); - missingFiles.Add(item); - } - }); + }); moddedDictionary = outputDict.ToDictionary(k => k.Key, k => k.Value); @@ -2546,9 +2860,14 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa { Logger.LogError(ex, "[BASE-{appBase}] Something went wrong during calculation replacements", applicationBase); } - if (hasMigrationChanges) _fileDbManager.WriteOutFullCsv(); + + if (hasMigrationChanges == 1) + _fileDbManager.WriteOutFullCsv(); + st.Stop(); - Logger.LogDebug("[BASE-{appBase}] ModdedPaths calculated in {time}ms, missing files: {count}, total files: {total}", applicationBase, st.ElapsedMilliseconds, missingFiles.Count, moddedDictionary.Keys.Count); + Logger.LogDebug("[BASE-{appBase}] ModdedPaths calculated in {time}ms, missing files: {count}, total files: {total}", + applicationBase, st.ElapsedMilliseconds, missingFiles.Count, moddedDictionary.Keys.Count); + return [.. missingFiles]; } @@ -2699,10 +3018,10 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa ApplyCharacterData(pending.ApplicationId, pending.CharacterData, pending.Forced); } catch (Exception ex) - { - Logger.LogError(ex, "Failed applying queued data for {handler}", GetLogIdentifier()); - } - }); + { + Logger.LogError(ex, "Failed applying queued data for {handler}", GetLogIdentifier()); + } + }); } private void HandleActorTracked(ActorObjectService.ActorDescriptor descriptor) diff --git a/LightlessSync/Plugin.cs b/LightlessSync/Plugin.cs index 4e1ed4e..55a3dae 100644 --- a/LightlessSync/Plugin.cs +++ b/LightlessSync/Plugin.cs @@ -428,6 +428,7 @@ public sealed class Plugin : IDalamudPlugin return cfg; }); services.AddSingleton(sp => new ServerConfigService(configDir)); + services.AddSingleton(sp => new PenumbraJanitorConfigService(configDir)); services.AddSingleton(sp => new NotesConfigService(configDir)); services.AddSingleton(sp => new PairTagConfigService(configDir)); services.AddSingleton(sp => new SyncshellTagConfigService(configDir)); @@ -441,6 +442,7 @@ public sealed class Plugin : IDalamudPlugin services.AddSingleton>(sp => sp.GetRequiredService()); services.AddSingleton>(sp => sp.GetRequiredService()); services.AddSingleton>(sp => sp.GetRequiredService()); + services.AddSingleton>(sp => sp.GetRequiredService()); services.AddSingleton>(sp => sp.GetRequiredService()); services.AddSingleton>(sp => sp.GetRequiredService()); services.AddSingleton>(sp => sp.GetRequiredService()); diff --git a/LightlessSync/Services/ActorTracking/ActorObjectService.cs b/LightlessSync/Services/ActorTracking/ActorObjectService.cs index e443496..3813fb4 100644 --- a/LightlessSync/Services/ActorTracking/ActorObjectService.cs +++ b/LightlessSync/Services/ActorTracking/ActorObjectService.cs @@ -14,6 +14,7 @@ using BattleNpcSubKind = FFXIVClientStructs.FFXIV.Client.Game.Object.BattleNpcSu using IPlayerCharacter = Dalamud.Game.ClientState.Objects.SubKinds.IPlayerCharacter; using DalamudObjectKind = Dalamud.Game.ClientState.Objects.Enums.ObjectKind; using LightlessObjectKind = LightlessSync.API.Data.Enum.ObjectKind; +using System.Runtime.InteropServices; namespace LightlessSync.Services.ActorTracking; @@ -57,6 +58,8 @@ public sealed class ActorObjectService : IHostedService, IDisposable, IMediatorS private bool _hooksActive; private static readonly TimeSpan SnapshotRefreshInterval = TimeSpan.FromSeconds(1); private DateTime _nextRefreshAllowed = DateTime.MinValue; + private int _warmStartQueued; + private int _warmStartRan; public ActorObjectService( ILogger logger, @@ -74,7 +77,21 @@ public sealed class ActorObjectService : IHostedService, IDisposable, IMediatorS _clientState = clientState; _condition = condition; _mediator = mediator; + _mediator.Subscribe(this, _ => + { + QueueWarmStart("PenumbraInitialized"); + }); + _mediator.Subscribe(this, _ => + { + QueueWarmStart("Connected"); + }); + + // Optional: helps after zoning + _mediator.Subscribe(this, _ => + { + QueueWarmStart("ZoneSwitchEnd"); + }); _mediator.Subscribe(this, (msg) => { if (!msg.OwnedObject) return; @@ -96,7 +113,6 @@ public sealed class ActorObjectService : IHostedService, IDisposable, IMediatorS } private bool IsZoning => _condition[ConditionFlag.BetweenAreas] || _condition[ConditionFlag.BetweenAreas51]; - private ActorSnapshot Snapshot => Volatile.Read(ref _snapshot); private GposeSnapshot CurrentGposeSnapshot => Volatile.Read(ref _gposeSnapshot); @@ -341,6 +357,8 @@ public sealed class ActorObjectService : IHostedService, IDisposable, IMediatorS public Task StopAsync(CancellationToken cancellationToken) { + _warmStartRan = 0; + DisposeHooks(); _activePlayers.Clear(); _gposePlayers.Clear(); @@ -1147,6 +1165,57 @@ public sealed class ActorObjectService : IHostedService, IDisposable, IMediatorS PublishGposeSnapshot(); } + private void QueueWarmStart(string reason) + { + if (Interlocked.Exchange(ref _warmStartQueued, 1) == 1) + return; + + _ = Task.Run(async () => + { + try + { + if (Interlocked.Exchange(ref _warmStartRan, 1) == 1) + return; + + await Task.Delay(500).ConfigureAwait(false); + + if (IsZoning) + return; + + await _framework.RunOnFrameworkThread(() => + { + RefreshTrackedActorsInternal(); + + var snapshot = Snapshot; + + var published = new HashSet(); + + foreach (var d in snapshot.PlayerDescriptors) + { + if (d.Address != nint.Zero && published.Add(d.Address)) + _mediator.Publish(new ActorTrackedMessage(d)); + } + + foreach (var d in snapshot.OwnedDescriptors) + { + if (d.Address != nint.Zero && published.Add(d.Address)) + _mediator.Publish(new ActorTrackedMessage(d)); + } + + _logger.LogDebug("WarmStart republished {count} actors ({reason})", published.Count, reason); + }).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "WarmStart failed ({reason})", reason); + } + finally + { + Interlocked.Exchange(ref _warmStartQueued, 0); + } + }); + } + private unsafe void TrackGposeObject(GameObject* gameObject) { if (gameObject == null) @@ -1240,6 +1309,7 @@ public sealed class ActorObjectService : IHostedService, IDisposable, IMediatorS return true; } + [StructLayout(LayoutKind.Auto)] private readonly record struct LoadState(bool IsValid, bool IsLoaded) { public static LoadState Invalid => new(false, false); diff --git a/LightlessSync/Services/LightFinder/LightFinderPlateHandler.cs b/LightlessSync/Services/LightFinder/LightFinderPlateHandler.cs index 97217c8..f224fd0 100644 --- a/LightlessSync/Services/LightFinder/LightFinderPlateHandler.cs +++ b/LightlessSync/Services/LightFinder/LightFinderPlateHandler.cs @@ -68,6 +68,7 @@ public unsafe class LightFinderPlateHandler : IHostedService, IMediatorSubscribe ImGuiWindowFlags.NoMove | ImGuiWindowFlags.NoSavedSettings | ImGuiWindowFlags.NoNav | + ImGuiWindowFlags.NoFocusOnAppearing | ImGuiWindowFlags.NoInputs; private readonly List _uiRects = new(128); diff --git a/LightlessSync/Services/Mediator/Messages.cs b/LightlessSync/Services/Mediator/Messages.cs index e6db9e7..548e201 100644 --- a/LightlessSync/Services/Mediator/Messages.cs +++ b/LightlessSync/Services/Mediator/Messages.cs @@ -138,5 +138,6 @@ public record GroupCollectionChangedMessage : MessageBase; public record OpenUserProfileMessage(UserData User) : MessageBase; public record LocationSharingMessage(UserData User, LocationInfo LocationInfo, DateTimeOffset ExpireAt) : MessageBase; public record MapChangedMessage(uint MapId) : MessageBase; +public record PenumbraTempCollectionsCleanedMessage : MessageBase; #pragma warning restore S2094 #pragma warning restore MA0048 // File name must match type name diff --git a/LightlessSync/Services/PenumbraTempCollectionJanitor.cs b/LightlessSync/Services/PenumbraTempCollectionJanitor.cs index 03fb53b..20b5450 100644 --- a/LightlessSync/Services/PenumbraTempCollectionJanitor.cs +++ b/LightlessSync/Services/PenumbraTempCollectionJanitor.cs @@ -8,14 +8,14 @@ namespace LightlessSync.Services; public sealed class PenumbraTempCollectionJanitor : DisposableMediatorSubscriberBase { private readonly IpcManager _ipc; - private readonly LightlessConfigService _config; + private readonly PenumbraJanitorConfigService _config; private int _ran; public PenumbraTempCollectionJanitor( ILogger logger, LightlessMediator mediator, IpcManager ipc, - LightlessConfigService config) : base(logger, mediator) + PenumbraJanitorConfigService config) : base(logger, mediator) { _ipc = ipc; _config = config; @@ -67,5 +67,8 @@ public sealed class PenumbraTempCollectionJanitor : DisposableMediatorSubscriber _config.Current.OrphanableTempCollections.Clear(); _config.Save(); + + // Notify cleanup complete + Mediator.Publish(new PenumbraTempCollectionsCleanedMessage()); } } \ No newline at end of file diff --git a/LightlessSync/UI/DtrEntry.cs b/LightlessSync/UI/DtrEntry.cs index 5aa69eb..c771164 100644 --- a/LightlessSync/UI/DtrEntry.cs +++ b/LightlessSync/UI/DtrEntry.cs @@ -103,7 +103,7 @@ public sealed class DtrEntry : IDisposable, IHostedService public async Task StopAsync(CancellationToken cancellationToken) { - _cancellationTokenSource.Cancel(); + await _cancellationTokenSource.CancelAsync().ConfigureAwait(false); if (_dalamudUtilService.IsOnFrameworkThread) { diff --git a/LightlessSync/UI/LightlessNotificationUI.cs b/LightlessSync/UI/LightlessNotificationUI.cs index 1d0a477..aa199b0 100644 --- a/LightlessSync/UI/LightlessNotificationUI.cs +++ b/LightlessSync/UI/LightlessNotificationUI.cs @@ -33,13 +33,13 @@ public class LightlessNotificationUi : WindowMediatorSubscriberBase private readonly Dictionary _notificationTargetYOffsets = []; private readonly Dictionary _notificationBackgrounds = []; - public LightlessNotificationUi(ILogger logger, LightlessMediator mediator, PerformanceCollectorService performanceCollector, LightlessConfigService configService) + public LightlessNotificationUi(ILogger logger, LightlessMediator mediator, PerformanceCollectorService performanceCollector, LightlessConfigService configService) : base(logger, mediator, "Lightless Notifications##LightlessNotifications", performanceCollector) { _configService = configService; Flags = ImGuiWindowFlags.NoDecoration | - ImGuiWindowFlags.NoMove | - ImGuiWindowFlags.NoResize | + ImGuiWindowFlags.NoMove | + ImGuiWindowFlags.NoResize | ImGuiWindowFlags.NoSavedSettings | ImGuiWindowFlags.NoFocusOnAppearing | ImGuiWindowFlags.NoNav | @@ -47,6 +47,7 @@ public class LightlessNotificationUi : WindowMediatorSubscriberBase ImGuiWindowFlags.NoCollapse | ImGuiWindowFlags.NoTitleBar | ImGuiWindowFlags.NoScrollbar | + ImGuiWindowFlags.NoFocusOnAppearing | ImGuiWindowFlags.AlwaysAutoResize; PositionCondition = ImGuiCond.Always; diff --git a/LightlessSync/Utils/MemoryProcessProbe.cs b/LightlessSync/Utils/MemoryProcessProbe.cs new file mode 100644 index 0000000..ea277ac --- /dev/null +++ b/LightlessSync/Utils/MemoryProcessProbe.cs @@ -0,0 +1,41 @@ +using System; +using System.Runtime.InteropServices; + +namespace LightlessSync.Utils; + +internal static class MemoryProcessProbe +{ + [DllImport("kernel32.dll")] + private static extern nint GetCurrentProcess(); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool ReadProcessMemory( + nint hProcess, + nint lpBaseAddress, + byte[] lpBuffer, + int dwSize, + out nint lpNumberOfBytesRead); + + private static readonly nint _proc = GetCurrentProcess(); + + public static bool TryReadIntPtr(nint address, out nint value) + { + value = nint.Zero; + + if (address == nint.Zero) + return false; + + if ((ulong)address < 0x10000UL) + return false; + + var buf = new byte[IntPtr.Size]; + if (!ReadProcessMemory(_proc, address, buf, buf.Length, out var read) || read != (nint)buf.Length) + return false; + + value = IntPtr.Size == 8 + ? (nint)BitConverter.ToInt64(buf, 0) + : (nint)BitConverter.ToInt32(buf, 0); + + return true; + } +} \ No newline at end of file diff --git a/LightlessSync/Utils/VariousExtensions.cs b/LightlessSync/Utils/VariousExtensions.cs index d250279..3e0545c 100644 --- a/LightlessSync/Utils/VariousExtensions.cs +++ b/LightlessSync/Utils/VariousExtensions.cs @@ -3,8 +3,6 @@ using LightlessSync.API.Data; using LightlessSync.API.Data.Enum; using LightlessSync.PlayerData.Pairs; using Microsoft.Extensions.Logging; -using System.Collections.Generic; -using System.Linq; using System.Text.Json; namespace LightlessSync.Utils; @@ -56,164 +54,168 @@ public static class VariousExtensions return new CancellationTokenSource(); } - public static Dictionary> CheckUpdatedData(this CharacterData newData, Guid applicationBase, - CharacterData? oldData, ILogger logger, IPairPerformanceSubject cachedPlayer, bool forceApplyCustomization, bool forceApplyMods, + public static Dictionary> CheckUpdatedData( + this CharacterData newData, + Guid applicationBase, + CharacterData? oldData, + ILogger logger, + IPairPerformanceSubject cachedPlayer, + bool forceApplyCustomization, + bool forceApplyMods, bool suppressForcedRedrawOnForcedModApply = false) { oldData ??= new(); + static bool HasFiles(List? list) => list is { Count: > 0 }; + static bool HasText(string? s) => !string.IsNullOrEmpty(s); + static string Norm(string? s) => s ?? string.Empty; + + var forceRedrawOnForcedApply = forceApplyMods && !suppressForcedRedrawOnForcedModApply; + var charaDataToUpdate = new Dictionary>(); + foreach (ObjectKind objectKind in Enum.GetValues()) { - charaDataToUpdate[objectKind] = []; - oldData.FileReplacements.TryGetValue(objectKind, out var existingFileReplacements); - newData.FileReplacements.TryGetValue(objectKind, out var newFileReplacements); - oldData.GlamourerData.TryGetValue(objectKind, out var existingGlamourerData); - newData.GlamourerData.TryGetValue(objectKind, out var newGlamourerData); + var set = new HashSet(); - bool hasNewButNotOldFileReplacements = newFileReplacements != null && existingFileReplacements == null; - bool hasOldButNotNewFileReplacements = existingFileReplacements != null && newFileReplacements == null; + oldData.FileReplacements.TryGetValue(objectKind, out var oldFileRepls); + newData.FileReplacements.TryGetValue(objectKind, out var newFileRepls); - bool hasNewButNotOldGlamourerData = newGlamourerData != null && existingGlamourerData == null; - bool hasOldButNotNewGlamourerData = existingGlamourerData != null && newGlamourerData == null; + oldData.GlamourerData.TryGetValue(objectKind, out var oldGlam); + newData.GlamourerData.TryGetValue(objectKind, out var newGlam); - bool hasNewAndOldFileReplacements = newFileReplacements != null && existingFileReplacements != null; - bool hasNewAndOldGlamourerData = newGlamourerData != null && existingGlamourerData != null; - var forceRedrawOnForcedApply = forceApplyMods && !suppressForcedRedrawOnForcedModApply; + var oldHasFiles = HasFiles(oldFileRepls); + var newHasFiles = HasFiles(newFileRepls); - if (hasNewButNotOldFileReplacements || hasOldButNotNewFileReplacements || hasNewButNotOldGlamourerData || hasOldButNotNewGlamourerData) + if (oldHasFiles != newHasFiles) { - logger.LogDebug("[BASE-{appBase}] Updating {object}/{kind} (Some new data arrived: NewButNotOldFiles:{hasNewButNotOldFileReplacements}," + - " OldButNotNewFiles:{hasOldButNotNewFileReplacements}, NewButNotOldGlam:{hasNewButNotOldGlamourerData}, OldButNotNewGlam:{hasOldButNotNewGlamourerData}) => {change}, {change2}", - applicationBase, - cachedPlayer, objectKind, hasNewButNotOldFileReplacements, hasOldButNotNewFileReplacements, hasNewButNotOldGlamourerData, hasOldButNotNewGlamourerData, PlayerChanges.ModFiles, PlayerChanges.Glamourer); - charaDataToUpdate[objectKind].Add(PlayerChanges.ModFiles); - charaDataToUpdate[objectKind].Add(PlayerChanges.Glamourer); - charaDataToUpdate[objectKind].Add(PlayerChanges.ForcedRedraw); - } - else - { - if (hasNewAndOldFileReplacements) + logger.LogDebug("[BASE-{appBase}] Updating {object}/{kind} (File presence changed old={old} new={new}) => {change}", + applicationBase, cachedPlayer, objectKind, oldHasFiles, newHasFiles, PlayerChanges.ModFiles); + + set.Add(PlayerChanges.ModFiles); + if (objectKind != ObjectKind.Player || forceRedrawOnForcedApply) { - var oldList = oldData.FileReplacements[objectKind]; - var newList = newData.FileReplacements[objectKind]; - var listsAreEqual = oldList.SequenceEqual(newList, PlayerData.Data.FileReplacementDataComparer.Instance); - if (!listsAreEqual || forceApplyMods) - { - logger.LogDebug("[BASE-{appBase}] Updating {object}/{kind} (FileReplacements not equal) => {change}", applicationBase, cachedPlayer, objectKind, PlayerChanges.ModFiles); - charaDataToUpdate[objectKind].Add(PlayerChanges.ModFiles); - if (objectKind != ObjectKind.Player || forceRedrawOnForcedApply) - { - charaDataToUpdate[objectKind].Add(PlayerChanges.ForcedRedraw); - } - else - { - var existingFace = existingFileReplacements.Where(g => g.GamePaths.Any(p => p.Contains("/face/", StringComparison.OrdinalIgnoreCase))) - .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); - var existingHair = existingFileReplacements.Where(g => g.GamePaths.Any(p => p.Contains("/hair/", StringComparison.OrdinalIgnoreCase))) - .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); - var existingTail = existingFileReplacements.Where(g => g.GamePaths.Any(p => p.Contains("/tail/", StringComparison.OrdinalIgnoreCase))) - .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); - var newFace = newFileReplacements.Where(g => g.GamePaths.Any(p => p.Contains("/face/", StringComparison.OrdinalIgnoreCase))) - .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); - var newHair = newFileReplacements.Where(g => g.GamePaths.Any(p => p.Contains("/hair/", StringComparison.OrdinalIgnoreCase))) - .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); - var newTail = newFileReplacements.Where(g => g.GamePaths.Any(p => p.Contains("/tail/", StringComparison.OrdinalIgnoreCase))) - .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); - var existingTransients = existingFileReplacements.Where(g => g.GamePaths.Any(g => !g.EndsWith("mdl") && !g.EndsWith("tex") && !g.EndsWith("mtrl"))) - .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); - var newTransients = newFileReplacements.Where(g => g.GamePaths.Any(g => !g.EndsWith("mdl") && !g.EndsWith("tex") && !g.EndsWith("mtrl"))) - .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); - - logger.LogTrace("[BASE-{appbase}] ExistingFace: {of}, NewFace: {fc}; ExistingHair: {eh}, NewHair: {nh}; ExistingTail: {et}, NewTail: {nt}; ExistingTransient: {etr}, NewTransient: {ntr}", applicationBase, - existingFace.Count, newFace.Count, existingHair.Count, newHair.Count, existingTail.Count, newTail.Count, existingTransients.Count, newTransients.Count); - - var differentFace = !existingFace.SequenceEqual(newFace, PlayerData.Data.FileReplacementDataComparer.Instance); - var differentHair = !existingHair.SequenceEqual(newHair, PlayerData.Data.FileReplacementDataComparer.Instance); - var differentTail = !existingTail.SequenceEqual(newTail, PlayerData.Data.FileReplacementDataComparer.Instance); - var differenTransients = !existingTransients.SequenceEqual(newTransients, PlayerData.Data.FileReplacementDataComparer.Instance); - if (differentFace || differentHair || differentTail || differenTransients) - { - logger.LogDebug("[BASE-{appbase}] Different Subparts: Face: {face}, Hair: {hair}, Tail: {tail}, Transients: {transients} => {change}", applicationBase, - differentFace, differentHair, differentTail, differenTransients, PlayerChanges.ForcedRedraw); - charaDataToUpdate[objectKind].Add(PlayerChanges.ForcedRedraw); - } - } - } + set.Add(PlayerChanges.ForcedRedraw); } + } + else if (newHasFiles) + { + var listsAreEqual = oldFileRepls!.SequenceEqual(newFileRepls!, PlayerData.Data.FileReplacementDataComparer.Instance); - if (hasNewAndOldGlamourerData) + if (!listsAreEqual || forceApplyMods) { - bool glamourerDataDifferent = !string.Equals(oldData.GlamourerData[objectKind], newData.GlamourerData[objectKind], StringComparison.Ordinal); - if (glamourerDataDifferent || forceApplyCustomization) + logger.LogDebug("[BASE-{appBase}] Updating {object}/{kind} (FileReplacements changed or forceApplyMods) => {change}", + applicationBase, cachedPlayer, objectKind, PlayerChanges.ModFiles); + + set.Add(PlayerChanges.ModFiles); + + if (objectKind != ObjectKind.Player || forceRedrawOnForcedApply) { - logger.LogDebug("[BASE-{appBase}] Updating {object}/{kind} (Glamourer different) => {change}", applicationBase, cachedPlayer, objectKind, PlayerChanges.Glamourer); - charaDataToUpdate[objectKind].Add(PlayerChanges.Glamourer); + set.Add(PlayerChanges.ForcedRedraw); + } + else + { + var existingFace = oldFileRepls.Where(g => g.GamePaths.Any(p => p.Contains("/face/", StringComparison.OrdinalIgnoreCase))) + .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); + var existingHair = oldFileRepls.Where(g => g.GamePaths.Any(p => p.Contains("/hair/", StringComparison.OrdinalIgnoreCase))) + .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); + var existingTail = oldFileRepls.Where(g => g.GamePaths.Any(p => p.Contains("/tail/", StringComparison.OrdinalIgnoreCase))) + .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); + + var newFace = newFileRepls!.Where(g => g.GamePaths.Any(p => p.Contains("/face/", StringComparison.OrdinalIgnoreCase))) + .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); + var newHair = newFileRepls.Where(g => g.GamePaths.Any(p => p.Contains("/hair/", StringComparison.OrdinalIgnoreCase))) + .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); + var newTail = newFileRepls.Where(g => g.GamePaths.Any(p => p.Contains("/tail/", StringComparison.OrdinalIgnoreCase))) + .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); + + var existingTransients = oldFileRepls.Where(g => g.GamePaths.Any(p => !p.EndsWith("mdl", StringComparison.OrdinalIgnoreCase) + && !p.EndsWith("tex", StringComparison.OrdinalIgnoreCase) + && !p.EndsWith("mtrl", StringComparison.OrdinalIgnoreCase))) + .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); + + var newTransients = newFileRepls.Where(g => g.GamePaths.Any(p => !p.EndsWith("mdl", StringComparison.OrdinalIgnoreCase) + && !p.EndsWith("tex", StringComparison.OrdinalIgnoreCase) + && !p.EndsWith("mtrl", StringComparison.OrdinalIgnoreCase))) + .OrderBy(g => string.IsNullOrEmpty(g.Hash) ? g.FileSwapPath : g.Hash, StringComparer.OrdinalIgnoreCase).ToList(); + + var differentFace = !existingFace.SequenceEqual(newFace, PlayerData.Data.FileReplacementDataComparer.Instance); + var differentHair = !existingHair.SequenceEqual(newHair, PlayerData.Data.FileReplacementDataComparer.Instance); + var differentTail = !existingTail.SequenceEqual(newTail, PlayerData.Data.FileReplacementDataComparer.Instance); + var differentTransients = !existingTransients.SequenceEqual(newTransients, PlayerData.Data.FileReplacementDataComparer.Instance); + + if (differentFace || differentHair || differentTail || differentTransients) + set.Add(PlayerChanges.ForcedRedraw); } } } - oldData.CustomizePlusData.TryGetValue(objectKind, out var oldCustomizePlusData); - newData.CustomizePlusData.TryGetValue(objectKind, out var newCustomizePlusData); + var oldGlamNorm = Norm(oldGlam); + var newGlamNorm = Norm(newGlam); - oldCustomizePlusData ??= string.Empty; - newCustomizePlusData ??= string.Empty; - - bool customizeDataDifferent = !string.Equals(oldCustomizePlusData, newCustomizePlusData, StringComparison.Ordinal); - if (customizeDataDifferent || (forceApplyCustomization && !string.IsNullOrEmpty(newCustomizePlusData))) + if (!string.Equals(oldGlamNorm, newGlamNorm, StringComparison.Ordinal) + || (forceApplyCustomization && HasText(newGlamNorm))) { - logger.LogDebug("[BASE-{appBase}] Updating {object}/{kind} (Diff customize data) => {change}", applicationBase, cachedPlayer, objectKind, PlayerChanges.Customize); - charaDataToUpdate[objectKind].Add(PlayerChanges.Customize); + logger.LogDebug("[BASE-{appBase}] Updating {object}/{kind} (Glamourer different) => {change}", + applicationBase, cachedPlayer, objectKind, PlayerChanges.Glamourer); + set.Add(PlayerChanges.Glamourer); } - if (objectKind != ObjectKind.Player) continue; + oldData.CustomizePlusData.TryGetValue(objectKind, out var oldC); + newData.CustomizePlusData.TryGetValue(objectKind, out var newC); - bool manipDataDifferent = !string.Equals(oldData.ManipulationData, newData.ManipulationData, StringComparison.Ordinal); - if (manipDataDifferent || forceRedrawOnForcedApply) + var oldCNorm = Norm(oldC); + var newCNorm = Norm(newC); + + if (!string.Equals(oldCNorm, newCNorm, StringComparison.Ordinal) + || (forceApplyCustomization && HasText(newCNorm))) { - logger.LogDebug("[BASE-{appBase}] Updating {object}/{kind} (Diff manip data) => {change}", applicationBase, cachedPlayer, objectKind, PlayerChanges.ModManip); - charaDataToUpdate[objectKind].Add(PlayerChanges.ModManip); - charaDataToUpdate[objectKind].Add(PlayerChanges.ForcedRedraw); + logger.LogDebug("[BASE-{appBase}] Updating {object}/{kind} (Customize+ different) => {change}", + applicationBase, cachedPlayer, objectKind, PlayerChanges.Customize); + set.Add(PlayerChanges.Customize); } - bool heelsOffsetDifferent = !string.Equals(oldData.HeelsData, newData.HeelsData, StringComparison.Ordinal); - if (heelsOffsetDifferent || (forceApplyCustomization && !string.IsNullOrEmpty(newData.HeelsData))) + if (objectKind == ObjectKind.Player) { - logger.LogDebug("[BASE-{appBase}] Updating {object}/{kind} (Diff heels data) => {change}", applicationBase, cachedPlayer, objectKind, PlayerChanges.Heels); - charaDataToUpdate[objectKind].Add(PlayerChanges.Heels); + var oldManip = Norm(oldData.ManipulationData); + var newManip = Norm(newData.ManipulationData); + + if (!string.Equals(oldManip, newManip, StringComparison.Ordinal) || forceRedrawOnForcedApply) + { + logger.LogDebug("[BASE-{appBase}] Updating {object}/{kind} (Manip different) => {change}", + applicationBase, cachedPlayer, objectKind, PlayerChanges.ModManip); + set.Add(PlayerChanges.ModManip); + set.Add(PlayerChanges.ForcedRedraw); + } + + if (!string.Equals(Norm(oldData.HeelsData), Norm(newData.HeelsData), StringComparison.Ordinal) + || (forceApplyCustomization && HasText(newData.HeelsData))) + set.Add(PlayerChanges.Heels); + + if (!string.Equals(Norm(oldData.HonorificData), Norm(newData.HonorificData), StringComparison.Ordinal) + || (forceApplyCustomization && HasText(newData.HonorificData))) + set.Add(PlayerChanges.Honorific); + + if (!string.Equals(Norm(oldData.MoodlesData), Norm(newData.MoodlesData), StringComparison.Ordinal) + || (forceApplyCustomization && HasText(newData.MoodlesData))) + set.Add(PlayerChanges.Moodles); + + if (!string.Equals(Norm(oldData.PetNamesData), Norm(newData.PetNamesData), StringComparison.Ordinal) + || (forceApplyCustomization && HasText(newData.PetNamesData))) + set.Add(PlayerChanges.PetNames); } - bool honorificDataDifferent = !string.Equals(oldData.HonorificData, newData.HonorificData, StringComparison.Ordinal); - if (honorificDataDifferent || (forceApplyCustomization && !string.IsNullOrEmpty(newData.HonorificData))) - { - logger.LogDebug("[BASE-{appBase}] Updating {object}/{kind} (Diff honorific data) => {change}", applicationBase, cachedPlayer, objectKind, PlayerChanges.Honorific); - charaDataToUpdate[objectKind].Add(PlayerChanges.Honorific); - } - - bool moodlesDataDifferent = !string.Equals(oldData.MoodlesData, newData.MoodlesData, StringComparison.Ordinal); - if (moodlesDataDifferent || (forceApplyCustomization && !string.IsNullOrEmpty(newData.MoodlesData))) - { - logger.LogDebug("[BASE-{appBase}] Updating {object}/{kind} (Diff moodles data) => {change}", applicationBase, cachedPlayer, objectKind, PlayerChanges.Moodles); - charaDataToUpdate[objectKind].Add(PlayerChanges.Moodles); - } - - bool petNamesDataDifferent = !string.Equals(oldData.PetNamesData, newData.PetNamesData, StringComparison.Ordinal); - if (petNamesDataDifferent || (forceApplyCustomization && !string.IsNullOrEmpty(newData.PetNamesData))) - { - logger.LogDebug("[BASE-{appBase}] Updating {object}/{kind} (Diff petnames data) => {change}", applicationBase, cachedPlayer, objectKind, PlayerChanges.PetNames); - charaDataToUpdate[objectKind].Add(PlayerChanges.PetNames); - } + if (set.Count > 0) + charaDataToUpdate[objectKind] = set; } - foreach (KeyValuePair> data in charaDataToUpdate.ToList()) - { - if (!data.Value.Any()) charaDataToUpdate.Remove(data.Key); - else charaDataToUpdate[data.Key] = [.. data.Value.OrderByDescending(p => (int)p)]; - } + foreach (var k in charaDataToUpdate.Keys.ToList()) + charaDataToUpdate[k] = [.. charaDataToUpdate[k].OrderBy(p => (int)p)]; return charaDataToUpdate; } + public static T DeepClone(this T obj) { return JsonSerializer.Deserialize(JsonSerializer.Serialize(obj))!; diff --git a/LightlessSync/WebAPI/Files/FileDownloadManager.cs b/LightlessSync/WebAPI/Files/FileDownloadManager.cs index 97f8af7..d86f8f7 100644 --- a/LightlessSync/WebAPI/Files/FileDownloadManager.cs +++ b/LightlessSync/WebAPI/Files/FileDownloadManager.cs @@ -436,11 +436,9 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase Logger.LogDebug("GUID {requestId} on server {uri} for files {files}", requestId, transfers[0].DownloadUri, string.Join(", ", transfers.Select(c => c.Hash))); - // Wait for ready WITHOUT holding a slot SetStatus(statusKey, DownloadStatus.WaitingForQueue); await WaitForDownloadReady(transfers, requestId, ct).ConfigureAwait(false); - // Hold slot ONLY for the GET SetStatus(statusKey, DownloadStatus.WaitingForSlot); await using ((await AcquireSlotAsync(ct).ConfigureAwait(false)).ConfigureAwait(false)) { @@ -462,7 +460,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase bool skipDecimation) { SetStatus(downloadStatusKey, DownloadStatus.Decompressing); - MarkTransferredFiles(downloadStatusKey, 1); + + var extracted = 0; try { @@ -471,6 +470,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase { while (fileBlockStream.Position < fileBlockStream.Length) { + ct.ThrowIfCancellationRequested(); + (string fileHash, long fileLengthBytes) = ReadBlockFileHeader(fileBlockStream); try @@ -480,72 +481,69 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase var len = checked((int)fileLengthBytes); + if (fileBlockStream.Position + len > fileBlockStream.Length) + throw new EndOfStreamException(); + if (!replacementLookup.TryGetValue(fileHash, out var repl)) { - Logger.LogWarning("{dlName}: No replacement mapping for {fileHash}", downloadLabel, fileHash); - // still need to skip bytes: - var skip = checked((int)fileLengthBytes); - fileBlockStream.Position += skip; + Logger.LogWarning("{dlName}: No replacement mapping for {fileHash}, skipping {len} bytes", + downloadLabel, fileHash, len); + + fileBlockStream.Seek(len, SeekOrigin.Current); continue; } var filePath = _fileDbManager.GetCacheFilePath(fileHash, repl.Extension); - Logger.LogTrace("{dlName}: Decompressing {file}:{len} => {dest}", downloadLabel, fileHash, fileLengthBytes, filePath); + Logger.LogTrace("{dlName}: Extracting {fileHash}:{len} => {dest}", + downloadLabel, fileHash, len, filePath); var compressed = new byte[len]; - await ReadExactlyAsync(fileBlockStream, compressed.AsMemory(0, len), ct).ConfigureAwait(false); - MungeBuffer(compressed); - var decompressed = LZ4Wrapper.Unwrap(compressed); - - if (rawSizeLookup.TryGetValue(fileHash, out var expectedRawSize) - && expectedRawSize > 0 - && decompressed.LongLength != expectedRawSize) - { - await _fileCompactor.WriteAllBytesAsync(filePath, Array.Empty(), ct).ConfigureAwait(false); - PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale, skipDecimation); - continue; - } - MungeBuffer(compressed); await _decompressGate.WaitAsync(ct).ConfigureAwait(false); + byte[] decompressed; try { - // offload CPU-intensive decompression to threadpool to free up worker - await Task.Run(async () => - { - var sw = System.Diagnostics.Stopwatch.StartNew(); - - // decompress - var decompressed = LZ4Wrapper.Unwrap(compressed); - - Logger.LogTrace("{dlName}: Unwrap {fileHash} took {ms}ms (compressed {c} bytes, decompressed {d} bytes)", - downloadLabel, fileHash, sw.ElapsedMilliseconds, compressed.Length, decompressed?.Length ?? -1); - - // write to file without compacting during download - await File.WriteAllBytesAsync(filePath, decompressed, ct).ConfigureAwait(false); - PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale, skipDecimation); - }, ct).ConfigureAwait(false); + decompressed = await Task.Run(() => LZ4Wrapper.Unwrap(compressed), ct).ConfigureAwait(false); } finally { _decompressGate.Release(); } + + if (rawSizeLookup.TryGetValue(fileHash, out var expectedRawSize) + && expectedRawSize > 0 + && decompressed.LongLength != expectedRawSize) + { + Logger.LogWarning( + "{dlName}: Size mismatch for {fileHash} (expected {expected}, got {actual}). Treating as corrupt.", + downloadLabel, fileHash, expectedRawSize, decompressed.LongLength); + + try { if (File.Exists(filePath)) File.Delete(filePath); } catch { /* ignore */ } + continue; + } + + await _fileCompactor.WriteAllBytesAsync(filePath, decompressed, ct, enqueueCompaction: false).ConfigureAwait(false); + PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale, skipDecimation); + + extracted++; + MarkTransferredFiles(downloadStatusKey, extracted); } catch (EndOfStreamException) { - Logger.LogWarning("{dlName}: Failure to extract file {fileHash}, stream ended prematurely", downloadLabel, fileHash); + Logger.LogWarning("{dlName}: Block ended mid-entry while extracting {fileHash}", downloadLabel, fileHash); + break; } - catch (Exception e) + catch (Exception ex) { - Logger.LogWarning(e, "{dlName}: Error during decompression", downloadLabel); + Logger.LogWarning(ex, "{dlName}: Error extracting {fileHash} from block", downloadLabel, fileHash); } } - } - SetStatus(downloadStatusKey, DownloadStatus.Completed); + SetStatus(downloadStatusKey, DownloadStatus.Completed); + } } catch (EndOfStreamException) { @@ -601,11 +599,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase _orchestrator.ForbiddenTransfers.Add(new DownloadFileTransfer(dto)); } - CurrentDownloads = downloadFileInfoFromService + CurrentDownloads = [.. downloadFileInfoFromService .Distinct() .Select(d => new DownloadFileTransfer(d)) - .Where(d => d.CanBeTransferred) - .ToList(); + .Where(d => d.CanBeTransferred)]; return CurrentDownloads; } @@ -1033,48 +1030,58 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase private async Task ProcessDeferredCompressionsAsync(CancellationToken ct) { - if (_deferredCompressionQueue.IsEmpty) + if (_deferredCompressionQueue.IsEmpty || !_configService.Current.UseCompactor) return; - var filesToCompress = new List(); + // Drain queue into a unique set (same file can be enqueued multiple times) + var filesToCompact = new HashSet(StringComparer.OrdinalIgnoreCase); while (_deferredCompressionQueue.TryDequeue(out var filePath)) { - if (File.Exists(filePath)) - filesToCompress.Add(filePath); + if (!string.IsNullOrWhiteSpace(filePath)) + filesToCompact.Add(filePath); } - if (filesToCompress.Count == 0) + if (filesToCompact.Count == 0) return; - Logger.LogDebug("Starting deferred compression of {count} files", filesToCompress.Count); + Logger.LogDebug("Starting deferred compaction of {count} files", filesToCompact.Count); - var compressionWorkers = Math.Clamp(Environment.ProcessorCount / 4, 2, 4); + var enqueueWorkers = Math.Clamp(Environment.ProcessorCount / 4, 1, 2); - await Parallel.ForEachAsync(filesToCompress, - new ParallelOptions - { - MaxDegreeOfParallelism = compressionWorkers, - CancellationToken = ct + await Parallel.ForEachAsync( + filesToCompact, + new ParallelOptions + { + MaxDegreeOfParallelism = enqueueWorkers, + CancellationToken = ct }, async (filePath, token) => { - try - { + try + { + token.ThrowIfCancellationRequested(); + + if (!File.Exists(filePath)) + return; + await Task.Yield(); - if (_configService.Current.UseCompactor && File.Exists(filePath)) - { - var bytes = await File.ReadAllBytesAsync(filePath, token).ConfigureAwait(false); - await _fileCompactor.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false); - Logger.LogTrace("Compressed file: {filePath}", filePath); - } + + _fileCompactor.RequestCompaction(filePath); + + Logger.LogTrace("Deferred compaction queued: {filePath}", filePath); + } + catch (OperationCanceledException) + { + Logger.LogTrace("Deferred compaction cancelled for file: {filePath}", filePath); + throw; } catch (Exception ex) { - Logger.LogWarning(ex, "Failed to compress file: {filePath}", filePath); + Logger.LogWarning(ex, "Failed to queue deferred compaction for file: {filePath}", filePath); } }).ConfigureAwait(false); - Logger.LogDebug("Completed deferred compression of {count} files", filesToCompress.Count); + Logger.LogDebug("Completed queuing deferred compaction of {count} files", filesToCompact.Count); } private sealed class InlineProgress : IProgress