From d6fe09ba8ecb959c1917297ea7cbb07e64d05038 Mon Sep 17 00:00:00 2001 From: cake Date: Fri, 2 Jan 2026 03:56:59 +0100 Subject: [PATCH 01/13] Testing PAP handling changes. --- LightlessSync/FileCache/FileCacheManager.cs | 96 ++++++- .../PlayerData/Factories/PlayerDataFactory.cs | 195 ++++++++++---- .../PlayerData/Pairs/PairHandlerAdapter.cs | 236 ++++++++++++++++- .../Pairs/PairHandlerAdapterFactory.cs | 8 +- LightlessSync/Services/XivDataAnalyzer.cs | 246 +++++++++++++++--- LightlessSync/UI/DownloadUi.cs | 2 +- .../WebAPI/Files/FileDownloadManager.cs | 94 ++----- 7 files changed, 708 insertions(+), 169 deletions(-) diff --git a/LightlessSync/FileCache/FileCacheManager.cs b/LightlessSync/FileCache/FileCacheManager.cs index b0becf3..5fe044c 100644 --- a/LightlessSync/FileCache/FileCacheManager.cs +++ b/LightlessSync/FileCache/FileCacheManager.cs @@ -27,6 +27,7 @@ public sealed class FileCacheManager : IHostedService private readonly ConcurrentDictionary> _fileCaches = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _fileCachesByPrefixedPath = new(StringComparer.OrdinalIgnoreCase); private readonly SemaphoreSlim _getCachesByPathsSemaphore = new(1, 1); + private readonly SemaphoreSlim _evictSemaphore = new(1, 1); private readonly Lock _fileWriteLock = new(); private readonly IpcManager _ipcManager; private readonly ILogger _logger; @@ -226,13 +227,23 @@ public sealed class FileCacheManager : IHostedService var compressed = LZ4Wrapper.WrapHC(raw, 0, raw.Length); var tmpPath = compressedPath + ".tmp"; - await File.WriteAllBytesAsync(tmpPath, compressed, token).ConfigureAwait(false); - File.Move(tmpPath, compressedPath, overwrite: true); + try + { + await File.WriteAllBytesAsync(tmpPath, compressed, token).ConfigureAwait(false); + File.Move(tmpPath, compressedPath, overwrite: true); + } + finally + { + try { if (File.Exists(tmpPath)) File.Delete(tmpPath); } catch { /* ignore */ } + } - var compressedSize = compressed.LongLength; + var compressedSize = new FileInfo(compressedPath).Length; SetSizeInfo(hash, originalSize, compressedSize); UpdateEntitiesSizes(hash, originalSize, compressedSize); + var maxBytes = GiBToBytes(_configService.Current.MaxLocalCacheInGiB); + await EnforceCacheLimitAsync(maxBytes, token).ConfigureAwait(false); + return compressed; } finally @@ -877,6 +888,83 @@ public sealed class FileCacheManager : IHostedService }, token).ConfigureAwait(false); } + private async Task EnforceCacheLimitAsync(long maxBytes, CancellationToken token) + { + if (string.IsNullOrWhiteSpace(CacheFolder) || maxBytes <= 0) return; + + await _evictSemaphore.WaitAsync(token).ConfigureAwait(false); + try + { + Directory.CreateDirectory(CacheFolder); + + foreach (var tmp in Directory.EnumerateFiles(CacheFolder, "*" + _compressedCacheExtension + ".tmp")) + { + try { File.Delete(tmp); } catch { /* ignore */ } + } + + var files = Directory.EnumerateFiles(CacheFolder, "*" + _compressedCacheExtension, SearchOption.TopDirectoryOnly) + .Select(p => new FileInfo(p)) + .Where(fi => fi.Exists) + .OrderBy(fi => fi.LastWriteTimeUtc) + .ToList(); + + long total = files.Sum(f => f.Length); + if (total <= maxBytes) return; + + foreach (var fi in files) + { + token.ThrowIfCancellationRequested(); + if (total <= maxBytes) break; + + var hash = Path.GetFileNameWithoutExtension(fi.Name); + + try + { + var len = fi.Length; + fi.Delete(); + total -= len; + _sizeCache.TryRemove(hash, out _); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to evict cache file {file}", fi.FullName); + } + } + } + finally + { + _evictSemaphore.Release(); + } + } + + private static long GiBToBytes(double gib) + { + if (double.IsNaN(gib) || double.IsInfinity(gib) || gib <= 0) + return 0; + + var bytes = gib * 1024d * 1024d * 1024d; + + if (bytes >= long.MaxValue) return long.MaxValue; + + return (long)Math.Round(bytes, MidpointRounding.AwayFromZero); + } + + private void CleanupOrphanCompressedCache() + { + if (string.IsNullOrWhiteSpace(CacheFolder) || !Directory.Exists(CacheFolder)) + return; + + foreach (var path in Directory.EnumerateFiles(CacheFolder, "*" + _compressedCacheExtension)) + { + var hash = Path.GetFileNameWithoutExtension(path); + if (!_fileCaches.ContainsKey(hash)) + { + try { File.Delete(path); } + catch (Exception ex) { _logger.LogWarning(ex, "Failed deleting orphan {file}", path); } + } + } + } + public async Task StartAsync(CancellationToken cancellationToken) { _logger.LogInformation("Starting FileCacheManager"); @@ -1060,6 +1148,8 @@ public sealed class FileCacheManager : IHostedService { await WriteOutFullCsvAsync(cancellationToken).ConfigureAwait(false); } + + CleanupOrphanCompressedCache(); } _logger.LogInformation("Started FileCacheManager"); diff --git a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs index 9ecfcc3..4a20467 100644 --- a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs +++ b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs @@ -21,6 +21,7 @@ public class PlayerDataFactory private readonly XivDataAnalyzer _modelAnalyzer; private readonly LightlessMediator _lightlessMediator; private readonly TransientResourceManager _transientResourceManager; + private static readonly SemaphoreSlim _papParseLimiter = new(1, 1); public PlayerDataFactory(ILogger logger, DalamudUtilService dalamudUtil, IpcManager ipcManager, TransientResourceManager transientResourceManager, FileCacheManager fileReplacementFactory, @@ -121,7 +122,6 @@ public class PlayerDataFactory _logger.LogDebug("Building character data for {obj}", playerRelatedObject); var logDebug = _logger.IsEnabled(LogLevel.Debug); - // wait until chara is not drawing and present so nothing spontaneously explodes await _dalamudUtil.WaitWhileCharacterIsDrawing(_logger, playerRelatedObject, Guid.NewGuid(), 30000, ct: ct).ConfigureAwait(false); int totalWaitTime = 10000; while (!await _dalamudUtil.IsObjectPresentAsync(await _dalamudUtil.CreateGameObjectAsync(playerRelatedObject.Address).ConfigureAwait(false)).ConfigureAwait(false) && totalWaitTime > 0) @@ -135,7 +135,6 @@ public class PlayerDataFactory DateTime start = DateTime.UtcNow; - // penumbra call, it's currently broken Dictionary>? resolvedPaths; resolvedPaths = (await _ipcManager.Penumbra.GetCharacterData(_logger, playerRelatedObject).ConfigureAwait(false)); @@ -144,8 +143,7 @@ public class PlayerDataFactory ct.ThrowIfCancellationRequested(); fragment.FileReplacements = - new HashSet(resolvedPaths.Select(c => new FileReplacement([.. c.Value], c.Key)), FileReplacementComparer.Instance) - .Where(p => p.HasFileReplacement).ToHashSet(); + [.. new HashSet(resolvedPaths.Select(c => new FileReplacement([.. c.Value], c.Key)), FileReplacementComparer.Instance).Where(p => p.HasFileReplacement)]; fragment.FileReplacements.RemoveWhere(c => c.GamePaths.Any(g => !CacheMonitor.AllowedFileExtensions.Any(e => g.EndsWith(e, StringComparison.OrdinalIgnoreCase)))); ct.ThrowIfCancellationRequested(); @@ -169,8 +167,6 @@ public class PlayerDataFactory await _transientResourceManager.WaitForRecording(ct).ConfigureAwait(false); - // if it's pet then it's summoner, if it's summoner we actually want to keep all filereplacements alive at all times - // or we get into redraw city for every change and nothing works properly if (objectKind == ObjectKind.Pet) { foreach (var item in fragment.FileReplacements.Where(i => i.HasFileReplacement).SelectMany(p => p.GamePaths)) @@ -189,10 +185,8 @@ public class PlayerDataFactory _logger.LogDebug("Handling transient update for {obj}", playerRelatedObject); - // remove all potentially gathered paths from the transient resource manager that are resolved through static resolving - _transientResourceManager.ClearTransientPaths(objectKind, fragment.FileReplacements.SelectMany(c => c.GamePaths).ToList()); + _transientResourceManager.ClearTransientPaths(objectKind, [.. fragment.FileReplacements.SelectMany(c => c.GamePaths)]); - // get all remaining paths and resolve them var transientPaths = ManageSemiTransientData(objectKind); var resolvedTransientPaths = await GetFileReplacementsFromPaths(playerRelatedObject, transientPaths, new HashSet(StringComparer.Ordinal)).ConfigureAwait(false); @@ -213,12 +207,10 @@ public class PlayerDataFactory } } - // clean up all semi transient resources that don't have any file replacement (aka null resolve) _transientResourceManager.CleanUpSemiTransientResources(objectKind, [.. fragment.FileReplacements]); ct.ThrowIfCancellationRequested(); - // make sure we only return data that actually has file replacements fragment.FileReplacements = new HashSet(fragment.FileReplacements.Where(v => v.HasFileReplacement).OrderBy(v => v.ResolvedPath, StringComparer.Ordinal), FileReplacementComparer.Instance); // gather up data from ipc @@ -270,13 +262,17 @@ public class PlayerDataFactory Dictionary>? boneIndices = null; var hasPapFiles = false; + if (objectKind == ObjectKind.Player) { hasPapFiles = fragment.FileReplacements.Any(f => !f.IsFileSwap && f.GamePaths.First().EndsWith("pap", StringComparison.OrdinalIgnoreCase)); + if (hasPapFiles) { - boneIndices = await _dalamudUtil.RunOnFrameworkThread(() => _modelAnalyzer.GetSkeletonBoneIndices(playerRelatedObject)).ConfigureAwait(false); + boneIndices = await _dalamudUtil + .RunOnFrameworkThread(() => _modelAnalyzer.GetSkeletonBoneIndices(playerRelatedObject)) + .ConfigureAwait(false); } } @@ -284,9 +280,16 @@ public class PlayerDataFactory { try { +#if DEBUG + if (hasPapFiles && boneIndices != null) + { + _modelAnalyzer.DumpLocalSkeletonIndices(playerRelatedObject); + } +#endif if (hasPapFiles) { - await VerifyPlayerAnimationBones(boneIndices, (fragment as CharacterDataFragmentPlayer)!, ct).ConfigureAwait(false); + await VerifyPlayerAnimationBones(boneIndices, (fragment as CharacterDataFragmentPlayer)!, ct) + .ConfigureAwait(false); } } catch (OperationCanceledException e) @@ -305,74 +308,174 @@ public class PlayerDataFactory return fragment; } - private async Task VerifyPlayerAnimationBones(Dictionary>? boneIndices, CharacterDataFragmentPlayer fragment, CancellationToken ct) + private async Task VerifyPlayerAnimationBones( + Dictionary>? playerBoneIndices, + CharacterDataFragmentPlayer fragment, + CancellationToken ct) { - if (boneIndices == null) return; + if (playerBoneIndices == null || playerBoneIndices.Count == 0) + return; + + var playerBoneSets = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + foreach (var (rawLocalKey, indices) in playerBoneIndices) + { + if (indices == null || indices.Count == 0) + continue; + var key = XivDataAnalyzer.CanonicalizeSkeletonKey(rawLocalKey); + if (string.IsNullOrEmpty(key)) + continue; + + if (!playerBoneSets.TryGetValue(key, out var set)) + playerBoneSets[key] = set = new HashSet(); + + foreach (var idx in indices) + set.Add(idx); + } + + if (playerBoneSets.Count == 0) + return; if (_logger.IsEnabled(LogLevel.Debug)) { - foreach (var kvp in boneIndices) + foreach (var kvp in playerBoneSets) { - _logger.LogDebug("Found {skellyname} ({idx} bone indices) on player: {bones}", kvp.Key, kvp.Value.Any() ? kvp.Value.Max() : 0, string.Join(',', kvp.Value)); + _logger.LogDebug( + "Found local skeleton bucket '{bucket}' ({count} indices, max {max})", + kvp.Key, + kvp.Value.Count, + kvp.Value.Count > 0 ? kvp.Value.Max() : 0); } } - var maxPlayerBoneIndex = boneIndices.SelectMany(kvp => kvp.Value).DefaultIfEmpty().Max(); - if (maxPlayerBoneIndex <= 0) return; + var papFiles = fragment.FileReplacements + .Where(f => !f.IsFileSwap && f.GamePaths.First().EndsWith("pap", StringComparison.OrdinalIgnoreCase)) + .ToList(); + + if (papFiles.Count == 0) + return; + + var papGroupsByHash = papFiles + .Where(f => !string.IsNullOrEmpty(f.Hash)) + .GroupBy(f => f.Hash, StringComparer.OrdinalIgnoreCase) + .ToList(); int noValidationFailed = 0; - foreach (var file in fragment.FileReplacements.Where(f => !f.IsFileSwap && f.GamePaths.First().EndsWith("pap", StringComparison.OrdinalIgnoreCase)).ToList()) + + static ushort MaxIndex(List list) + { + if (list == null || list.Count == 0) return 0; + ushort max = 0; + for (int i = 0; i < list.Count; i++) + if (list[i] > max) max = list[i]; + return max; + } + + static bool ShouldIgnorePap(Dictionary> pap) + { + foreach (var kv in pap) + { + if (kv.Value == null || kv.Value.Count == 0) + continue; + + if (MaxIndex(kv.Value) > 105) + return false; + } + return true; + } + + foreach (var group in papGroupsByHash) { ct.ThrowIfCancellationRequested(); - var skeletonIndices = await _dalamudUtil.RunOnFrameworkThread(() => _modelAnalyzer.GetBoneIndicesFromPap(file.Hash)).ConfigureAwait(false); - bool validationFailed = false; - if (skeletonIndices != null) + var hash = group.Key; + + Dictionary>? papSkeletonIndices; + + await _papParseLimiter.WaitAsync(ct).ConfigureAwait(false); + try { - // 105 is the maximum vanilla skellington spoopy bone index - if (skeletonIndices.All(k => k.Value.Max() <= 105)) - { - _logger.LogTrace("All indices of {path} are <= 105, ignoring", file.ResolvedPath); + papSkeletonIndices = await Task.Run(() => _modelAnalyzer.GetBoneIndicesFromPap(hash), ct) + .ConfigureAwait(false); + } + finally + { + _papParseLimiter.Release(); + } + + if (papSkeletonIndices == null || papSkeletonIndices.Count == 0) + continue; + + if (ShouldIgnorePap(papSkeletonIndices)) + { + _logger.LogTrace("All indices of PAP hash {hash} are <= 105, ignoring", hash); + continue; + } + + bool invalid = false; + string? reason = null; + + foreach (var (rawPapName, usedIndices) in papSkeletonIndices) + { + var papKey = XivDataAnalyzer.CanonicalizeSkeletonKey(rawPapName); + if (string.IsNullOrEmpty(papKey)) continue; + + if (!playerBoneSets.TryGetValue(papKey, out var available)) + { + invalid = true; + reason = $"Missing skeleton bucket '{papKey}' (raw '{rawPapName}') on local player."; + break; } - _logger.LogDebug("Verifying bone indices for {path}, found {x} skeletons", file.ResolvedPath, skeletonIndices.Count); - - foreach (var boneCount in skeletonIndices) + for (int i = 0; i < usedIndices.Count; i++) { - var maxAnimationIndex = boneCount.Value.DefaultIfEmpty().Max(); - if (maxAnimationIndex > maxPlayerBoneIndex) + var idx = usedIndices[i]; + if (!available.Contains(idx)) { - _logger.LogWarning("Found more bone indices on the animation {path} skeleton {skl} (max indice {idx}) than on any player related skeleton (max indice {idx2})", - file.ResolvedPath, boneCount.Key, maxAnimationIndex, maxPlayerBoneIndex); - validationFailed = true; + invalid = true; + reason = $"Skeleton '{papKey}' missing bone index {idx} (raw '{rawPapName}')."; break; } } + + if (invalid) + break; } - if (validationFailed) + if (!invalid) + continue; + + noValidationFailed++; + + _logger.LogWarning( + "Animation PAP hash {hash} is not compatible with local skeletons; dropping all mappings for this hash. Reason: {reason}", + hash, + reason); + + foreach (var file in group.ToList()) { - noValidationFailed++; _logger.LogDebug("Removing {file} from sent file replacements and transient data", file.ResolvedPath); fragment.FileReplacements.Remove(file); - foreach (var gamePath in file.GamePaths) - { - _transientResourceManager.RemoveTransientResource(ObjectKind.Player, gamePath); - } - } + foreach (var gamePath in file.GamePaths) + _transientResourceManager.RemoveTransientResource(ObjectKind.Player, gamePath); + } } if (noValidationFailed > 0) { - _lightlessMediator.Publish(new NotificationMessage("Invalid Skeleton Setup", - $"Your client is attempting to send {noValidationFailed} animation files with invalid bone data. Those animation files have been removed from your sent data. " + - $"Verify that you are using the correct skeleton for those animation files (Check /xllog for more information).", - NotificationType.Warning, TimeSpan.FromSeconds(10))); + _lightlessMediator.Publish(new NotificationMessage( + "Invalid Skeleton Setup", + $"Your client is attempting to send {noValidationFailed} animation file groups with bone indices not present on your current skeleton. " + + "Those animation files have been removed from your sent data. Verify that you are using the correct skeleton for those animations " + + "(Check /xllog for more information).", + NotificationType.Warning, + TimeSpan.FromSeconds(10))); } } + private async Task> GetFileReplacementsFromPaths(GameObjectHandler handler, HashSet forwardResolve, HashSet reverseResolve) { var forwardPaths = forwardResolve.ToArray(); diff --git a/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs b/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs index b0f2710..fc469db 100644 --- a/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs +++ b/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs @@ -22,6 +22,7 @@ using Microsoft.Extensions.Logging; using DalamudObjectKind = Dalamud.Game.ClientState.Objects.Enums.ObjectKind; using ObjectKind = LightlessSync.API.Data.Enum.ObjectKind; using FileReplacementDataComparer = LightlessSync.PlayerData.Data.FileReplacementDataComparer; +using LightlessSync.LightlessConfiguration.Models; namespace LightlessSync.PlayerData.Pairs; @@ -46,6 +47,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa private readonly TextureDownscaleService _textureDownscaleService; private readonly PairStateCache _pairStateCache; private readonly PairPerformanceMetricsCache _performanceMetricsCache; + private readonly XivDataAnalyzer _modelAnalyzer; private readonly PenumbraTempCollectionJanitor _tempCollectionJanitor; private readonly PairManager _pairManager; private CancellationTokenSource? _applicationCancellationTokenSource; @@ -90,6 +92,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa ".avfx", ".scd" }; + private readonly ConcurrentDictionary _blockedPapHashes = new(StringComparer.OrdinalIgnoreCase); private DateTime? _invisibleSinceUtc; private DateTime? _visibilityEvictionDueAtUtc; private DateTime _nextActorLookupUtc = DateTime.MinValue; @@ -184,7 +187,8 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa TextureDownscaleService textureDownscaleService, PairStateCache pairStateCache, PairPerformanceMetricsCache performanceMetricsCache, - PenumbraTempCollectionJanitor tempCollectionJanitor) : base(logger, mediator) + PenumbraTempCollectionJanitor tempCollectionJanitor, + XivDataAnalyzer modelAnalyzer) : base(logger, mediator) { _pairManager = pairManager; Ident = ident; @@ -203,6 +207,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa _pairStateCache = pairStateCache; _performanceMetricsCache = performanceMetricsCache; _tempCollectionJanitor = tempCollectionJanitor; + _modelAnalyzer = modelAnalyzer; } public void Initialize() @@ -1669,11 +1674,36 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa return; } + SplitPapMappings(moddedPaths, out var withoutPap, out var papOnly); + await _ipcManager.Penumbra.AssignTemporaryCollectionAsync(Logger, penumbraCollection, objIndex.Value).ConfigureAwait(false); - await _ipcManager.Penumbra.SetTemporaryModsAsync(Logger, _applicationId, penumbraCollection, - moddedPaths.ToDictionary(k => k.Key.GamePath, k => k.Value, StringComparer.Ordinal)).ConfigureAwait(false); - _lastAppliedModdedPaths = new Dictionary<(string GamePath, string? Hash), string>(moddedPaths, moddedPaths.Comparer); + await _ipcManager.Penumbra.SetTemporaryModsAsync( + Logger, _applicationId, penumbraCollection, + withoutPap.ToDictionary(k => k.Key.GamePath, k => k.Value, StringComparer.Ordinal)) + .ConfigureAwait(false); + + await _ipcManager.Penumbra.RedrawAsync(Logger, handlerForApply, _applicationId, token).ConfigureAwait(false); + if (handlerForApply.Address != nint.Zero) + await _actorObjectService.WaitForFullyLoadedAsync(handlerForApply.Address, token).ConfigureAwait(false); + + var removedPap = await StripIncompatiblePapAsync(handlerForApply, charaData, papOnly, token).ConfigureAwait(false); + if (removedPap > 0) + { + Logger.LogTrace("[{applicationId}] Removed {removedPap} incompatible PAP mappings found for {handler}", _applicationId, removedPap, GetLogIdentifier()); + } + + var merged = new Dictionary<(string GamePath, string? Hash), string>(withoutPap, withoutPap.Comparer); + foreach (var kv in papOnly) + merged[kv.Key] = kv.Value; + + await _ipcManager.Penumbra.SetTemporaryModsAsync( + Logger, _applicationId, penumbraCollection, + merged.ToDictionary(k => k.Key.GamePath, k => k.Value, StringComparer.Ordinal)) + .ConfigureAwait(false); + + _lastAppliedModdedPaths = new Dictionary<(string GamePath, string? Hash), string>(merged, merged.Comparer); + LastAppliedDataBytes = -1; foreach (var path in moddedPaths.Values.Distinct(StringComparer.OrdinalIgnoreCase).Select(v => new FileInfo(v)).Where(p => p.Exists)) { @@ -1983,9 +2013,17 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa foreach (var gamePath in item.GamePaths) { + if (gamePath.EndsWith(".pap", StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrEmpty(item.Hash) + && _blockedPapHashes.ContainsKey(item.Hash)) + { + continue; + } + var preferredPath = skipDownscaleForPair ? fileCache.ResolvedFilepath : _textureDownscaleService.GetPreferredPath(item.Hash, fileCache.ResolvedFilepath); + outputDict[(gamePath, item.Hash)] = preferredPath; } } @@ -2295,7 +2333,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa HandleVisibilityLoss(logChange: false); } - private bool TryResolveDescriptorHash(ActorObjectService.ActorDescriptor descriptor, out string hashedCid) + private static bool TryResolveDescriptorHash(ActorObjectService.ActorDescriptor descriptor, out string hashedCid) { hashedCid = descriptor.HashedContentId ?? string.Empty; if (!string.IsNullOrEmpty(hashedCid)) @@ -2308,6 +2346,194 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa return !string.IsNullOrEmpty(hashedCid); } + private static bool ContainsIndexCompat(HashSet available, ushort idx) + { + if (available.Contains(idx)) return true; + + if (idx > 0 && available.Contains((ushort)(idx - 1))) return true; + if (idx < ushort.MaxValue && available.Contains((ushort)(idx + 1))) return true; + + return false; + } + + private static bool IsPapCompatible( + IReadOnlyDictionary> localBoneSets, + IReadOnlyDictionary> papBoneIndices, + out string reason) + { + var groups = papBoneIndices + .Select(kvp => new + { + Raw = kvp.Key, + Key = XivDataAnalyzer.CanonicalizeSkeletonKey(kvp.Key), + Indices = kvp.Value + }) + .Where(x => !string.IsNullOrEmpty(x.Key) && x.Indices is { Count: > 0 }) + .GroupBy(x => x.Key, StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (groups.Count == 0) + { + reason = "No bindings found in the PAP"; + return false; + } + + var relevant = groups.Where(g => localBoneSets.ContainsKey(g.Key)).ToList(); + + if (relevant.Count == 0) + { + var papKeys = string.Join(", ", groups.Select(g => g.Key).Distinct(StringComparer.OrdinalIgnoreCase)); + var localKeys = string.Join(", ", localBoneSets.Keys); + reason = $"No matching skeleton bucket between PAP [{papKeys}] and local [{localKeys}]."; + return false; + } + + foreach (var g in relevant) + { + var available = localBoneSets[g.Key]; + + bool anyVariantOk = false; + foreach (var variant in g) + { + bool ok = true; + foreach (var idx in variant.Indices) + { + if (!ContainsIndexCompat(available, idx)) + { + ok = false; + break; + } + } + + if (ok) + { + anyVariantOk = true; + break; + } + } + + if (!anyVariantOk) + { + var first = g.First(); + ushort? missing = null; + foreach (var idx in first.Indices) + { + if (!ContainsIndexCompat(available, idx)) + { + missing = idx; + break; + } + } + + reason = missing.HasValue + ? $"Skeleton '{g.Key}' missing bone index {missing.Value}. (raw '{first.Raw}')" + : $"Skeleton '{g.Key}' missing required bone indices. (raw '{first.Raw}')"; + return false; + } + } + + reason = string.Empty; + return true; + } + + private static void SplitPapMappings( + Dictionary<(string GamePath, string? Hash), string> moddedPaths, + out Dictionary<(string GamePath, string? Hash), string> withoutPap, + out Dictionary<(string GamePath, string? Hash), string> papOnly) + { + withoutPap = new(moddedPaths.Comparer); + papOnly = new(moddedPaths.Comparer); + + foreach (var kv in moddedPaths) + { + var gamePath = kv.Key.GamePath; + if (gamePath.EndsWith(".pap", StringComparison.OrdinalIgnoreCase)) + papOnly[kv.Key] = kv.Value; + else + withoutPap[kv.Key] = kv.Value; + } + } + + private async Task StripIncompatiblePapAsync( + GameObjectHandler handlerForApply, + CharacterData charaData, + Dictionary<(string GamePath, string? Hash), string> papOnly, + CancellationToken token) + { + if (papOnly.Count == 0) + return 0; + + var boneIndices = await _dalamudUtil.RunOnFrameworkThread( + () => _modelAnalyzer.GetSkeletonBoneIndices(handlerForApply)) + .ConfigureAwait(false); + + if (boneIndices == null || boneIndices.Count == 0) + return papOnly.Count; + + var localBoneSets = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + foreach (var (rawKey, list) in boneIndices) + { + var key = XivDataAnalyzer.CanonicalizeSkeletonKey(rawKey); + if (string.IsNullOrEmpty(key) || list is null || list.Count == 0) + continue; + + if (!localBoneSets.TryGetValue(key, out var set)) + localBoneSets[key] = set = new HashSet(); + + foreach (var v in list) + set.Add(v); + } + + int removed = 0; + + foreach (var hash in papOnly.Keys.Select(k => k.Hash).Where(h => !string.IsNullOrEmpty(h)).Distinct(StringComparer.OrdinalIgnoreCase).ToList()) + { + token.ThrowIfCancellationRequested(); + + var papIndices = await _dalamudUtil.RunOnFrameworkThread( + () => _modelAnalyzer.GetBoneIndicesFromPap(hash!)) + .ConfigureAwait(false); + + if (papIndices == null || papIndices.Count == 0) + continue; + + if (papIndices.All(k => k.Value.DefaultIfEmpty().Max() <= 105)) + continue; + + if (!IsPapCompatible(localBoneSets, papIndices, out var reason)) + { + var keysToRemove = papOnly.Keys.Where(k => string.Equals(k.Hash, hash, StringComparison.OrdinalIgnoreCase)).ToList(); + foreach (var k in keysToRemove) + papOnly.Remove(k); + + removed += keysToRemove.Count; + if (hash == null) + continue; + + if (_blockedPapHashes.TryAdd(hash, 0)) + { + Logger.LogWarning("Blocked remote object PAP (hash {hash}) for {handler}: {reason}", hash, GetLogIdentifier(), reason); + } + + if (charaData.FileReplacements.TryGetValue(ObjectKind.Player, out var list)) + { + list.RemoveAll(r => string.Equals(r.Hash, hash, StringComparison.OrdinalIgnoreCase) + && r.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase))); + } + } + } + + var nullHashKeys = papOnly.Keys.Where(k => string.IsNullOrEmpty(k.Hash)).ToList(); + foreach (var k in nullHashKeys) + { + papOnly.Remove(k); + removed++; + } + + return removed; + } + private async Task ApplyCustomizeAsync(nint address, string customizeData, ObjectKind kind) { _customizeIds[kind] = await _ipcManager.CustomizePlus.SetBodyScaleAsync(address, customizeData).ConfigureAwait(false); diff --git a/LightlessSync/PlayerData/Pairs/PairHandlerAdapterFactory.cs b/LightlessSync/PlayerData/Pairs/PairHandlerAdapterFactory.cs index 5169820..3e36006 100644 --- a/LightlessSync/PlayerData/Pairs/PairHandlerAdapterFactory.cs +++ b/LightlessSync/PlayerData/Pairs/PairHandlerAdapterFactory.cs @@ -32,6 +32,7 @@ internal sealed class PairHandlerAdapterFactory : IPairHandlerAdapterFactory private readonly PairStateCache _pairStateCache; private readonly PairPerformanceMetricsCache _pairPerformanceMetricsCache; private readonly PenumbraTempCollectionJanitor _tempCollectionJanitor; + private readonly XivDataAnalyzer _modelAnalyzer; public PairHandlerAdapterFactory( ILoggerFactory loggerFactory, @@ -50,7 +51,8 @@ internal sealed class PairHandlerAdapterFactory : IPairHandlerAdapterFactory TextureDownscaleService textureDownscaleService, PairStateCache pairStateCache, PairPerformanceMetricsCache pairPerformanceMetricsCache, - PenumbraTempCollectionJanitor tempCollectionJanitor) + PenumbraTempCollectionJanitor tempCollectionJanitor, + XivDataAnalyzer modelAnalyzer) { _loggerFactory = loggerFactory; _mediator = mediator; @@ -69,6 +71,7 @@ internal sealed class PairHandlerAdapterFactory : IPairHandlerAdapterFactory _pairStateCache = pairStateCache; _pairPerformanceMetricsCache = pairPerformanceMetricsCache; _tempCollectionJanitor = tempCollectionJanitor; + _modelAnalyzer = modelAnalyzer; } public IPairHandlerAdapter Create(string ident) @@ -95,6 +98,7 @@ internal sealed class PairHandlerAdapterFactory : IPairHandlerAdapterFactory _textureDownscaleService, _pairStateCache, _pairPerformanceMetricsCache, - _tempCollectionJanitor); + _tempCollectionJanitor, + _modelAnalyzer); } } diff --git a/LightlessSync/Services/XivDataAnalyzer.cs b/LightlessSync/Services/XivDataAnalyzer.cs index 9d32883..6cd9cef 100644 --- a/LightlessSync/Services/XivDataAnalyzer.cs +++ b/LightlessSync/Services/XivDataAnalyzer.cs @@ -9,6 +9,7 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.PlayerData.Handlers; using Microsoft.Extensions.Logging; using System.Runtime.InteropServices; +using System.Text.RegularExpressions; namespace LightlessSync.Services; @@ -29,67 +30,140 @@ public sealed class XivDataAnalyzer public unsafe Dictionary>? GetSkeletonBoneIndices(GameObjectHandler handler) { - if (handler.Address == nint.Zero) return null; - var chara = (CharacterBase*)(((Character*)handler.Address)->GameObject.DrawObject); - if (chara->GetModelType() != CharacterBase.ModelType.Human) return null; - var resHandles = chara->Skeleton->SkeletonResourceHandles; - Dictionary> outputIndices = []; + if (handler is null || handler.Address == nint.Zero) + return null; + + Dictionary> sets = new(StringComparer.OrdinalIgnoreCase); + try { - for (int i = 0; i < chara->Skeleton->PartialSkeletonCount; i++) + var drawObject = ((Character*)handler.Address)->GameObject.DrawObject; + if (drawObject == null) + return null; + + var chara = (CharacterBase*)drawObject; + if (chara->GetModelType() != CharacterBase.ModelType.Human) + return null; + + var skeleton = chara->Skeleton; + if (skeleton == null) + return null; + + var resHandles = skeleton->SkeletonResourceHandles; + var partialCount = skeleton->PartialSkeletonCount; + if (partialCount <= 0) + return null; + + for (int i = 0; i < partialCount; i++) { var handle = *(resHandles + i); - _logger.LogTrace("Iterating over SkeletonResourceHandle #{i}:{x}", i, ((nint)handle).ToString("X")); - if ((nint)handle == nint.Zero) continue; - var curBones = handle->BoneCount; - // this is unrealistic, the filename shouldn't ever be that long - if (handle->FileName.Length > 1024) continue; - var skeletonName = handle->FileName.ToString(); - if (string.IsNullOrEmpty(skeletonName)) continue; - outputIndices[skeletonName] = []; - for (ushort boneIdx = 0; boneIdx < curBones; boneIdx++) + if ((nint)handle == nint.Zero) + continue; + + if (handle->FileName.Length > 1024) + continue; + + var rawName = handle->FileName.ToString(); + if (string.IsNullOrWhiteSpace(rawName)) + continue; + + var skeletonKey = CanonicalizeSkeletonKey(rawName); + if (string.IsNullOrEmpty(skeletonKey)) + continue; + + var boneCount = handle->BoneCount; + if (boneCount == 0) + continue; + + var havokSkel = handle->HavokSkeleton; + if ((nint)havokSkel == nint.Zero) + continue; + + if (!sets.TryGetValue(skeletonKey, out var set)) { - var boneName = handle->HavokSkeleton->Bones[boneIdx].Name.String; - if (boneName == null) continue; - outputIndices[skeletonName].Add((ushort)(boneIdx + 1)); + set = []; + sets[skeletonKey] = set; } + + uint maxExclusive = boneCount; + uint ushortExclusive = (uint)ushort.MaxValue + 1u; + if (maxExclusive > ushortExclusive) + maxExclusive = ushortExclusive; + + for (uint boneIdx = 0; boneIdx < maxExclusive; boneIdx++) + { + var name = havokSkel->Bones[boneIdx].Name.String; + if (name == null) + continue; + + set.Add((ushort)boneIdx); + } + + _logger.LogTrace("Local skeleton raw file='{raw}', key='{key}', boneCount={count}", + rawName, skeletonKey, boneCount); } } catch (Exception ex) { _logger.LogWarning(ex, "Could not process skeleton data"); + return null; } - return (outputIndices.Count != 0 && outputIndices.Values.All(u => u.Count > 0)) ? outputIndices : null; + if (sets.Count == 0) + return null; + + var output = new Dictionary>(sets.Count, StringComparer.OrdinalIgnoreCase); + foreach (var (key, set) in sets) + { + if (set.Count == 0) + continue; + + var list = set.ToList(); + list.Sort(); + output[key] = list; + } + + return (output.Count != 0 && output.Values.All(v => v.Count > 0)) ? output : null; } public unsafe Dictionary>? GetBoneIndicesFromPap(string hash) { - if (_configService.Current.BonesDictionary.TryGetValue(hash, out var bones)) return bones; + if (_configService.Current.BonesDictionary.TryGetValue(hash, out var cached)) + return cached; var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash); - if (cacheEntity == null) return null; + if (cacheEntity == null || string.IsNullOrEmpty(cacheEntity.ResolvedFilepath) || !File.Exists(cacheEntity.ResolvedFilepath)) + return null; - using BinaryReader reader = new(File.Open(cacheEntity.ResolvedFilepath, FileMode.Open, FileAccess.Read, FileShare.Read)); + using var fs = File.Open(cacheEntity.ResolvedFilepath, FileMode.Open, FileAccess.Read, FileShare.Read); + using var reader = new BinaryReader(fs); - // most of this shit is from vfxeditor, surely nothing will change in the pap format :copium: + // most of this is from vfxeditor reader.ReadInt32(); // ignore reader.ReadInt32(); // ignore - reader.ReadInt16(); // read 2 (num animations) - reader.ReadInt16(); // read 2 (modelid) - var type = reader.ReadByte();// read 1 (type) - if (type != 0) return null; // it's not human, just ignore it, whatever + reader.ReadInt16(); // num animations + reader.ReadInt16(); // modelid + var type = reader.ReadByte(); // type + if (type != 0) + return null; // not human - reader.ReadByte(); // read 1 (variant) + reader.ReadByte(); // variant reader.ReadInt32(); // ignore var havokPosition = reader.ReadInt32(); var footerPosition = reader.ReadInt32(); + + if (havokPosition <= 0 || footerPosition <= havokPosition || footerPosition > fs.Length) + return null; + var havokDataSize = footerPosition - havokPosition; reader.BaseStream.Position = havokPosition; + var havokData = reader.ReadBytes(havokDataSize); - if (havokData.Length <= 8) return null; // no havok data + if (havokData.Length <= 8) + return null; var output = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var tempHavokDataPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()) + ".hkx"; var tempHavokDataPathAnsi = Marshal.StringToHGlobalAnsi(tempHavokDataPath); @@ -102,54 +176,150 @@ public sealed class XivDataAnalyzer loadoptions->ClassNameRegistry = hkBuiltinTypeRegistry.Instance()->GetClassNameRegistry(); loadoptions->Flags = new hkFlags { - Storage = (int)(hkSerializeUtil.LoadOptionBits.Default) + Storage = (int)hkSerializeUtil.LoadOptionBits.Default }; var resource = hkSerializeUtil.LoadFromFile((byte*)tempHavokDataPathAnsi, null, loadoptions); if (resource == null) - { throw new InvalidOperationException("Resource was null after loading"); - } var rootLevelName = @"hkRootLevelContainer"u8; fixed (byte* n1 = rootLevelName) { var container = (hkRootLevelContainer*)resource->GetContentsPointer(n1, hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry()); + if (container == null) + return null; + var animationName = @"hkaAnimationContainer"u8; fixed (byte* n2 = animationName) { var animContainer = (hkaAnimationContainer*)container->findObjectByName(n2, null); + if (animContainer == null) + return null; + for (int i = 0; i < animContainer->Bindings.Length; i++) { var binding = animContainer->Bindings[i].ptr; + if (binding == null) + continue; + + var rawSkel = binding->OriginalSkeletonName.String; + var skeletonKey = CanonicalizeSkeletonKey(rawSkel); + if (string.IsNullOrEmpty(skeletonKey)) + continue; + var boneTransform = binding->TransformTrackToBoneIndices; - string name = binding->OriginalSkeletonName.String! + "_" + i; - output[name] = []; + if (boneTransform.Length <= 0) + continue; + + if (!output.TryGetValue(skeletonKey, out var list)) + { + list = new List(boneTransform.Length); + output[skeletonKey] = list; + } + for (int boneIdx = 0; boneIdx < boneTransform.Length; boneIdx++) { - output[name].Add((ushort)boneTransform[boneIdx]); + list.Add((ushort)boneTransform[boneIdx]); } - output[name].Sort(); } - } } + + foreach (var key in output.Keys.ToList()) + { + output[key] = [.. output[key] + .Distinct() + .Order()]; + } } catch (Exception ex) { _logger.LogWarning(ex, "Could not load havok file in {path}", tempHavokDataPath); + return null; } finally { Marshal.FreeHGlobal(tempHavokDataPathAnsi); - File.Delete(tempHavokDataPath); + try { File.Delete(tempHavokDataPath); } catch { /* ignore */ } } _configService.Current.BonesDictionary[hash] = output; _configService.Save(); + return output; } + private static readonly Regex _bucketPathRegex = + new(@"(?i)(?:^|/)(?c\d{4})(?:/|$)", RegexOptions.Compiled); + + private static readonly Regex _bucketSklRegex = + new(@"(?i)\bskl_(?c\d{4})[a-z]\d{4}\b", RegexOptions.Compiled); + + private static readonly Regex _bucketLooseRegex = + new(@"(?i)(?c\d{4})(?!\d)", RegexOptions.Compiled); + + public static string CanonicalizeSkeletonKey(string? raw) + { + if (string.IsNullOrWhiteSpace(raw)) + return string.Empty; + + var s = raw.Replace('\\', '/').Trim(); + + var underscore = s.LastIndexOf('_'); + if (underscore > 0 && underscore + 1 < s.Length && char.IsDigit(s[underscore + 1])) + s = s[..underscore]; + + if (s.StartsWith("skeleton", StringComparison.OrdinalIgnoreCase)) + return "skeleton"; + + var m = _bucketPathRegex.Match(s); + if (m.Success) + return m.Groups["bucket"].Value.ToLowerInvariant(); + + m = _bucketSklRegex.Match(s); + if (m.Success) + return m.Groups["bucket"].Value.ToLowerInvariant(); + + m = _bucketLooseRegex.Match(s); + if (m.Success) + return m.Groups["bucket"].Value.ToLowerInvariant(); + + return string.Empty; + } + + public void DumpLocalSkeletonIndices(GameObjectHandler handler, string? filter = null) + { + var skels = GetSkeletonBoneIndices(handler); + if (skels == null) + { + _logger.LogTrace("DumpLocalSkeletonIndices: local skeleton indices are null or not found"); + return; + } + + var keys = skels.Keys + .Order(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + _logger.LogTrace("Local skeleton indices found ({count}): {keys}", + keys.Length, + string.Join(", ", keys)); + + if (!string.IsNullOrWhiteSpace(filter)) + { + var hits = keys.Where(k => + k.Equals(filter, StringComparison.OrdinalIgnoreCase) || + k.StartsWith(filter + "_", StringComparison.OrdinalIgnoreCase) || + filter.StartsWith(k + "_", StringComparison.OrdinalIgnoreCase) || + k.Contains(filter, StringComparison.OrdinalIgnoreCase)) + .ToArray(); + + _logger.LogTrace("Matches found for '{filter}': {hits}", + filter, + hits.Length == 0 ? "" : string.Join(", ", hits)); + } + } + public async Task GetTrianglesByHash(string hash) { if (_configService.Current.TriangleDictionary.TryGetValue(hash, out var cachedTris) && cachedTris > 0) diff --git a/LightlessSync/UI/DownloadUi.cs b/LightlessSync/UI/DownloadUi.cs index 2d9cdc1..02416bf 100644 --- a/LightlessSync/UI/DownloadUi.cs +++ b/LightlessSync/UI/DownloadUi.cs @@ -167,7 +167,7 @@ public class DownloadUi : WindowMediatorSubscriberBase List>> transfers; try { - transfers = _currentDownloads.ToList(); + transfers = [.. _currentDownloads]; } catch (ArgumentException) { diff --git a/LightlessSync/WebAPI/Files/FileDownloadManager.cs b/LightlessSync/WebAPI/Files/FileDownloadManager.cs index 8aa2b0b..693b40b 100644 --- a/LightlessSync/WebAPI/Files/FileDownloadManager.cs +++ b/LightlessSync/WebAPI/Files/FileDownloadManager.cs @@ -404,76 +404,32 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase private async Task WaitForDownloadReady(List downloadFileTransfer, Guid requestId, CancellationToken downloadCt) { - bool alreadyCancelled = false; - try + while (true) { - CancellationTokenSource localTimeoutCts = new(); - localTimeoutCts.CancelAfter(TimeSpan.FromSeconds(5)); - CancellationTokenSource composite = CancellationTokenSource.CreateLinkedTokenSource(downloadCt, localTimeoutCts.Token); + downloadCt.ThrowIfCancellationRequested(); - while (!_orchestrator.IsDownloadReady(requestId)) + if (_orchestrator.IsDownloadReady(requestId)) + break; + + using var resp = await _orchestrator.SendRequestAsync( + HttpMethod.Get, + LightlessFiles.RequestCheckQueueFullPath(downloadFileTransfer[0].DownloadUri, requestId), + downloadFileTransfer.Select(t => t.Hash).ToList(), + downloadCt).ConfigureAwait(false); + + resp.EnsureSuccessStatusCode(); + + var body = (await resp.Content.ReadAsStringAsync(downloadCt).ConfigureAwait(false)).Trim(); + if (string.Equals(body, "true", StringComparison.OrdinalIgnoreCase) || + body.Contains("\"ready\":true", StringComparison.OrdinalIgnoreCase)) { - try - { - await Task.Delay(250, composite.Token).ConfigureAwait(false); - } - catch (TaskCanceledException) - { - if (downloadCt.IsCancellationRequested) throw; - - var req = await _orchestrator.SendRequestAsync( - HttpMethod.Get, - LightlessFiles.RequestCheckQueueFullPath(downloadFileTransfer[0].DownloadUri, requestId), - downloadFileTransfer.Select(c => c.Hash).ToList(), - downloadCt).ConfigureAwait(false); - - req.EnsureSuccessStatusCode(); - - localTimeoutCts.Dispose(); - composite.Dispose(); - - localTimeoutCts = new(); - localTimeoutCts.CancelAfter(TimeSpan.FromSeconds(5)); - composite = CancellationTokenSource.CreateLinkedTokenSource(downloadCt, localTimeoutCts.Token); - } + break; } - localTimeoutCts.Dispose(); - composite.Dispose(); - - Logger.LogDebug("Download {requestId} ready", requestId); + await Task.Delay(250, downloadCt).ConfigureAwait(false); } - catch (TaskCanceledException) - { - try - { - await _orchestrator.SendRequestAsync(HttpMethod.Get, LightlessFiles.RequestCancelFullPath(downloadFileTransfer[0].DownloadUri, requestId)) - .ConfigureAwait(false); - alreadyCancelled = true; - } - catch - { - // ignore - } - throw; - } - finally - { - if (downloadCt.IsCancellationRequested && !alreadyCancelled) - { - try - { - await _orchestrator.SendRequestAsync(HttpMethod.Get, LightlessFiles.RequestCancelFullPath(downloadFileTransfer[0].DownloadUri, requestId)) - .ConfigureAwait(false); - } - catch - { - // ignore - } - } - _orchestrator.ClearDownloadRequest(requestId); - } + _orchestrator.ClearDownloadRequest(requestId); } private async Task DownloadQueuedBlockFileAsync( @@ -532,11 +488,9 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase try { - // sanity check length if (fileLengthBytes < 0 || fileLengthBytes > int.MaxValue) throw new InvalidDataException($"Invalid block entry length: {fileLengthBytes}"); - // safe cast after check var len = checked((int)fileLengthBytes); if (!replacementLookup.TryGetValue(fileHash, out var repl)) @@ -546,11 +500,9 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase continue; } - // decompress var filePath = _fileDbManager.GetCacheFilePath(fileHash, repl.Extension); Logger.LogTrace("{dlName}: Decompressing {file}:{len} => {dest}", downloadLabel, fileHash, fileLengthBytes, filePath); - // read compressed data var compressed = new byte[len]; await ReadExactlyAsync(fileBlockStream, compressed.AsMemory(0, len), ct).ConfigureAwait(false); @@ -563,20 +515,17 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase MungeBuffer(compressed); - // limit concurrent decompressions await _decompressGate.WaitAsync(ct).ConfigureAwait(false); try { var sw = System.Diagnostics.Stopwatch.StartNew(); - // decompress var decompressed = LZ4Wrapper.Unwrap(compressed); Logger.LogTrace("{dlName}: Unwrap {fileHash} took {ms}ms (compressed {c} bytes, decompressed {d} bytes)", downloadLabel, fileHash, sw.ElapsedMilliseconds, compressed.Length, decompressed?.Length ?? -1); - // write to file - await _fileCompactor.WriteAllBytesAsync(filePath, decompressed, ct).ConfigureAwait(false); + await _fileCompactor.WriteAllBytesAsync(filePath, bytes: decompressed, ct).ConfigureAwait(false); PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale); } finally @@ -793,7 +742,6 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase try { - // download (with slot) var progress = CreateInlineProgress(bytes => AddTransferredBytes(statusKey, bytes)); // Download slot held on get @@ -974,14 +922,12 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase if (!_orchestrator.IsInitialized) throw new InvalidOperationException("FileTransferManager is not initialized"); - // batch request var response = await _orchestrator.SendRequestAsync( HttpMethod.Get, LightlessFiles.ServerFilesGetSizesFullPath(_orchestrator.FilesCdnUri!), hashes, ct).ConfigureAwait(false); - // ensure success return await response.Content.ReadFromJsonAsync>(cancellationToken: ct).ConfigureAwait(false) ?? []; } -- 2.49.1 From 7d2a914c8418f7a63d5e8da87a95e74254f2af8f Mon Sep 17 00:00:00 2001 From: defnotken Date: Thu, 1 Jan 2026 20:57:37 -0600 Subject: [PATCH 02/13] Queue File compacting to let workers download as priority, Offload decompression task --- .../WebAPI/Files/FileDownloadManager.cs | 89 ++++++++++++++++--- 1 file changed, 78 insertions(+), 11 deletions(-) diff --git a/LightlessSync/WebAPI/Files/FileDownloadManager.cs b/LightlessSync/WebAPI/Files/FileDownloadManager.cs index 8aa2b0b..28614a1 100644 --- a/LightlessSync/WebAPI/Files/FileDownloadManager.cs +++ b/LightlessSync/WebAPI/Files/FileDownloadManager.cs @@ -30,6 +30,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase private readonly ConcurrentDictionary _activeDownloadStreams; private readonly SemaphoreSlim _decompressGate = new(Math.Max(1, Environment.ProcessorCount / 2), Math.Max(1, Environment.ProcessorCount / 2)); + + private readonly ConcurrentQueue _deferredCompressionQueue = new(); private volatile bool _disableDirectDownloads; private int _consecutiveDirectDownloadFailures; @@ -556,7 +558,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase if (len == 0) { - await _fileCompactor.WriteAllBytesAsync(filePath, Array.Empty(), ct).ConfigureAwait(false); + await File.WriteAllBytesAsync(filePath, Array.Empty(), ct).ConfigureAwait(false); PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale); continue; } @@ -567,17 +569,21 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase await _decompressGate.WaitAsync(ct).ConfigureAwait(false); try { - var sw = System.Diagnostics.Stopwatch.StartNew(); + // offload CPU-intensive decompression to threadpool to free up worker + await Task.Run(async () => + { + var sw = System.Diagnostics.Stopwatch.StartNew(); - // decompress - var decompressed = LZ4Wrapper.Unwrap(compressed); + // decompress + var decompressed = LZ4Wrapper.Unwrap(compressed); - Logger.LogTrace("{dlName}: Unwrap {fileHash} took {ms}ms (compressed {c} bytes, decompressed {d} bytes)", - downloadLabel, fileHash, sw.ElapsedMilliseconds, compressed.Length, decompressed?.Length ?? -1); + Logger.LogTrace("{dlName}: Unwrap {fileHash} took {ms}ms (compressed {c} bytes, decompressed {d} bytes)", + downloadLabel, fileHash, sw.ElapsedMilliseconds, compressed.Length, decompressed?.Length ?? -1); - // write to file - await _fileCompactor.WriteAllBytesAsync(filePath, decompressed, ct).ConfigureAwait(false); - PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale); + // write to file without compacting during download + await File.WriteAllBytesAsync(filePath, decompressed, ct).ConfigureAwait(false); + PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale); + }, ct).ConfigureAwait(false); } finally { @@ -752,8 +758,16 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase if (gameObjectHandler is not null) Mediator.Publish(new DownloadStartedMessage(gameObjectHandler, _downloadStatus)); + // work based on cpu count and slots + var coreCount = Environment.ProcessorCount; + var baseWorkers = Math.Min(slots, coreCount); + + // only add buffer if decompression has capacity AND we have cores to spare + var availableDecompressSlots = _decompressGate.CurrentCount; + var extraWorkers = (availableDecompressSlots > 0 && coreCount >= 6) ? 2 : 0; + // allow some extra workers so downloads can continue while earlier items decompress. - var workerDop = Math.Clamp(slots * 2, 2, 16); + var workerDop = Math.Clamp(baseWorkers + extraWorkers, 2, coreCount); // batch downloads Task batchTask = batchChunks.Length == 0 @@ -769,6 +783,9 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase await Task.WhenAll(batchTask, directTask).ConfigureAwait(false); + // process deferred compressions after all downloads complete + await ProcessDeferredCompressionsAsync(ct).ConfigureAwait(false); + Logger.LogDebug("Download end: {id}", objectName); ClearDownload(); } @@ -873,7 +890,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase byte[] compressedBytes = await File.ReadAllBytesAsync(tempFilename, ct).ConfigureAwait(false); var decompressedBytes = LZ4Wrapper.Unwrap(compressedBytes); - await _fileCompactor.WriteAllBytesAsync(finalFilename, decompressedBytes, ct).ConfigureAwait(false); + await File.WriteAllBytesAsync(finalFilename, decompressedBytes, ct).ConfigureAwait(false); PersistFileToStorage(directDownload.Hash, finalFilename, repl.GamePath, skipDownscale); MarkTransferredFiles(directDownload.DirectDownloadUrl!, 1); @@ -1001,6 +1018,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase fi.LastAccessTime = DateTime.Today; fi.LastWriteTime = RandomDayInThePast().Invoke(); + // queue file for deferred compression instead of compressing immediately + if (_configService.Current.UseCompactor) + _deferredCompressionQueue.Enqueue(filePath); + try { var entry = _fileDbManager.CreateCacheEntry(filePath); @@ -1026,6 +1047,52 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase private static IProgress CreateInlineProgress(Action callback) => new InlineProgress(callback); + private async Task ProcessDeferredCompressionsAsync(CancellationToken ct) + { + if (_deferredCompressionQueue.IsEmpty) + return; + + var filesToCompress = new List(); + while (_deferredCompressionQueue.TryDequeue(out var filePath)) + { + if (File.Exists(filePath)) + filesToCompress.Add(filePath); + } + + if (filesToCompress.Count == 0) + return; + + Logger.LogDebug("Starting deferred compression of {count} files", filesToCompress.Count); + + var compressionWorkers = Math.Clamp(Environment.ProcessorCount / 4, 2, 4); + + await Parallel.ForEachAsync(filesToCompress, + new ParallelOptions + { + MaxDegreeOfParallelism = compressionWorkers, + CancellationToken = ct + }, + async (filePath, token) => + { + try + { + await Task.Yield(); + if (_configService.Current.UseCompactor && File.Exists(filePath)) + { + var bytes = await File.ReadAllBytesAsync(filePath, token).ConfigureAwait(false); + await _fileCompactor.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false); + Logger.LogTrace("Compressed file: {filePath}", filePath); + } + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Failed to compress file: {filePath}", filePath); + } + }).ConfigureAwait(false); + + Logger.LogDebug("Completed deferred compression of {count} files", filesToCompress.Count); + } + private sealed class InlineProgress : IProgress { private readonly Action _callback; -- 2.49.1 From 96f8d33cdeeeee5c8df866b8627a141b344dc089 Mon Sep 17 00:00:00 2001 From: defnotken Date: Thu, 1 Jan 2026 21:57:48 -0600 Subject: [PATCH 03/13] Fixing dls ui and fixed cancel cache validation breaking menu. --- LightlessSync/UI/DownloadUi.cs | 16 +++++++++++++--- LightlessSync/UI/SettingsUi.cs | 15 +++++++++++++-- LightlessSync/packages.lock.json | 23 ++++++++++++++++++++++- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/LightlessSync/UI/DownloadUi.cs b/LightlessSync/UI/DownloadUi.cs index 02416bf..f0ed3fb 100644 --- a/LightlessSync/UI/DownloadUi.cs +++ b/LightlessSync/UI/DownloadUi.cs @@ -25,6 +25,8 @@ public class DownloadUi : WindowMediatorSubscriberBase private readonly ConcurrentDictionary _uploadingPlayers = new(); private readonly Dictionary _smoothed = []; private readonly Dictionary _downloadSpeeds = []; + private readonly Dictionary _downloadInitialTotals = []; + private byte _transferBoxTransparency = 100; private bool _notificationDismissed = true; @@ -66,6 +68,10 @@ public class DownloadUi : WindowMediatorSubscriberBase Mediator.Subscribe(this, (msg) => { _currentDownloads[msg.DownloadId] = msg.DownloadStatus; + // Capture initial totals when download starts + var totalFiles = msg.DownloadStatus.Values.Sum(s => s.TotalFiles); + var totalBytes = msg.DownloadStatus.Values.Sum(s => s.TotalBytes); + _downloadInitialTotals[msg.DownloadId] = (totalFiles, totalBytes); _notificationDismissed = false; }); Mediator.Subscribe(this, (msg) => @@ -435,9 +441,13 @@ public class DownloadUi : WindowMediatorSubscriberBase var handler = transfer.Key; var statuses = transfer.Value.Values; - var playerTotalFiles = statuses.Sum(s => s.TotalFiles); - var playerTransferredFiles = statuses.Sum(s => s.TransferredFiles); - var playerTotalBytes = statuses.Sum(s => s.TotalBytes); + var (playerTotalFiles, playerTotalBytes) = _downloadInitialTotals.TryGetValue(handler, out var totals) + ? totals + : (statuses.Sum(s => s.TotalFiles), statuses.Sum(s => s.TotalBytes)); + + var playerTransferredFiles = statuses.Count(s => + s.DownloadStatus == DownloadStatus.Decompressing || + s.TransferredBytes >= s.TotalBytes); var playerTransferredBytes = statuses.Sum(s => s.TransferredBytes); totalFiles += playerTotalFiles; diff --git a/LightlessSync/UI/SettingsUi.cs b/LightlessSync/UI/SettingsUi.cs index 1c86580..0efc91a 100644 --- a/LightlessSync/UI/SettingsUi.cs +++ b/LightlessSync/UI/SettingsUi.cs @@ -1925,14 +1925,25 @@ public class SettingsUi : WindowMediatorSubscriberBase { using (ImRaii.PushIndent(20f)) { - if (_validationTask.IsCompleted) + if (_validationTask.IsCompletedSuccessfully) { UiSharedService.TextWrapped( $"The storage validation has completed and removed {_validationTask.Result.Count} invalid files from storage."); } + else if (_validationTask.IsCanceled) + { + UiSharedService.ColorTextWrapped( + "Storage validation was cancelled.", + UIColors.Get("LightlessYellow")); + } + else if (_validationTask.IsFaulted) + { + UiSharedService.ColorTextWrapped( + "Storage validation failed with an error.", + UIColors.Get("DimRed")); + } else { - UiSharedService.TextWrapped( $"Storage validation is running: {_currentProgress.Item1}/{_currentProgress.Item2}"); if (_currentProgress.Item3 != null) diff --git a/LightlessSync/packages.lock.json b/LightlessSync/packages.lock.json index d47880c..45d7722 100644 --- a/LightlessSync/packages.lock.json +++ b/LightlessSync/packages.lock.json @@ -76,6 +76,19 @@ "Microsoft.AspNetCore.SignalR.Common": "10.0.1" } }, + "Microsoft.Extensions.Caching.Memory": { + "type": "Direct", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "NxqSP0Ky4dZ5ybszdZCqs1X2C70s+dXflqhYBUh/vhcQVTIooNCXIYnLVbafoAFGZMs51d9+rHxveXs0ZC3SQQ==", + "dependencies": { + "Microsoft.Extensions.Caching.Abstractions": "10.0.1", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.1", + "Microsoft.Extensions.Logging.Abstractions": "10.0.1", + "Microsoft.Extensions.Options": "10.0.1", + "Microsoft.Extensions.Primitives": "10.0.1" + } + }, "Microsoft.Extensions.Hosting": { "type": "Direct", "requested": "[10.0.1, )", @@ -233,6 +246,14 @@ "Microsoft.AspNetCore.SignalR.Common": "10.0.1" } }, + "Microsoft.Extensions.Caching.Abstractions": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "Vb1vVAQDxHpXVdL9fpOX2BzeV7bbhzG4pAcIKRauRl0/VfkE8mq0f+fYC+gWICh3dlzTZInJ/cTeBS2MgU/XvQ==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.1" + } + }, "Microsoft.Extensions.Configuration": { "type": "Transitive", "resolved": "10.0.1", @@ -618,7 +639,7 @@ "FlatSharp.Compiler": "[7.9.0, )", "FlatSharp.Runtime": "[7.9.0, )", "OtterGui": "[1.0.0, )", - "Penumbra.Api": "[5.13.0, )", + "Penumbra.Api": "[5.13.1, )", "Penumbra.String": "[1.0.7, )" } }, -- 2.49.1 From 34878911858f204bba5aa41a74cc32863d4c5d20 Mon Sep 17 00:00:00 2001 From: defnotken Date: Thu, 1 Jan 2026 22:23:46 -0600 Subject: [PATCH 04/13] Testing asyncing the transient task --- .../PlayerData/Factories/PlayerDataFactory.cs | 74 +++++++++++++------ 1 file changed, 53 insertions(+), 21 deletions(-) diff --git a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs index 4a20467..31df865 100644 --- a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs +++ b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs @@ -165,30 +165,67 @@ public class PlayerDataFactory } } - await _transientResourceManager.WaitForRecording(ct).ConfigureAwait(false); + Task? getHeelsOffset = null; + Task? getGlamourerData = null; + Task? getCustomizeData = null; + Task? getHonorificTitle = null; + Task? getMoodlesData = null; - if (objectKind == ObjectKind.Pet) + if (objectKind == ObjectKind.Player) { - foreach (var item in fragment.FileReplacements.Where(i => i.HasFileReplacement).SelectMany(p => p.GamePaths)) + getHeelsOffset = _ipcManager.Heels.GetOffsetAsync(); + getGlamourerData = _ipcManager.Glamourer.GetCharacterCustomizationAsync(playerRelatedObject.Address); + getCustomizeData = _ipcManager.CustomizePlus.GetScaleAsync(playerRelatedObject.Address); + getHonorificTitle = _ipcManager.Honorific.GetTitle(); + getMoodlesData = _ipcManager.Moodles.GetStatusAsync(playerRelatedObject.Address); + } + else + { + getGlamourerData = _ipcManager.Glamourer.GetCharacterCustomizationAsync(playerRelatedObject.Address); + getCustomizeData = _ipcManager.CustomizePlus.GetScaleAsync(playerRelatedObject.Address); + } + + var staticReplacements = fragment.FileReplacements.ToHashSet(); + + Task<(IReadOnlyDictionary ResolvedPaths, HashSet? ClearedReplacements)> transientTask = Task.Run(async () => + { + await _transientResourceManager.WaitForRecording(ct).ConfigureAwait(false); + + HashSet? clearedReplacements = null; + + if (objectKind == ObjectKind.Pet) { - if (_transientResourceManager.AddTransientResource(objectKind, item)) + foreach (var item in staticReplacements.Where(i => i.HasFileReplacement).SelectMany(p => p.GamePaths)) { - _logger.LogDebug("Marking static {item} for Pet as transient", item); + if (_transientResourceManager.AddTransientResource(objectKind, item)) + { + _logger.LogDebug("Marking static {item} for Pet as transient", item); + } } + + _logger.LogTrace("Clearing {count} Static Replacements for Pet", staticReplacements.Count); + clearedReplacements = staticReplacements; } - _logger.LogTrace("Clearing {count} Static Replacements for Pet", fragment.FileReplacements.Count); - fragment.FileReplacements.Clear(); - } + ct.ThrowIfCancellationRequested(); + + _logger.LogDebug("Handling transient update for {obj}", playerRelatedObject); + + _transientResourceManager.ClearTransientPaths(objectKind, [.. staticReplacements.SelectMany(c => c.GamePaths)]); + + var transientPaths = ManageSemiTransientData(objectKind); + IReadOnlyDictionary resolved = await GetFileReplacementsFromPaths(playerRelatedObject, transientPaths, new HashSet(StringComparer.Ordinal)).ConfigureAwait(false); + return (resolved, clearedReplacements); + }, ct); ct.ThrowIfCancellationRequested(); - _logger.LogDebug("Handling transient update for {obj}", playerRelatedObject); + var (resolvedTransientPaths, clearedForPet) = await transientTask.ConfigureAwait(false); - _transientResourceManager.ClearTransientPaths(objectKind, [.. fragment.FileReplacements.SelectMany(c => c.GamePaths)]); - - var transientPaths = ManageSemiTransientData(objectKind); - var resolvedTransientPaths = await GetFileReplacementsFromPaths(playerRelatedObject, transientPaths, new HashSet(StringComparer.Ordinal)).ConfigureAwait(false); + if (clearedForPet != null) + { + fragment.FileReplacements.Clear(); + } if (logDebug) { @@ -213,11 +250,6 @@ public class PlayerDataFactory fragment.FileReplacements = new HashSet(fragment.FileReplacements.Where(v => v.HasFileReplacement).OrderBy(v => v.ResolvedPath, StringComparer.Ordinal), FileReplacementComparer.Instance); - // gather up data from ipc - Task getHeelsOffset = _ipcManager.Heels.GetOffsetAsync(); - Task getGlamourerData = _ipcManager.Glamourer.GetCharacterCustomizationAsync(playerRelatedObject.Address); - Task getCustomizeData = _ipcManager.CustomizePlus.GetScaleAsync(playerRelatedObject.Address); - Task getHonorificTitle = _ipcManager.Honorific.GetTitle(); fragment.GlamourerString = await getGlamourerData.ConfigureAwait(false); _logger.LogDebug("Glamourer is now: {data}", fragment.GlamourerString); var customizeScale = await getCustomizeData.ConfigureAwait(false); @@ -229,13 +261,13 @@ public class PlayerDataFactory var playerFragment = (fragment as CharacterDataFragmentPlayer)!; playerFragment.ManipulationString = _ipcManager.Penumbra.GetMetaManipulations(); - playerFragment!.HonorificData = await getHonorificTitle.ConfigureAwait(false); + playerFragment!.HonorificData = await getHonorificTitle!.ConfigureAwait(false); _logger.LogDebug("Honorific is now: {data}", playerFragment!.HonorificData); - playerFragment!.HeelsData = await getHeelsOffset.ConfigureAwait(false); + playerFragment!.HeelsData = await getHeelsOffset!.ConfigureAwait(false); _logger.LogDebug("Heels is now: {heels}", playerFragment!.HeelsData); - playerFragment!.MoodlesData = await _ipcManager.Moodles.GetStatusAsync(playerRelatedObject.Address).ConfigureAwait(false) ?? string.Empty; + playerFragment!.MoodlesData = await getMoodlesData!.ConfigureAwait(false) ?? string.Empty; _logger.LogDebug("Moodles is now: {moodles}", playerFragment!.MoodlesData); playerFragment!.PetNamesData = _ipcManager.PetNames.GetLocalNames(); -- 2.49.1 From 277d368f83cbff9ac7e443f64f990b634d5d419e Mon Sep 17 00:00:00 2001 From: cake Date: Fri, 2 Jan 2026 07:43:30 +0100 Subject: [PATCH 05/13] Adjustments in PAP handling. --- .../PlayerData/Pairs/PairHandlerAdapter.cs | 74 ++++++++++--------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs b/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs index fc469db..e11f290 100644 --- a/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs +++ b/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs @@ -2357,7 +2357,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa } private static bool IsPapCompatible( - IReadOnlyDictionary> localBoneSets, + IReadOnlyDictionary> targetBoneSets, IReadOnlyDictionary> papBoneIndices, out string reason) { @@ -2378,56 +2378,60 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa return false; } - var relevant = groups.Where(g => localBoneSets.ContainsKey(g.Key)).ToList(); + var papKeys = string.Join(", ", groups.Select(g => g.Key).Distinct(StringComparer.OrdinalIgnoreCase)); + var targetKeys = string.Join(", ", targetBoneSets.Keys); - if (relevant.Count == 0) + foreach (var g in groups) { - var papKeys = string.Join(", ", groups.Select(g => g.Key).Distinct(StringComparer.OrdinalIgnoreCase)); - var localKeys = string.Join(", ", localBoneSets.Keys); - reason = $"No matching skeleton bucket between PAP [{papKeys}] and local [{localKeys}]."; - return false; - } + var candidates = targetBoneSets + .Where(kvp => string.Equals(kvp.Key, g.Key, StringComparison.OrdinalIgnoreCase)) + .ToList(); - foreach (var g in relevant) - { - var available = localBoneSets[g.Key]; - - bool anyVariantOk = false; - foreach (var variant in g) + if (candidates.Count == 0) { - bool ok = true; - foreach (var idx in variant.Indices) + if (targetBoneSets.Count == 1) { - if (!ContainsIndexCompat(available, idx)) - { - ok = false; - break; - } + candidates = targetBoneSets.ToList(); } - - if (ok) + else { - anyVariantOk = true; - break; + reason = $"No matching skeleton bucket between PAP [{papKeys}] and target [{targetKeys}]."; + return false; } } - if (!anyVariantOk) + bool groupOk = false; + string? lastFail = null; + + foreach (var (targetKey, available) in candidates) { - var first = g.First(); - ushort? missing = null; - foreach (var idx in first.Indices) + foreach (var variant in g) { - if (!ContainsIndexCompat(available, idx)) + bool ok = true; + foreach (var idx in variant.Indices) { - missing = idx; + if (!ContainsIndexCompat(available, idx)) + { + ok = false; + lastFail = $"Target bucket '{targetKey}' missing bone index {idx}. (pap raw '{variant.Raw}')"; + break; + } + } + + if (ok) + { + groupOk = true; break; } } - reason = missing.HasValue - ? $"Skeleton '{g.Key}' missing bone index {missing.Value}. (raw '{first.Raw}')" - : $"Skeleton '{g.Key}' missing required bone indices. (raw '{first.Raw}')"; + if (groupOk) break; + } + + if (!groupOk) + { + reason = lastFail + ?? $"Target skeleton missing required bone indices for PAP bucket '{g.Key}'."; return false; } } @@ -2479,7 +2483,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa continue; if (!localBoneSets.TryGetValue(key, out var set)) - localBoneSets[key] = set = new HashSet(); + localBoneSets[key] = set = []; foreach (var v in list) set.Add(v); -- 2.49.1 From bb365442cfd4ef4e6d724a19e7a141bd876ef282 Mon Sep 17 00:00:00 2001 From: defnotken Date: Fri, 2 Jan 2026 09:29:04 -0600 Subject: [PATCH 06/13] Maybe? --- .../FileCache/TransientResourceManager.cs | 90 +++++++---- .../PlayerData/Factories/PlayerDataFactory.cs | 145 +++++++++++------- 2 files changed, 147 insertions(+), 88 deletions(-) diff --git a/LightlessSync/FileCache/TransientResourceManager.cs b/LightlessSync/FileCache/TransientResourceManager.cs index a8b467e..11073dc 100644 --- a/LightlessSync/FileCache/TransientResourceManager.cs +++ b/LightlessSync/FileCache/TransientResourceManager.cs @@ -297,7 +297,7 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase private void DalamudUtil_FrameworkUpdate() { - RefreshPlayerRelatedAddressMap(); + _ = Task.Run(() => RefreshPlayerRelatedAddressMap()); lock (_cacheAdditionLock) { @@ -306,20 +306,64 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase if (_lastClassJobId != _dalamudUtil.ClassJobId) { - _lastClassJobId = _dalamudUtil.ClassJobId; - if (SemiTransientResources.TryGetValue(ObjectKind.Pet, out HashSet? value)) - { - value?.Clear(); - } - - PlayerConfig.JobSpecificCache.TryGetValue(_dalamudUtil.ClassJobId, out var jobSpecificData); - SemiTransientResources[ObjectKind.Player] = PlayerConfig.GlobalPersistentCache.Concat(jobSpecificData ?? []).ToHashSet(StringComparer.OrdinalIgnoreCase); - PlayerConfig.JobSpecificPetCache.TryGetValue(_dalamudUtil.ClassJobId, out var petSpecificData); - SemiTransientResources[ObjectKind.Pet] = new HashSet( - petSpecificData ?? [], - StringComparer.OrdinalIgnoreCase); + UpdateClassJobCache(); } + CleanupAbsentObjects(); + } + + private void RefreshPlayerRelatedAddressMap() + { + var tempMap = new ConcurrentDictionary(); + var updatedFrameAddresses = new ConcurrentDictionary(); + + lock (_playerRelatedLock) + { + foreach (var handler in _playerRelatedPointers) + { + var address = (nint)handler.Address; + if (address != nint.Zero) + { + tempMap[address] = handler; + updatedFrameAddresses[address] = handler.ObjectKind; + } + } + } + + _playerRelatedByAddress.Clear(); + foreach (var kvp in tempMap) + { + _playerRelatedByAddress[kvp.Key] = kvp.Value; + } + + _cachedFrameAddresses.Clear(); + foreach (var kvp in updatedFrameAddresses) + { + _cachedFrameAddresses[kvp.Key] = kvp.Value; + } + } + + private void UpdateClassJobCache() + { + _lastClassJobId = _dalamudUtil.ClassJobId; + if (SemiTransientResources.TryGetValue(ObjectKind.Pet, out HashSet? value)) + { + value?.Clear(); + } + + PlayerConfig.JobSpecificCache.TryGetValue(_dalamudUtil.ClassJobId, out var jobSpecificData); + SemiTransientResources[ObjectKind.Player] = PlayerConfig.GlobalPersistentCache + .Concat(jobSpecificData ?? []) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + PlayerConfig.JobSpecificPetCache.TryGetValue(_dalamudUtil.ClassJobId, out var petSpecificData); + SemiTransientResources[ObjectKind.Pet] = new HashSet( + petSpecificData ?? [], + StringComparer.OrdinalIgnoreCase); + } + + private void CleanupAbsentObjects() + { foreach (var kind in Enum.GetValues(typeof(ObjectKind)).Cast()) { if (!_cachedFrameAddresses.Any(k => k.Value == kind) && TransientResources.Remove(kind, out _)) @@ -349,26 +393,6 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase _semiTransientResources = null; } - private void RefreshPlayerRelatedAddressMap() - { - _playerRelatedByAddress.Clear(); - var updatedFrameAddresses = new ConcurrentDictionary(); - lock (_playerRelatedLock) - { - foreach (var handler in _playerRelatedPointers) - { - var address = (nint)handler.Address; - if (address != nint.Zero) - { - _playerRelatedByAddress[address] = handler; - updatedFrameAddresses[address] = handler.ObjectKind; - } - } - } - - _cachedFrameAddresses = updatedFrameAddresses; - } - private void HandleActorTracked(ActorObjectService.ActorDescriptor descriptor) { if (descriptor.IsInGpose) diff --git a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs index 31df865..aeb04f3 100644 --- a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs +++ b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs @@ -142,29 +142,23 @@ public class PlayerDataFactory ct.ThrowIfCancellationRequested(); - fragment.FileReplacements = - [.. new HashSet(resolvedPaths.Select(c => new FileReplacement([.. c.Value], c.Key)), FileReplacementComparer.Instance).Where(p => p.HasFileReplacement)]; - fragment.FileReplacements.RemoveWhere(c => c.GamePaths.Any(g => !CacheMonitor.AllowedFileExtensions.Any(e => g.EndsWith(e, StringComparison.OrdinalIgnoreCase)))); + var fileReplacementsTask = Task.Run(() => + { + var replacements = new HashSet( + resolvedPaths.Select(c => new FileReplacement([.. c.Value], c.Key)), + FileReplacementComparer.Instance) + .Where(p => p.HasFileReplacement) + .ToHashSet(); + + replacements.RemoveWhere(c => c.GamePaths.Any(g => + !CacheMonitor.AllowedFileExtensions.Any(e => + g.EndsWith(e, StringComparison.OrdinalIgnoreCase)))); + + return replacements; + }, ct); ct.ThrowIfCancellationRequested(); - if (logDebug) - { - _logger.LogDebug("== Static Replacements =="); - foreach (var replacement in fragment.FileReplacements.Where(i => i.HasFileReplacement).OrderBy(i => i.GamePaths.First(), StringComparer.OrdinalIgnoreCase)) - { - _logger.LogDebug("=> {repl}", replacement); - ct.ThrowIfCancellationRequested(); - } - } - else - { - foreach (var replacement in fragment.FileReplacements.Where(i => i.HasFileReplacement)) - { - ct.ThrowIfCancellationRequested(); - } - } - Task? getHeelsOffset = null; Task? getGlamourerData = null; Task? getCustomizeData = null; @@ -185,38 +179,32 @@ public class PlayerDataFactory getCustomizeData = _ipcManager.CustomizePlus.GetScaleAsync(playerRelatedObject.Address); } + fragment.FileReplacements = await fileReplacementsTask.ConfigureAwait(false); + + if (logDebug) + { + _logger.LogDebug("== Static Replacements =="); + foreach (var replacement in fragment.FileReplacements.Where(i => i.HasFileReplacement).OrderBy(i => i.GamePaths.First(), StringComparer.OrdinalIgnoreCase)) + { + _logger.LogDebug("=> {repl}", replacement); + ct.ThrowIfCancellationRequested(); + } + } + else + { + foreach (var replacement in fragment.FileReplacements.Where(i => i.HasFileReplacement)) + { + ct.ThrowIfCancellationRequested(); + } + } + var staticReplacements = fragment.FileReplacements.ToHashSet(); - Task<(IReadOnlyDictionary ResolvedPaths, HashSet? ClearedReplacements)> transientTask = Task.Run(async () => - { - await _transientResourceManager.WaitForRecording(ct).ConfigureAwait(false); - - HashSet? clearedReplacements = null; - - if (objectKind == ObjectKind.Pet) - { - foreach (var item in staticReplacements.Where(i => i.HasFileReplacement).SelectMany(p => p.GamePaths)) - { - if (_transientResourceManager.AddTransientResource(objectKind, item)) - { - _logger.LogDebug("Marking static {item} for Pet as transient", item); - } - } - - _logger.LogTrace("Clearing {count} Static Replacements for Pet", staticReplacements.Count); - clearedReplacements = staticReplacements; - } - - ct.ThrowIfCancellationRequested(); - - _logger.LogDebug("Handling transient update for {obj}", playerRelatedObject); - - _transientResourceManager.ClearTransientPaths(objectKind, [.. staticReplacements.SelectMany(c => c.GamePaths)]); - - var transientPaths = ManageSemiTransientData(objectKind); - IReadOnlyDictionary resolved = await GetFileReplacementsFromPaths(playerRelatedObject, transientPaths, new HashSet(StringComparer.Ordinal)).ConfigureAwait(false); - return (resolved, clearedReplacements); - }, ct); + Task<(IReadOnlyDictionary ResolvedPaths, HashSet? ClearedReplacements)> transientTask = ProcessTransientDataAsync( + objectKind, + playerRelatedObject, + staticReplacements, + ct); ct.ThrowIfCancellationRequested(); @@ -278,12 +266,17 @@ public class PlayerDataFactory var toCompute = fragment.FileReplacements.Where(f => !f.IsFileSwap).ToArray(); _logger.LogDebug("Getting Hashes for {amount} Files", toCompute.Length); - var computedPaths = _fileCacheManager.GetFileCachesByPaths(toCompute.Select(c => c.ResolvedPath).ToArray()); - foreach (var file in toCompute) + + await Task.Run(() => { - ct.ThrowIfCancellationRequested(); - file.Hash = computedPaths[file.ResolvedPath]?.Hash ?? string.Empty; - } + var computedPaths = _fileCacheManager.GetFileCachesByPaths(toCompute.Select(c => c.ResolvedPath).ToArray()); + foreach (var file in toCompute) + { + ct.ThrowIfCancellationRequested(); + file.Hash = computedPaths[file.ResolvedPath]?.Hash ?? string.Empty; + } + }, ct).ConfigureAwait(false); + var removed = fragment.FileReplacements.RemoveWhere(f => !f.IsFileSwap && string.IsNullOrEmpty(f.Hash)); if (removed > 0) { @@ -507,6 +500,48 @@ public class PlayerDataFactory } } + private async Task<(IReadOnlyDictionary ResolvedPaths, HashSet? ClearedReplacements)> ProcessTransientDataAsync( + ObjectKind objectKind, + GameObjectHandler playerRelatedObject, + HashSet staticReplacements, + CancellationToken ct) + { + await _transientResourceManager.WaitForRecording(ct).ConfigureAwait(false); + + HashSet? clearedReplacements = null; + + var gamePaths = staticReplacements + .Where(i => i.HasFileReplacement) + .SelectMany(p => p.GamePaths) + .ToList(); + + if (objectKind == ObjectKind.Pet) + { + foreach (var item in gamePaths) + { + if (_transientResourceManager.AddTransientResource(objectKind, item)) + { + _logger.LogDebug("Marking static {item} for Pet as transient", item); + } + } + + _logger.LogTrace("Clearing {count} Static Replacements for Pet", staticReplacements.Count); + clearedReplacements = staticReplacements; + } + + ct.ThrowIfCancellationRequested(); + + _transientResourceManager.ClearTransientPaths(objectKind, gamePaths); + + var transientPaths = ManageSemiTransientData(objectKind); + IReadOnlyDictionary resolved = await GetFileReplacementsFromPaths( + playerRelatedObject, + transientPaths, + new HashSet(StringComparer.Ordinal)).ConfigureAwait(false); + + return (resolved, clearedReplacements); + } + private async Task> GetFileReplacementsFromPaths(GameObjectHandler handler, HashSet forwardResolve, HashSet reverseResolve) { -- 2.49.1 From e8c157d8ac7dd17a75f2e596cc693dffd642adaf Mon Sep 17 00:00:00 2001 From: defnotken Date: Fri, 2 Jan 2026 10:23:32 -0600 Subject: [PATCH 07/13] Creating temp havok file to not crash the analyzer --- LightlessSync/Services/XivDataAnalyzer.cs | 42 ++++++++++++++++++----- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/LightlessSync/Services/XivDataAnalyzer.cs b/LightlessSync/Services/XivDataAnalyzer.cs index 6cd9cef..f3f7e9d 100644 --- a/LightlessSync/Services/XivDataAnalyzer.cs +++ b/LightlessSync/Services/XivDataAnalyzer.cs @@ -164,12 +164,25 @@ public sealed class XivDataAnalyzer var output = new Dictionary>(StringComparer.OrdinalIgnoreCase); - var tempHavokDataPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()) + ".hkx"; - var tempHavokDataPathAnsi = Marshal.StringToHGlobalAnsi(tempHavokDataPath); + // write to temp file + var tempHavokDataPath = Path.Combine(Path.GetTempPath(), $"lightless_{Guid.NewGuid():N}.hkx"); + var tempHavokDataPathAnsi = IntPtr.Zero; try { - File.WriteAllBytes(tempHavokDataPath, havokData); + using (var tempFs = new FileStream(tempHavokDataPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, 4096, FileOptions.DeleteOnClose)) + { + tempFs.Write(havokData, 0, havokData.Length); + tempFs.Flush(true); + } + + if (!File.Exists(tempHavokDataPath)) + { + _logger.LogWarning("Temporary havok file was deleted before it could be loaded: {path}", tempHavokDataPath); + return null; + } + + tempHavokDataPathAnsi = Marshal.StringToHGlobalAnsi(tempHavokDataPath); var loadoptions = stackalloc hkSerializeUtil.LoadOptions[1]; loadoptions->TypeInfoRegistry = hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry(); @@ -181,7 +194,10 @@ public sealed class XivDataAnalyzer var resource = hkSerializeUtil.LoadFromFile((byte*)tempHavokDataPathAnsi, null, loadoptions); if (resource == null) - throw new InvalidOperationException("Resource was null after loading"); + { + _logger.LogWarning("Havok resource was null after loading from {path}", tempHavokDataPath); + return null; + } var rootLevelName = @"hkRootLevelContainer"u8; fixed (byte* n1 = rootLevelName) @@ -229,8 +245,8 @@ public sealed class XivDataAnalyzer foreach (var key in output.Keys.ToList()) { output[key] = [.. output[key] - .Distinct() - .Order()]; + .Distinct() + .Order()]; } } catch (Exception ex) @@ -240,8 +256,18 @@ public sealed class XivDataAnalyzer } finally { - Marshal.FreeHGlobal(tempHavokDataPathAnsi); - try { File.Delete(tempHavokDataPath); } catch { /* ignore */ } + if (tempHavokDataPathAnsi != IntPtr.Zero) + Marshal.FreeHGlobal(tempHavokDataPathAnsi); + + try + { + if (File.Exists(tempHavokDataPath)) + File.Delete(tempHavokDataPath); + } + catch (Exception ex) + { + _logger.LogTrace(ex, "Could not delete temporary havok file: {path}", tempHavokDataPath); + } } _configService.Current.BonesDictionary[hash] = output; -- 2.49.1 From 14c4c1d6693f34414a8e3cdcd2d6f50590a1fa99 Mon Sep 17 00:00:00 2001 From: cake Date: Fri, 2 Jan 2026 18:30:37 +0100 Subject: [PATCH 08/13] Added caching in the playerdata factory, refactored --- .../PlayerData/Factories/PlayerDataFactory.cs | 473 ++++++++++-------- LightlessSync/Services/DalamudUtilService.cs | 38 +- 2 files changed, 294 insertions(+), 217 deletions(-) diff --git a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs index aeb04f3..371a2a9 100644 --- a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs +++ b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs @@ -8,6 +8,8 @@ using LightlessSync.PlayerData.Handlers; using LightlessSync.Services; using LightlessSync.Services.Mediator; using Microsoft.Extensions.Logging; +using System.Collections.Concurrent; +using System.Diagnostics; namespace LightlessSync.PlayerData.Factories; @@ -23,9 +25,27 @@ public class PlayerDataFactory private readonly TransientResourceManager _transientResourceManager; private static readonly SemaphoreSlim _papParseLimiter = new(1, 1); - public PlayerDataFactory(ILogger logger, DalamudUtilService dalamudUtil, IpcManager ipcManager, - TransientResourceManager transientResourceManager, FileCacheManager fileReplacementFactory, - PerformanceCollectorService performanceCollector, XivDataAnalyzer modelAnalyzer, LightlessMediator lightlessMediator) + // Transient resolved entries threshold + private const int _maxTransientResolvedEntries = 1000; + + // Character build caches + private readonly ConcurrentDictionary> _characterBuildInflight = new(); + private readonly ConcurrentDictionary _characterBuildCache = new(); + + // Time out thresholds + private static readonly TimeSpan _characterCacheTtl = TimeSpan.FromMilliseconds(750); + private static readonly TimeSpan _softReturnIfBusyAfter = TimeSpan.FromMilliseconds(250); + private static readonly TimeSpan _hardBuildTimeout = TimeSpan.FromSeconds(30); + + public PlayerDataFactory( + ILogger logger, + DalamudUtilService dalamudUtil, + IpcManager ipcManager, + TransientResourceManager transientResourceManager, + FileCacheManager fileReplacementFactory, + PerformanceCollectorService performanceCollector, + XivDataAnalyzer modelAnalyzer, + LightlessMediator lightlessMediator) { _logger = logger; _dalamudUtil = dalamudUtil; @@ -37,13 +57,12 @@ public class PlayerDataFactory _lightlessMediator = lightlessMediator; _logger.LogTrace("Creating {this}", nameof(PlayerDataFactory)); } + private sealed record CacheEntry(CharacterDataFragment Fragment, DateTime CreatedUtc); public async Task BuildCharacterData(GameObjectHandler playerRelatedObject, CancellationToken token) { if (!_ipcManager.Initialized) - { throw new InvalidOperationException("Penumbra or Glamourer is not connected"); - } if (playerRelatedObject == null) return null; @@ -68,16 +87,17 @@ public class PlayerDataFactory if (pointerIsZero) { - _logger.LogTrace("Pointer was zero for {objectKind}", playerRelatedObject.ObjectKind); + _logger.LogTrace("Pointer was zero for {objectKind}; couldn't build character", playerRelatedObject.ObjectKind); return null; } try { - return await _performanceCollector.LogPerformance(this, $"CreateCharacterData>{playerRelatedObject.ObjectKind}", async () => - { - return await CreateCharacterData(playerRelatedObject, token).ConfigureAwait(false); - }).ConfigureAwait(true); + return await _performanceCollector.LogPerformance( + this, + $"CreateCharacterData>{playerRelatedObject.ObjectKind}", + async () => await CreateCharacterData(playerRelatedObject, token).ConfigureAwait(false) + ).ConfigureAwait(false); } catch (OperationCanceledException) { @@ -93,17 +113,14 @@ public class PlayerDataFactory } private async Task CheckForNullDrawObject(IntPtr playerPointer) - { - return await _dalamudUtil.RunOnFrameworkThread(() => CheckForNullDrawObjectUnsafe(playerPointer)).ConfigureAwait(false); - } + => await _dalamudUtil.RunOnFrameworkThread(() => CheckForNullDrawObjectUnsafe(playerPointer)).ConfigureAwait(false); - private unsafe bool CheckForNullDrawObjectUnsafe(IntPtr playerPointer) + private unsafe static bool CheckForNullDrawObjectUnsafe(IntPtr playerPointer) { if (playerPointer == IntPtr.Zero) return true; var character = (Character*)playerPointer; - if (character == null) return true; @@ -114,111 +131,167 @@ public class PlayerDataFactory return gameObject->DrawObject == null; } - private async Task CreateCharacterData(GameObjectHandler playerRelatedObject, CancellationToken ct) + private static bool IsCacheFresh(CacheEntry entry) + => (DateTime.UtcNow - entry.CreatedUtc) <= _characterCacheTtl; + + private Task CreateCharacterData(GameObjectHandler playerRelatedObject, CancellationToken ct) + => CreateCharacterDataCoalesced(playerRelatedObject, ct); + + private async Task CreateCharacterDataCoalesced(GameObjectHandler obj, CancellationToken ct) + { + var key = obj.Address; + + if (_characterBuildCache.TryGetValue(key, out var cached) && IsCacheFresh(cached) && !_characterBuildInflight.ContainsKey(key)) + return cached.Fragment; + + var buildTask = _characterBuildInflight.GetOrAdd(key, _ => BuildAndCacheAsync(obj, key)); + + if (_characterBuildCache.TryGetValue(key, out cached)) + { + var completed = await Task.WhenAny(buildTask, Task.Delay(_softReturnIfBusyAfter, ct)).ConfigureAwait(false); + if (completed != buildTask && (DateTime.UtcNow - cached.CreatedUtc) <= TimeSpan.FromSeconds(5)) + { + return cached.Fragment; + } + } + + return await WithCancellation(buildTask, ct).ConfigureAwait(false); + } + + private async Task BuildAndCacheAsync(GameObjectHandler obj, nint key) + { + try + { + using var cts = new CancellationTokenSource(_hardBuildTimeout); + var fragment = await CreateCharacterDataInternal(obj, cts.Token).ConfigureAwait(false); + + _characterBuildCache[key] = new CacheEntry(fragment, DateTime.UtcNow); + PruneCharacterCacheIfNeeded(); + + return fragment; + } + finally + { + _characterBuildInflight.TryRemove(key, out _); + } + } + + private void PruneCharacterCacheIfNeeded() + { + if (_characterBuildCache.Count < 2048) return; + + var cutoff = DateTime.UtcNow - TimeSpan.FromSeconds(10); + foreach (var kv in _characterBuildCache) + { + if (kv.Value.CreatedUtc < cutoff) + _characterBuildCache.TryRemove(kv.Key, out _); + } + } + + private static async Task WithCancellation(Task task, CancellationToken ct) + => await task.WaitAsync(ct).ConfigureAwait(false); + + private async Task CreateCharacterDataInternal(GameObjectHandler playerRelatedObject, CancellationToken ct) { var objectKind = playerRelatedObject.ObjectKind; CharacterDataFragment fragment = objectKind == ObjectKind.Player ? new CharacterDataFragmentPlayer() : new(); - _logger.LogDebug("Building character data for {obj}", playerRelatedObject); var logDebug = _logger.IsEnabled(LogLevel.Debug); + var sw = Stopwatch.StartNew(); - await _dalamudUtil.WaitWhileCharacterIsDrawing(_logger, playerRelatedObject, Guid.NewGuid(), 30000, ct: ct).ConfigureAwait(false); - int totalWaitTime = 10000; - while (!await _dalamudUtil.IsObjectPresentAsync(await _dalamudUtil.CreateGameObjectAsync(playerRelatedObject.Address).ConfigureAwait(false)).ConfigureAwait(false) && totalWaitTime > 0) - { - _logger.LogTrace("Character is null but it shouldn't be, waiting"); - await Task.Delay(50, ct).ConfigureAwait(false); - totalWaitTime -= 50; - } + _logger.LogDebug("Building character data for {obj}", playerRelatedObject); + + await EnsureObjectPresentAsync(playerRelatedObject, ct).ConfigureAwait(false); + ct.ThrowIfCancellationRequested(); + + var waitRecordingTask = _transientResourceManager.WaitForRecording(ct); + + await _dalamudUtil.WaitWhileCharacterIsDrawing(_logger, playerRelatedObject, Guid.NewGuid(), 30000, ct: ct) + .ConfigureAwait(false); ct.ThrowIfCancellationRequested(); - DateTime start = DateTime.UtcNow; - - Dictionary>? resolvedPaths; - - resolvedPaths = (await _ipcManager.Penumbra.GetCharacterData(_logger, playerRelatedObject).ConfigureAwait(false)); - if (resolvedPaths == null) throw new InvalidOperationException("Penumbra returned null data"); - - ct.ThrowIfCancellationRequested(); - - var fileReplacementsTask = Task.Run(() => - { - var replacements = new HashSet( - resolvedPaths.Select(c => new FileReplacement([.. c.Value], c.Key)), - FileReplacementComparer.Instance) - .Where(p => p.HasFileReplacement) - .ToHashSet(); - - replacements.RemoveWhere(c => c.GamePaths.Any(g => - !CacheMonitor.AllowedFileExtensions.Any(e => - g.EndsWith(e, StringComparison.OrdinalIgnoreCase)))); - - return replacements; - }, ct); - - ct.ThrowIfCancellationRequested(); + if (await CheckForNullDrawObject(playerRelatedObject.Address).ConfigureAwait(false)) + throw new InvalidOperationException("DrawObject became null during build (actor despawned)"); + Task getGlamourerData = _ipcManager.Glamourer.GetCharacterCustomizationAsync(playerRelatedObject.Address); + Task getCustomizeData = _ipcManager.CustomizePlus.GetScaleAsync(playerRelatedObject.Address); + Task? getMoodlesData = null; Task? getHeelsOffset = null; - Task? getGlamourerData = null; - Task? getCustomizeData = null; Task? getHonorificTitle = null; - Task? getMoodlesData = null; if (objectKind == ObjectKind.Player) { getHeelsOffset = _ipcManager.Heels.GetOffsetAsync(); - getGlamourerData = _ipcManager.Glamourer.GetCharacterCustomizationAsync(playerRelatedObject.Address); - getCustomizeData = _ipcManager.CustomizePlus.GetScaleAsync(playerRelatedObject.Address); getHonorificTitle = _ipcManager.Honorific.GetTitle(); getMoodlesData = _ipcManager.Moodles.GetStatusAsync(playerRelatedObject.Address); } - else - { - getGlamourerData = _ipcManager.Glamourer.GetCharacterCustomizationAsync(playerRelatedObject.Address); - getCustomizeData = _ipcManager.CustomizePlus.GetScaleAsync(playerRelatedObject.Address); - } - fragment.FileReplacements = await fileReplacementsTask.ConfigureAwait(false); + var resolvedPaths = await _ipcManager.Penumbra.GetCharacterData(_logger, playerRelatedObject).ConfigureAwait(false) ?? throw new InvalidOperationException("Penumbra returned null data; couldn't proceed with character"); + ct.ThrowIfCancellationRequested(); + + var staticBuildTask = Task.Run(() => BuildStaticReplacements(resolvedPaths), ct); + + fragment.FileReplacements = await staticBuildTask.ConfigureAwait(false); if (logDebug) { _logger.LogDebug("== Static Replacements =="); - foreach (var replacement in fragment.FileReplacements.Where(i => i.HasFileReplacement).OrderBy(i => i.GamePaths.First(), StringComparer.OrdinalIgnoreCase)) + foreach (var replacement in fragment.FileReplacements + .Where(i => i.HasFileReplacement) + .OrderBy(i => i.GamePaths.First(), StringComparer.OrdinalIgnoreCase)) { _logger.LogDebug("=> {repl}", replacement); ct.ThrowIfCancellationRequested(); } } - else - { - foreach (var replacement in fragment.FileReplacements.Where(i => i.HasFileReplacement)) - { - ct.ThrowIfCancellationRequested(); - } - } - var staticReplacements = fragment.FileReplacements.ToHashSet(); + var staticReplacements = new HashSet(fragment.FileReplacements, FileReplacementComparer.Instance); - Task<(IReadOnlyDictionary ResolvedPaths, HashSet? ClearedReplacements)> transientTask = ProcessTransientDataAsync( - objectKind, + var transientTask = ResolveTransientReplacementsAsync( playerRelatedObject, + objectKind, staticReplacements, + waitRecordingTask, ct); + fragment.GlamourerString = await getGlamourerData.ConfigureAwait(false); + _logger.LogDebug("Glamourer is now: {data}", fragment.GlamourerString); + + var customizeScale = await getCustomizeData.ConfigureAwait(false); + fragment.CustomizePlusScale = customizeScale ?? string.Empty; + _logger.LogDebug("Customize is now: {data}", fragment.CustomizePlusScale); + + if (objectKind == ObjectKind.Player) + { + CharacterDataFragmentPlayer? playerFragment = fragment as CharacterDataFragmentPlayer ?? throw new InvalidOperationException("Failed to cast CharacterDataFragment to Player variant"); + + playerFragment.ManipulationString = _ipcManager.Penumbra.GetMetaManipulations(); + playerFragment.HonorificData = await getHonorificTitle!.ConfigureAwait(false); + _logger.LogDebug("Honorific is now: {data}", playerFragment!.HonorificData); + + playerFragment.PetNamesData = _ipcManager.PetNames.GetLocalNames(); + _logger.LogDebug("Pet Nicknames is now: {petnames}", playerFragment!.PetNamesData); + + playerFragment.HeelsData = await getHeelsOffset!.ConfigureAwait(false); + _logger.LogDebug("Heels is now: {heels}", playerFragment!.HeelsData); + + playerFragment.MoodlesData = (await getMoodlesData!.ConfigureAwait(false)) ?? string.Empty; + _logger.LogDebug("Moodles is now: {moodles}", playerFragment!.MoodlesData); + } + ct.ThrowIfCancellationRequested(); var (resolvedTransientPaths, clearedForPet) = await transientTask.ConfigureAwait(false); - if (clearedForPet != null) - { fragment.FileReplacements.Clear(); - } if (logDebug) { _logger.LogDebug("== Transient Replacements =="); - foreach (var replacement in resolvedTransientPaths.Select(c => new FileReplacement([.. c.Value], c.Key)).OrderBy(f => f.ResolvedPath, StringComparer.Ordinal)) + foreach (var replacement in resolvedTransientPaths + .Select(c => new FileReplacement([.. c.Value], c.Key)) + .OrderBy(f => f.ResolvedPath, StringComparer.Ordinal)) { _logger.LogDebug("=> {repl}", replacement); fragment.FileReplacements.Add(replacement); @@ -227,40 +300,16 @@ public class PlayerDataFactory else { foreach (var replacement in resolvedTransientPaths.Select(c => new FileReplacement([.. c.Value], c.Key))) - { fragment.FileReplacements.Add(replacement); - } } _transientResourceManager.CleanUpSemiTransientResources(objectKind, [.. fragment.FileReplacements]); - ct.ThrowIfCancellationRequested(); - - fragment.FileReplacements = new HashSet(fragment.FileReplacements.Where(v => v.HasFileReplacement).OrderBy(v => v.ResolvedPath, StringComparer.Ordinal), FileReplacementComparer.Instance); - - fragment.GlamourerString = await getGlamourerData.ConfigureAwait(false); - _logger.LogDebug("Glamourer is now: {data}", fragment.GlamourerString); - var customizeScale = await getCustomizeData.ConfigureAwait(false); - fragment.CustomizePlusScale = customizeScale ?? string.Empty; - _logger.LogDebug("Customize is now: {data}", fragment.CustomizePlusScale); - - if (objectKind == ObjectKind.Player) - { - var playerFragment = (fragment as CharacterDataFragmentPlayer)!; - playerFragment.ManipulationString = _ipcManager.Penumbra.GetMetaManipulations(); - - playerFragment!.HonorificData = await getHonorificTitle!.ConfigureAwait(false); - _logger.LogDebug("Honorific is now: {data}", playerFragment!.HonorificData); - - playerFragment!.HeelsData = await getHeelsOffset!.ConfigureAwait(false); - _logger.LogDebug("Heels is now: {heels}", playerFragment!.HeelsData); - - playerFragment!.MoodlesData = await getMoodlesData!.ConfigureAwait(false) ?? string.Empty; - _logger.LogDebug("Moodles is now: {moodles}", playerFragment!.MoodlesData); - - playerFragment!.PetNamesData = _ipcManager.PetNames.GetLocalNames(); - _logger.LogDebug("Pet Nicknames is now: {petnames}", playerFragment!.PetNamesData); - } + fragment.FileReplacements = new HashSet( + fragment.FileReplacements + .Where(v => v.HasFileReplacement) + .OrderBy(v => v.ResolvedPath, StringComparer.Ordinal), + FileReplacementComparer.Instance); ct.ThrowIfCancellationRequested(); @@ -269,7 +318,7 @@ public class PlayerDataFactory await Task.Run(() => { - var computedPaths = _fileCacheManager.GetFileCachesByPaths(toCompute.Select(c => c.ResolvedPath).ToArray()); + var computedPaths = _fileCacheManager.GetFileCachesByPaths([.. toCompute.Select(c => c.ResolvedPath)]); foreach (var file in toCompute) { ct.ThrowIfCancellationRequested(); @@ -279,9 +328,7 @@ public class PlayerDataFactory var removed = fragment.FileReplacements.RemoveWhere(f => !f.IsFileSwap && string.IsNullOrEmpty(f.Hash)); if (removed > 0) - { _logger.LogDebug("Removed {amount} of invalid files", removed); - } ct.ThrowIfCancellationRequested(); @@ -299,21 +346,16 @@ public class PlayerDataFactory .RunOnFrameworkThread(() => _modelAnalyzer.GetSkeletonBoneIndices(playerRelatedObject)) .ConfigureAwait(false); } - } - if (objectKind == ObjectKind.Player) - { try { #if DEBUG if (hasPapFiles && boneIndices != null) - { _modelAnalyzer.DumpLocalSkeletonIndices(playerRelatedObject); - } #endif if (hasPapFiles) { - await VerifyPlayerAnimationBones(boneIndices, (fragment as CharacterDataFragmentPlayer)!, ct) + await VerifyPlayerAnimationBones(boneIndices, (CharacterDataFragmentPlayer)fragment, ct) .ConfigureAwait(false); } } @@ -328,11 +370,94 @@ public class PlayerDataFactory } } - _logger.LogInformation("Building character data for {obj} took {time}ms", objectKind, TimeSpan.FromTicks(DateTime.UtcNow.Ticks - start.Ticks).TotalMilliseconds); + _logger.LogInformation("Building character data for {obj} took {time}ms", + objectKind, sw.Elapsed.TotalMilliseconds); return fragment; } + private async Task EnsureObjectPresentAsync(GameObjectHandler handler, CancellationToken ct) + { + var remaining = 10000; + while (remaining > 0) + { + ct.ThrowIfCancellationRequested(); + + var obj = await _dalamudUtil.CreateGameObjectAsync(handler.Address).ConfigureAwait(false); + if (await _dalamudUtil.IsObjectPresentAsync(obj).ConfigureAwait(false)) + return; + + _logger.LogTrace("Character is null but it shouldn't be, waiting"); + await Task.Delay(50, ct).ConfigureAwait(false); + remaining -= 50; + } + } + + private static HashSet BuildStaticReplacements(Dictionary> resolvedPaths) + { + var set = new HashSet(FileReplacementComparer.Instance); + + foreach (var kvp in resolvedPaths) + { + var fr = new FileReplacement([.. kvp.Value], kvp.Key); + if (!fr.HasFileReplacement) continue; + + var allAllowed = fr.GamePaths.All(g => + CacheMonitor.AllowedFileExtensions.Any(e => g.EndsWith(e, StringComparison.OrdinalIgnoreCase))); + + if (!allAllowed) continue; + + set.Add(fr); + } + + return set; + } + + private async Task<(IReadOnlyDictionary ResolvedPaths, HashSet? ClearedReplacements)> + ResolveTransientReplacementsAsync( + GameObjectHandler obj, + ObjectKind objectKind, + HashSet staticReplacements, + Task waitRecordingTask, + CancellationToken ct) + { + await waitRecordingTask.ConfigureAwait(false); + + HashSet? clearedReplacements = null; + + if (objectKind == ObjectKind.Pet) + { + foreach (var item in staticReplacements.Where(i => i.HasFileReplacement).SelectMany(p => p.GamePaths)) + { + if (_transientResourceManager.AddTransientResource(objectKind, item)) + _logger.LogDebug("Marking static {item} for Pet as transient", item); + } + + _logger.LogTrace("Clearing {count} Static Replacements for Pet", staticReplacements.Count); + clearedReplacements = staticReplacements; + } + + ct.ThrowIfCancellationRequested(); + + _transientResourceManager.ClearTransientPaths(objectKind, [.. staticReplacements.SelectMany(c => c.GamePaths)]); + + var transientPaths = ManageSemiTransientData(objectKind); + if (transientPaths.Count == 0) + return (new Dictionary(StringComparer.Ordinal), clearedReplacements); + + var resolved = await GetFileReplacementsFromPaths(obj, transientPaths, new HashSet(StringComparer.Ordinal)) + .ConfigureAwait(false); + + if (_maxTransientResolvedEntries > 0 && resolved.Count > _maxTransientResolvedEntries) + { + _logger.LogWarning("Transient entries ({resolved}) are above the threshold {max}; Please consider disable some mods (VFX have heavy load) to reduce transient load", + resolved.Count, + _maxTransientResolvedEntries); + } + + return (resolved, clearedReplacements); + } + private async Task VerifyPlayerAnimationBones( Dictionary>? playerBoneIndices, CharacterDataFragmentPlayer fragment, @@ -347,12 +472,13 @@ public class PlayerDataFactory { if (indices == null || indices.Count == 0) continue; + var key = XivDataAnalyzer.CanonicalizeSkeletonKey(rawLocalKey); if (string.IsNullOrEmpty(key)) continue; if (!playerBoneSets.TryGetValue(key, out var set)) - playerBoneSets[key] = set = new HashSet(); + playerBoneSets[key] = set = []; foreach (var idx in indices) set.Add(idx); @@ -361,18 +487,6 @@ public class PlayerDataFactory if (playerBoneSets.Count == 0) return; - if (_logger.IsEnabled(LogLevel.Debug)) - { - foreach (var kvp in playerBoneSets) - { - _logger.LogDebug( - "Found local skeleton bucket '{bucket}' ({count} indices, max {max})", - kvp.Key, - kvp.Value.Count, - kvp.Value.Count > 0 ? kvp.Value.Max() : 0); - } - } - var papFiles = fragment.FileReplacements .Where(f => !f.IsFileSwap && f.GamePaths.First().EndsWith("pap", StringComparison.OrdinalIgnoreCase)) .ToList(); @@ -414,7 +528,6 @@ public class PlayerDataFactory ct.ThrowIfCancellationRequested(); var hash = group.Key; - Dictionary>? papSkeletonIndices; await _papParseLimiter.WaitAsync(ct).ConfigureAwait(false); @@ -432,10 +545,7 @@ public class PlayerDataFactory continue; if (ShouldIgnorePap(papSkeletonIndices)) - { - _logger.LogTrace("All indices of PAP hash {hash} are <= 105, ignoring", hash); continue; - } bool invalid = false; string? reason = null; @@ -480,7 +590,6 @@ public class PlayerDataFactory foreach (var file in group.ToList()) { - _logger.LogDebug("Removing {file} from sent file replacements and transient data", file.ResolvedPath); fragment.FileReplacements.Remove(file); foreach (var gamePath in file.GamePaths) @@ -500,75 +609,30 @@ public class PlayerDataFactory } } - private async Task<(IReadOnlyDictionary ResolvedPaths, HashSet? ClearedReplacements)> ProcessTransientDataAsync( - ObjectKind objectKind, - GameObjectHandler playerRelatedObject, - HashSet staticReplacements, - CancellationToken ct) - { - await _transientResourceManager.WaitForRecording(ct).ConfigureAwait(false); - - HashSet? clearedReplacements = null; - - var gamePaths = staticReplacements - .Where(i => i.HasFileReplacement) - .SelectMany(p => p.GamePaths) - .ToList(); - - if (objectKind == ObjectKind.Pet) - { - foreach (var item in gamePaths) - { - if (_transientResourceManager.AddTransientResource(objectKind, item)) - { - _logger.LogDebug("Marking static {item} for Pet as transient", item); - } - } - - _logger.LogTrace("Clearing {count} Static Replacements for Pet", staticReplacements.Count); - clearedReplacements = staticReplacements; - } - - ct.ThrowIfCancellationRequested(); - - _transientResourceManager.ClearTransientPaths(objectKind, gamePaths); - - var transientPaths = ManageSemiTransientData(objectKind); - IReadOnlyDictionary resolved = await GetFileReplacementsFromPaths( - playerRelatedObject, - transientPaths, - new HashSet(StringComparer.Ordinal)).ConfigureAwait(false); - - return (resolved, clearedReplacements); - } - - - private async Task> GetFileReplacementsFromPaths(GameObjectHandler handler, HashSet forwardResolve, HashSet reverseResolve) + private async Task> GetFileReplacementsFromPaths( + GameObjectHandler handler, + HashSet forwardResolve, + HashSet reverseResolve) { var forwardPaths = forwardResolve.ToArray(); var reversePaths = reverseResolve.ToArray(); Dictionary> resolvedPaths = new(StringComparer.Ordinal); + if (handler.ObjectKind != ObjectKind.Player) { var (objectIndex, forwardResolved, reverseResolved) = await _dalamudUtil.RunOnFrameworkThread(() => { var idx = handler.GetGameObject()?.ObjectIndex; if (!idx.HasValue) - { return ((int?)null, Array.Empty(), Array.Empty()); - } var resolvedForward = new string[forwardPaths.Length]; for (int i = 0; i < forwardPaths.Length; i++) - { resolvedForward[i] = _ipcManager.Penumbra.ResolveGameObjectPath(forwardPaths[i], idx.Value); - } var resolvedReverse = new string[reversePaths.Length][]; for (int i = 0; i < reversePaths.Length; i++) - { resolvedReverse[i] = _ipcManager.Penumbra.ReverseResolveGameObjectPath(reversePaths[i], idx.Value); - } return (idx, resolvedForward, resolvedReverse); }).ConfigureAwait(false); @@ -579,31 +643,21 @@ public class PlayerDataFactory { var filePath = forwardResolved[i]?.ToLowerInvariant(); if (string.IsNullOrEmpty(filePath)) - { continue; - } if (resolvedPaths.TryGetValue(filePath, out var list)) - { list.Add(forwardPaths[i].ToLowerInvariant()); - } else - { resolvedPaths[filePath] = [forwardPaths[i].ToLowerInvariant()]; - } } for (int i = 0; i < reversePaths.Length; i++) { var filePath = reversePaths[i].ToLowerInvariant(); if (resolvedPaths.TryGetValue(filePath, out var list)) - { list.AddRange(reverseResolved[i].Select(c => c.ToLowerInvariant())); - } else - { - resolvedPaths[filePath] = new List(reverseResolved[i].Select(c => c.ToLowerInvariant()).ToList()); - } + resolvedPaths[filePath] = [.. reverseResolved[i].Select(c => c.ToLowerInvariant()).ToList()]; } return resolvedPaths.ToDictionary(k => k.Key, k => k.Value.ToArray(), StringComparer.OrdinalIgnoreCase).AsReadOnly(); @@ -611,30 +665,23 @@ public class PlayerDataFactory } var (forward, reverse) = await _ipcManager.Penumbra.ResolvePathsAsync(forwardPaths, reversePaths).ConfigureAwait(false); + for (int i = 0; i < forwardPaths.Length; i++) { var filePath = forward[i].ToLowerInvariant(); if (resolvedPaths.TryGetValue(filePath, out var list)) - { list.Add(forwardPaths[i].ToLowerInvariant()); - } else - { resolvedPaths[filePath] = [forwardPaths[i].ToLowerInvariant()]; - } } for (int i = 0; i < reversePaths.Length; i++) { var filePath = reversePaths[i].ToLowerInvariant(); if (resolvedPaths.TryGetValue(filePath, out var list)) - { list.AddRange(reverse[i].Select(c => c.ToLowerInvariant())); - } else - { resolvedPaths[filePath] = new List(reverse[i].Select(c => c.ToLowerInvariant()).ToList()); - } } return resolvedPaths.ToDictionary(k => k.Key, k => k.Value.ToArray(), StringComparer.OrdinalIgnoreCase).AsReadOnly(); @@ -645,11 +692,29 @@ public class PlayerDataFactory _transientResourceManager.PersistTransientResources(objectKind); HashSet pathsToResolve = new(StringComparer.Ordinal); - foreach (var path in _transientResourceManager.GetSemiTransientResources(objectKind).Where(path => !string.IsNullOrEmpty(path))) + + int scanned = 0, skippedEmpty = 0, skippedVfx = 0; + + foreach (var path in _transientResourceManager.GetSemiTransientResources(objectKind)) { + scanned++; + + if (string.IsNullOrEmpty(path)) + { + skippedEmpty++; + continue; + } + pathsToResolve.Add(path); } + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "ManageSemiTransientData({kind}): scanned={scanned}, added={added}, skippedEmpty={skippedEmpty}, skippedVfx={skippedVfx}", + objectKind, scanned, pathsToResolve.Count, skippedEmpty, skippedVfx); + } + return pathsToResolve; } -} \ No newline at end of file +} diff --git a/LightlessSync/Services/DalamudUtilService.cs b/LightlessSync/Services/DalamudUtilService.cs index b278667..f47512a 100644 --- a/LightlessSync/Services/DalamudUtilService.cs +++ b/LightlessSync/Services/DalamudUtilService.cs @@ -22,8 +22,10 @@ using LightlessSync.Utils; using Lumina.Excel.Sheets; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using System.Diagnostics; using System.Numerics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Text; using BattleNpcSubKind = FFXIVClientStructs.FFXIV.Client.Game.Object.BattleNpcSubKind; using DalamudObjectKind = Dalamud.Game.ClientState.Objects.Enums.ObjectKind; @@ -843,31 +845,41 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber return Task.CompletedTask; } - public async Task WaitWhileCharacterIsDrawing(ILogger logger, GameObjectHandler handler, Guid redrawId, int timeOut = 5000, CancellationToken? ct = null) + public async Task WaitWhileCharacterIsDrawing( + ILogger logger, + GameObjectHandler handler, + Guid redrawId, + int timeOut = 5000, + CancellationToken? ct = null) { if (!_clientState.IsLoggedIn) return; - if (ct == null) - ct = CancellationToken.None; + var token = ct ?? CancellationToken.None; + + const int tick = 250; + const int initialSettle = 50; + + var sw = Stopwatch.StartNew(); - const int tick = 250; - int curWaitTime = 0; try { logger.LogTrace("[{redrawId}] Starting wait for {handler} to draw", redrawId, handler); - await Task.Delay(tick, ct.Value).ConfigureAwait(true); - curWaitTime += tick; - while ((!ct.Value.IsCancellationRequested) - && curWaitTime < timeOut - && await handler.IsBeingDrawnRunOnFrameworkAsync().ConfigureAwait(false)) // 0b100000000000 is "still rendering" or something + await Task.Delay(initialSettle, token).ConfigureAwait(false); + + while (!token.IsCancellationRequested + && sw.ElapsedMilliseconds < timeOut + && await handler.IsBeingDrawnRunOnFrameworkAsync().ConfigureAwait(false)) { logger.LogTrace("[{redrawId}] Waiting for {handler} to finish drawing", redrawId, handler); - curWaitTime += tick; - await Task.Delay(tick, ct.Value).ConfigureAwait(true); + await Task.Delay(tick, token).ConfigureAwait(false); } - logger.LogTrace("[{redrawId}] Finished drawing after {curWaitTime}ms", redrawId, curWaitTime); + logger.LogTrace("[{redrawId}] Finished drawing after {ms}ms", redrawId, sw.ElapsedMilliseconds); + } + catch (OperationCanceledException) + { + // ignore } catch (AccessViolationException ex) { -- 2.49.1 From e16ddb0a1d9e98bd590bd4114724ebb5bf777306 Mon Sep 17 00:00:00 2001 From: cake Date: Fri, 2 Jan 2026 19:29:50 +0100 Subject: [PATCH 09/13] change log to trace --- LightlessSync/Services/XivDataAnalyzer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LightlessSync/Services/XivDataAnalyzer.cs b/LightlessSync/Services/XivDataAnalyzer.cs index f3f7e9d..95700bb 100644 --- a/LightlessSync/Services/XivDataAnalyzer.cs +++ b/LightlessSync/Services/XivDataAnalyzer.cs @@ -178,7 +178,7 @@ public sealed class XivDataAnalyzer if (!File.Exists(tempHavokDataPath)) { - _logger.LogWarning("Temporary havok file was deleted before it could be loaded: {path}", tempHavokDataPath); + _logger.LogTrace("Temporary havok file was deleted before it could be loaded: {path}", tempHavokDataPath); return null; } -- 2.49.1 From e41a7149c5ecd8adb6e65d4b1a5284bad58ae717 Mon Sep 17 00:00:00 2001 From: cake Date: Sat, 3 Jan 2026 14:58:54 +0100 Subject: [PATCH 10/13] Refactored many parts, added settings for detection --- .../Configurations/LightlessConfig.cs | 2 + .../Factories/AnimationValidationMode.cs | 9 + .../PlayerData/Factories/PlayerDataFactory.cs | 160 +++++------ .../PlayerData/Pairs/PairHandlerAdapter.cs | 231 +++++---------- .../Pairs/PairHandlerAdapterFactory.cs | 9 +- LightlessSync/Services/XivDataAnalyzer.cs | 265 +++++++++++++++--- LightlessSync/UI/SettingsUi.cs | 85 +++++- 7 files changed, 481 insertions(+), 280 deletions(-) create mode 100644 LightlessSync/PlayerData/Factories/AnimationValidationMode.cs diff --git a/LightlessSync/LightlessConfiguration/Configurations/LightlessConfig.cs b/LightlessSync/LightlessConfiguration/Configurations/LightlessConfig.cs index 737f9ee..f38e100 100644 --- a/LightlessSync/LightlessConfiguration/Configurations/LightlessConfig.cs +++ b/LightlessSync/LightlessConfiguration/Configurations/LightlessConfig.cs @@ -4,6 +4,7 @@ using LightlessSync.LightlessConfiguration.Models; using LightlessSync.UI; using LightlessSync.UI.Models; using Microsoft.Extensions.Logging; +using LightlessSync.PlayerData.Factories; namespace LightlessSync.LightlessConfiguration.Configurations; @@ -156,4 +157,5 @@ public class LightlessConfig : ILightlessConfiguration public string? SelectedFinderSyncshell { get; set; } = null; public string LastSeenVersion { get; set; } = string.Empty; public HashSet OrphanableTempCollections { get; set; } = []; + public AnimationValidationMode AnimationValidationMode { get; set; } = AnimationValidationMode.Safe; } diff --git a/LightlessSync/PlayerData/Factories/AnimationValidationMode.cs b/LightlessSync/PlayerData/Factories/AnimationValidationMode.cs new file mode 100644 index 0000000..ca73117 --- /dev/null +++ b/LightlessSync/PlayerData/Factories/AnimationValidationMode.cs @@ -0,0 +1,9 @@ +namespace LightlessSync.PlayerData.Factories +{ + public enum AnimationValidationMode + { + Unsafe = 0, + Safe = 1, + Safest = 2, + } +} diff --git a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs index 371a2a9..6dc46ba 100644 --- a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs +++ b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs @@ -2,6 +2,7 @@ using LightlessSync.API.Data.Enum; using LightlessSync.FileCache; using LightlessSync.Interop.Ipc; +using LightlessSync.LightlessConfiguration; using LightlessSync.LightlessConfiguration.Models; using LightlessSync.PlayerData.Data; using LightlessSync.PlayerData.Handlers; @@ -20,6 +21,7 @@ public class PlayerDataFactory private readonly IpcManager _ipcManager; private readonly ILogger _logger; private readonly PerformanceCollectorService _performanceCollector; + private readonly LightlessConfigService _configService; private readonly XivDataAnalyzer _modelAnalyzer; private readonly LightlessMediator _lightlessMediator; private readonly TransientResourceManager _transientResourceManager; @@ -45,7 +47,8 @@ public class PlayerDataFactory FileCacheManager fileReplacementFactory, PerformanceCollectorService performanceCollector, XivDataAnalyzer modelAnalyzer, - LightlessMediator lightlessMediator) + LightlessMediator lightlessMediator, + LightlessConfigService configService) { _logger = logger; _dalamudUtil = dalamudUtil; @@ -55,6 +58,7 @@ public class PlayerDataFactory _performanceCollector = performanceCollector; _modelAnalyzer = modelAnalyzer; _lightlessMediator = lightlessMediator; + _configService = configService; _logger.LogTrace("Creating {this}", nameof(PlayerDataFactory)); } private sealed record CacheEntry(CharacterDataFragment Fragment, DateTime CreatedUtc); @@ -338,7 +342,7 @@ public class PlayerDataFactory if (objectKind == ObjectKind.Player) { hasPapFiles = fragment.FileReplacements.Any(f => - !f.IsFileSwap && f.GamePaths.First().EndsWith("pap", StringComparison.OrdinalIgnoreCase)); + !f.IsFileSwap && f.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase))); if (hasPapFiles) { @@ -353,6 +357,7 @@ public class PlayerDataFactory if (hasPapFiles && boneIndices != null) _modelAnalyzer.DumpLocalSkeletonIndices(playerRelatedObject); #endif + if (hasPapFiles) { await VerifyPlayerAnimationBones(boneIndices, (CharacterDataFragmentPlayer)fragment, ct) @@ -458,77 +463,72 @@ public class PlayerDataFactory return (resolved, clearedReplacements); } + private async Task VerifyPlayerAnimationBones( Dictionary>? playerBoneIndices, CharacterDataFragmentPlayer fragment, CancellationToken ct) { + var mode = _configService.Current.AnimationValidationMode; + + if (mode == AnimationValidationMode.Unsafe) + return; + if (playerBoneIndices == null || playerBoneIndices.Count == 0) return; - var playerBoneSets = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var localBoneSets = new Dictionary>(StringComparer.OrdinalIgnoreCase); foreach (var (rawLocalKey, indices) in playerBoneIndices) { - if (indices == null || indices.Count == 0) + if (indices is not { Count: > 0 }) continue; var key = XivDataAnalyzer.CanonicalizeSkeletonKey(rawLocalKey); if (string.IsNullOrEmpty(key)) continue; - if (!playerBoneSets.TryGetValue(key, out var set)) - playerBoneSets[key] = set = []; + if (!localBoneSets.TryGetValue(key, out var set)) + localBoneSets[key] = set = []; foreach (var idx in indices) set.Add(idx); } - if (playerBoneSets.Count == 0) + if (localBoneSets.Count == 0) return; - var papFiles = fragment.FileReplacements - .Where(f => !f.IsFileSwap && f.GamePaths.First().EndsWith("pap", StringComparison.OrdinalIgnoreCase)) - .ToList(); + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug("SEND local buckets: {b}", + string.Join(", ", localBoneSets.Keys.Order(StringComparer.Ordinal))); - if (papFiles.Count == 0) - return; + foreach (var kvp in localBoneSets.OrderBy(k => k.Key, StringComparer.OrdinalIgnoreCase)) + { + var min = kvp.Value.Count > 0 ? kvp.Value.Min() : 0; + var max = kvp.Value.Count > 0 ? kvp.Value.Max() : 0; + _logger.LogDebug("Local bucket {bucket}: count={count} min={min} max={max}", + kvp.Key, kvp.Value.Count, min, max); + } + } - var papGroupsByHash = papFiles - .Where(f => !string.IsNullOrEmpty(f.Hash)) - .GroupBy(f => f.Hash, StringComparer.OrdinalIgnoreCase) + var papGroups = fragment.FileReplacements + .Where(f => !f.IsFileSwap + && !string.IsNullOrEmpty(f.Hash) + && f.GamePaths is { Count: > 0 } + && f.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase))) + .GroupBy(f => f.Hash!, StringComparer.OrdinalIgnoreCase) .ToList(); int noValidationFailed = 0; - static ushort MaxIndex(List list) - { - if (list == null || list.Count == 0) return 0; - ushort max = 0; - for (int i = 0; i < list.Count; i++) - if (list[i] > max) max = list[i]; - return max; - } - - static bool ShouldIgnorePap(Dictionary> pap) - { - foreach (var kv in pap) - { - if (kv.Value == null || kv.Value.Count == 0) - continue; - - if (MaxIndex(kv.Value) > 105) - return false; - } - return true; - } - - foreach (var group in papGroupsByHash) + foreach (var g in papGroups) { ct.ThrowIfCancellationRequested(); - var hash = group.Key; - Dictionary>? papSkeletonIndices; + var hash = g.Key; + + Dictionary>? papSkeletonIndices = null; await _papParseLimiter.WaitAsync(ct).ConfigureAwait(false); try @@ -544,41 +544,36 @@ public class PlayerDataFactory if (papSkeletonIndices == null || papSkeletonIndices.Count == 0) continue; - if (ShouldIgnorePap(papSkeletonIndices)) + if (papSkeletonIndices.All(k => k.Value.DefaultIfEmpty().Max() <= 105)) continue; - bool invalid = false; - string? reason = null; - - foreach (var (rawPapName, usedIndices) in papSkeletonIndices) + if (_logger.IsEnabled(LogLevel.Debug)) { - var papKey = XivDataAnalyzer.CanonicalizeSkeletonKey(rawPapName); - if (string.IsNullOrEmpty(papKey)) - continue; - - if (!playerBoneSets.TryGetValue(papKey, out var available)) - { - invalid = true; - reason = $"Missing skeleton bucket '{papKey}' (raw '{rawPapName}') on local player."; - break; - } - - for (int i = 0; i < usedIndices.Count; i++) - { - var idx = usedIndices[i]; - if (!available.Contains(idx)) + var papBuckets = papSkeletonIndices + .Select(kvp => new { - invalid = true; - reason = $"Skeleton '{papKey}' missing bone index {idx} (raw '{rawPapName}')."; - break; - } - } + Raw = kvp.Key, + Key = XivDataAnalyzer.CanonicalizeSkeletonKey(kvp.Key), + Indices = kvp.Value + }) + .Where(x => x.Indices is { Count: > 0 }) + .GroupBy(x => string.IsNullOrEmpty(x.Key) ? x.Raw : x.Key!, StringComparer.OrdinalIgnoreCase) + .Select(grp => + { + var all = grp.SelectMany(v => v.Indices).ToList(); + var min = all.Count > 0 ? all.Min() : 0; + var max = all.Count > 0 ? all.Max() : 0; + var raws = string.Join(',', grp.Select(v => v.Raw).Distinct(StringComparer.OrdinalIgnoreCase)); + return $"{grp.Key}(min={min},max={max},raw=[{raws}])"; + }) + .ToList(); - if (invalid) - break; + _logger.LogDebug("SEND pap buckets for hash={hash}: {b}", + hash, + string.Join(" | ", papBuckets)); } - if (!invalid) + if (XivDataAnalyzer.IsPapCompatible(localBoneSets, papSkeletonIndices, mode, out var reason)) continue; noValidationFailed++; @@ -588,27 +583,36 @@ public class PlayerDataFactory hash, reason); - foreach (var file in group.ToList()) - { - fragment.FileReplacements.Remove(file); + var removedGamePaths = fragment.FileReplacements + .Where(fr => !fr.IsFileSwap + && string.Equals(fr.Hash, hash, StringComparison.OrdinalIgnoreCase) + && fr.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase))) + .SelectMany(fr => fr.GamePaths.Where(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase))) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); - foreach (var gamePath in file.GamePaths) - _transientResourceManager.RemoveTransientResource(ObjectKind.Player, gamePath); - } + fragment.FileReplacements.RemoveWhere(fr => + !fr.IsFileSwap + && string.Equals(fr.Hash, hash, StringComparison.OrdinalIgnoreCase) + && fr.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase))); + + foreach (var gp in removedGamePaths) + _transientResourceManager.RemoveTransientResource(ObjectKind.Player, gp); } if (noValidationFailed > 0) { _lightlessMediator.Publish(new NotificationMessage( "Invalid Skeleton Setup", - $"Your client is attempting to send {noValidationFailed} animation file groups with bone indices not present on your current skeleton. " + - "Those animation files have been removed from your sent data. Verify that you are using the correct skeleton for those animations " + - "(Check /xllog for more information).", + $"Your client is attempting to send {noValidationFailed} animation files that don't match your current skeleton validation mode ({mode}). " + + "Please adjust your skeleton/mods or change the validation mode if this is unexpected. " + + "Those animation files have been removed from your sent (player) data. (Check /xllog for details).", NotificationType.Warning, TimeSpan.FromSeconds(10))); } } + private async Task> GetFileReplacementsFromPaths( GameObjectHandler handler, HashSet forwardResolve, @@ -681,7 +685,7 @@ public class PlayerDataFactory if (resolvedPaths.TryGetValue(filePath, out var list)) list.AddRange(reverse[i].Select(c => c.ToLowerInvariant())); else - resolvedPaths[filePath] = new List(reverse[i].Select(c => c.ToLowerInvariant()).ToList()); + resolvedPaths[filePath] = [.. reverse[i].Select(c => c.ToLowerInvariant()).ToList()]; } return resolvedPaths.ToDictionary(k => k.Key, k => k.Value.ToArray(), StringComparer.OrdinalIgnoreCase).AsReadOnly(); diff --git a/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs b/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs index e11f290..cdfeaf0 100644 --- a/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs +++ b/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs @@ -22,7 +22,7 @@ using Microsoft.Extensions.Logging; using DalamudObjectKind = Dalamud.Game.ClientState.Objects.Enums.ObjectKind; using ObjectKind = LightlessSync.API.Data.Enum.ObjectKind; using FileReplacementDataComparer = LightlessSync.PlayerData.Data.FileReplacementDataComparer; -using LightlessSync.LightlessConfiguration.Models; +using LightlessSync.LightlessConfiguration; namespace LightlessSync.PlayerData.Pairs; @@ -49,6 +49,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa private readonly PairPerformanceMetricsCache _performanceMetricsCache; private readonly XivDataAnalyzer _modelAnalyzer; private readonly PenumbraTempCollectionJanitor _tempCollectionJanitor; + private readonly LightlessConfigService _configService; private readonly PairManager _pairManager; private CancellationTokenSource? _applicationCancellationTokenSource; private Guid _applicationId; @@ -188,7 +189,8 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa PairStateCache pairStateCache, PairPerformanceMetricsCache performanceMetricsCache, PenumbraTempCollectionJanitor tempCollectionJanitor, - XivDataAnalyzer modelAnalyzer) : base(logger, mediator) + XivDataAnalyzer modelAnalyzer, + LightlessConfigService configService) : base(logger, mediator) { _pairManager = pairManager; Ident = ident; @@ -208,6 +210,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa _performanceMetricsCache = performanceMetricsCache; _tempCollectionJanitor = tempCollectionJanitor; _modelAnalyzer = modelAnalyzer; + _configService = configService; } public void Initialize() @@ -1736,45 +1739,45 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa _needsCollectionRebuild = false; if (LastAppliedApproximateVRAMBytes < 0 || LastAppliedApproximateEffectiveVRAMBytes < 0) { - _playerPerformanceService.ComputeAndAutoPauseOnVRAMUsageThresholds(this, charaData, new List()); - } - if (LastAppliedDataTris < 0) - { - await _playerPerformanceService.CheckTriangleUsageThresholds(this, charaData).ConfigureAwait(false); - } + _playerPerformanceService.ComputeAndAutoPauseOnVRAMUsageThresholds(this, charaData, new List()); + } + if (LastAppliedDataTris < 0) + { + await _playerPerformanceService.CheckTriangleUsageThresholds(this, charaData).ConfigureAwait(false); + } - StorePerformanceMetrics(charaData); - _lastSuccessfulApplyAt = DateTime.UtcNow; - ClearFailureState(); - Logger.LogDebug("[{applicationId}] Application finished", _applicationId); - } - catch (OperationCanceledException) - { - Logger.LogDebug("[{applicationId}] Application cancelled for {handler}", _applicationId, GetLogIdentifier()); - _cachedData = charaData; - _pairStateCache.Store(Ident, charaData); - _forceFullReapply = true; - RecordFailure("Application cancelled", "Cancellation"); - } - catch (Exception ex) - { - if (ex is AggregateException aggr && aggr.InnerExceptions.Any(e => e is ArgumentNullException)) + StorePerformanceMetrics(charaData); + _lastSuccessfulApplyAt = DateTime.UtcNow; + ClearFailureState(); + Logger.LogDebug("[{applicationId}] Application finished", _applicationId); + } + catch (OperationCanceledException) { - IsVisible = false; - _forceApplyMods = true; + Logger.LogDebug("[{applicationId}] Application cancelled for {handler}", _applicationId, GetLogIdentifier()); _cachedData = charaData; _pairStateCache.Store(Ident, charaData); _forceFullReapply = true; - Logger.LogDebug("[{applicationId}] Cancelled, player turned null during application", _applicationId); + RecordFailure("Application cancelled", "Cancellation"); } - else + catch (Exception ex) { - Logger.LogWarning(ex, "[{applicationId}] Cancelled", _applicationId); - _forceFullReapply = true; + if (ex is AggregateException aggr && aggr.InnerExceptions.Any(e => e is ArgumentNullException)) + { + IsVisible = false; + _forceApplyMods = true; + _cachedData = charaData; + _pairStateCache.Store(Ident, charaData); + _forceFullReapply = true; + Logger.LogDebug("[{applicationId}] Cancelled, player turned null during application", _applicationId); + } + else + { + Logger.LogWarning(ex, "[{applicationId}] Cancelled", _applicationId); + _forceFullReapply = true; + } + RecordFailure($"Application failed: {ex.Message}", "Exception"); } - RecordFailure($"Application failed: {ex.Message}", "Exception"); } -} private void FrameworkUpdate() { @@ -2008,12 +2011,27 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa if (string.IsNullOrEmpty(new FileInfo(fileCache.ResolvedFilepath).Extension)) { hasMigrationChanges = true; - fileCache = _fileDbManager.MigrateFileHashToExtension(fileCache, item.GamePaths[0].Split(".")[^1]); + var anyGamePath = item.GamePaths.FirstOrDefault(); + + if (!string.IsNullOrEmpty(anyGamePath)) + { + var ext = Path.GetExtension(anyGamePath); + var extNoDot = ext.StartsWith('.') ? ext[1..] : ext; + + if (!string.IsNullOrEmpty(extNoDot)) + { + hasMigrationChanges = true; + fileCache = _fileDbManager.MigrateFileHashToExtension(fileCache, extNoDot); + } + } } foreach (var gamePath in item.GamePaths) { - if (gamePath.EndsWith(".pap", StringComparison.OrdinalIgnoreCase) + var mode = _configService.Current.AnimationValidationMode; + + if (mode != AnimationValidationMode.Unsafe + && gamePath.EndsWith(".pap", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(item.Hash) && _blockedPapHashes.ContainsKey(item.Hash)) { @@ -2346,100 +2364,6 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa return !string.IsNullOrEmpty(hashedCid); } - private static bool ContainsIndexCompat(HashSet available, ushort idx) - { - if (available.Contains(idx)) return true; - - if (idx > 0 && available.Contains((ushort)(idx - 1))) return true; - if (idx < ushort.MaxValue && available.Contains((ushort)(idx + 1))) return true; - - return false; - } - - private static bool IsPapCompatible( - IReadOnlyDictionary> targetBoneSets, - IReadOnlyDictionary> papBoneIndices, - out string reason) - { - var groups = papBoneIndices - .Select(kvp => new - { - Raw = kvp.Key, - Key = XivDataAnalyzer.CanonicalizeSkeletonKey(kvp.Key), - Indices = kvp.Value - }) - .Where(x => !string.IsNullOrEmpty(x.Key) && x.Indices is { Count: > 0 }) - .GroupBy(x => x.Key, StringComparer.OrdinalIgnoreCase) - .ToList(); - - if (groups.Count == 0) - { - reason = "No bindings found in the PAP"; - return false; - } - - var papKeys = string.Join(", ", groups.Select(g => g.Key).Distinct(StringComparer.OrdinalIgnoreCase)); - var targetKeys = string.Join(", ", targetBoneSets.Keys); - - foreach (var g in groups) - { - var candidates = targetBoneSets - .Where(kvp => string.Equals(kvp.Key, g.Key, StringComparison.OrdinalIgnoreCase)) - .ToList(); - - if (candidates.Count == 0) - { - if (targetBoneSets.Count == 1) - { - candidates = targetBoneSets.ToList(); - } - else - { - reason = $"No matching skeleton bucket between PAP [{papKeys}] and target [{targetKeys}]."; - return false; - } - } - - bool groupOk = false; - string? lastFail = null; - - foreach (var (targetKey, available) in candidates) - { - foreach (var variant in g) - { - bool ok = true; - foreach (var idx in variant.Indices) - { - if (!ContainsIndexCompat(available, idx)) - { - ok = false; - lastFail = $"Target bucket '{targetKey}' missing bone index {idx}. (pap raw '{variant.Raw}')"; - break; - } - } - - if (ok) - { - groupOk = true; - break; - } - } - - if (groupOk) break; - } - - if (!groupOk) - { - reason = lastFail - ?? $"Target skeleton missing required bone indices for PAP bucket '{g.Key}'."; - return false; - } - } - - reason = string.Empty; - return true; - } - private static void SplitPapMappings( Dictionary<(string GamePath, string? Hash), string> moddedPaths, out Dictionary<(string GamePath, string? Hash), string> withoutPap, @@ -2464,7 +2388,8 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa Dictionary<(string GamePath, string? Hash), string> papOnly, CancellationToken token) { - if (papOnly.Count == 0) + var mode = _configService.Current.AnimationValidationMode; + if (mode == AnimationValidationMode.Unsafe || papOnly.Count == 0) return 0; var boneIndices = await _dalamudUtil.RunOnFrameworkThread( @@ -2472,15 +2397,17 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa .ConfigureAwait(false); if (boneIndices == null || boneIndices.Count == 0) - return papOnly.Count; + { + var removedCount = papOnly.Count; + papOnly.Clear(); + return removedCount; + } var localBoneSets = new Dictionary>(StringComparer.OrdinalIgnoreCase); - foreach (var (rawKey, list) in boneIndices) { var key = XivDataAnalyzer.CanonicalizeSkeletonKey(rawKey); - if (string.IsNullOrEmpty(key) || list is null || list.Count == 0) - continue; + if (string.IsNullOrEmpty(key)) continue; if (!localBoneSets.TryGetValue(key, out var set)) localBoneSets[key] = set = []; @@ -2496,8 +2423,8 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa token.ThrowIfCancellationRequested(); var papIndices = await _dalamudUtil.RunOnFrameworkThread( - () => _modelAnalyzer.GetBoneIndicesFromPap(hash!)) - .ConfigureAwait(false); + () => _modelAnalyzer.GetBoneIndicesFromPap(hash!)) + .ConfigureAwait(false); if (papIndices == null || papIndices.Count == 0) continue; @@ -2505,26 +2432,22 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa if (papIndices.All(k => k.Value.DefaultIfEmpty().Max() <= 105)) continue; - if (!IsPapCompatible(localBoneSets, papIndices, out var reason)) + if (XivDataAnalyzer.IsPapCompatible(localBoneSets, papIndices, mode, out var reason)) + continue; + + var keysToRemove = papOnly.Keys.Where(k => string.Equals(k.Hash, hash, StringComparison.OrdinalIgnoreCase)).ToList(); + foreach (var k in keysToRemove) + papOnly.Remove(k); + + removed += keysToRemove.Count; + + if (_blockedPapHashes.TryAdd(hash!, 0)) + Logger.LogWarning("Blocked remote object PAP (hash {hash}) for {handler}: {reason}", hash, GetLogIdentifier(), reason); + + if (charaData.FileReplacements.TryGetValue(ObjectKind.Player, out var list)) { - var keysToRemove = papOnly.Keys.Where(k => string.Equals(k.Hash, hash, StringComparison.OrdinalIgnoreCase)).ToList(); - foreach (var k in keysToRemove) - papOnly.Remove(k); - - removed += keysToRemove.Count; - if (hash == null) - continue; - - if (_blockedPapHashes.TryAdd(hash, 0)) - { - Logger.LogWarning("Blocked remote object PAP (hash {hash}) for {handler}: {reason}", hash, GetLogIdentifier(), reason); - } - - if (charaData.FileReplacements.TryGetValue(ObjectKind.Player, out var list)) - { - list.RemoveAll(r => string.Equals(r.Hash, hash, StringComparison.OrdinalIgnoreCase) - && r.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase))); - } + list.RemoveAll(r => string.Equals(r.Hash, hash, StringComparison.OrdinalIgnoreCase) + && r.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase))); } } diff --git a/LightlessSync/PlayerData/Pairs/PairHandlerAdapterFactory.cs b/LightlessSync/PlayerData/Pairs/PairHandlerAdapterFactory.cs index 3e36006..4b4e95d 100644 --- a/LightlessSync/PlayerData/Pairs/PairHandlerAdapterFactory.cs +++ b/LightlessSync/PlayerData/Pairs/PairHandlerAdapterFactory.cs @@ -1,5 +1,6 @@ using LightlessSync.FileCache; using LightlessSync.Interop.Ipc; +using LightlessSync.LightlessConfiguration; using LightlessSync.PlayerData.Factories; using LightlessSync.Services; using LightlessSync.Services.ActorTracking; @@ -32,6 +33,7 @@ internal sealed class PairHandlerAdapterFactory : IPairHandlerAdapterFactory private readonly PairStateCache _pairStateCache; private readonly PairPerformanceMetricsCache _pairPerformanceMetricsCache; private readonly PenumbraTempCollectionJanitor _tempCollectionJanitor; + private readonly LightlessConfigService _configService; private readonly XivDataAnalyzer _modelAnalyzer; public PairHandlerAdapterFactory( @@ -52,7 +54,8 @@ internal sealed class PairHandlerAdapterFactory : IPairHandlerAdapterFactory PairStateCache pairStateCache, PairPerformanceMetricsCache pairPerformanceMetricsCache, PenumbraTempCollectionJanitor tempCollectionJanitor, - XivDataAnalyzer modelAnalyzer) + XivDataAnalyzer modelAnalyzer, + LightlessConfigService configService) { _loggerFactory = loggerFactory; _mediator = mediator; @@ -72,6 +75,7 @@ internal sealed class PairHandlerAdapterFactory : IPairHandlerAdapterFactory _pairPerformanceMetricsCache = pairPerformanceMetricsCache; _tempCollectionJanitor = tempCollectionJanitor; _modelAnalyzer = modelAnalyzer; + _configService = configService; } public IPairHandlerAdapter Create(string ident) @@ -99,6 +103,7 @@ internal sealed class PairHandlerAdapterFactory : IPairHandlerAdapterFactory _pairStateCache, _pairPerformanceMetricsCache, _tempCollectionJanitor, - _modelAnalyzer); + _modelAnalyzer, + _configService); } } diff --git a/LightlessSync/Services/XivDataAnalyzer.cs b/LightlessSync/Services/XivDataAnalyzer.cs index 95700bb..848fda3 100644 --- a/LightlessSync/Services/XivDataAnalyzer.cs +++ b/LightlessSync/Services/XivDataAnalyzer.cs @@ -6,6 +6,7 @@ using FFXIVClientStructs.Havok.Common.Serialize.Util; using LightlessSync.FileCache; using LightlessSync.Interop.GameModel; using LightlessSync.LightlessConfiguration; +using LightlessSync.PlayerData.Factories; using LightlessSync.PlayerData.Handlers; using Microsoft.Extensions.Logging; using System.Runtime.InteropServices; @@ -13,7 +14,7 @@ using System.Text.RegularExpressions; namespace LightlessSync.Services; -public sealed class XivDataAnalyzer +public sealed partial class XivDataAnalyzer { private readonly ILogger _logger; private readonly FileCacheManager _fileCacheManager; @@ -126,9 +127,12 @@ public sealed class XivDataAnalyzer return (output.Count != 0 && output.Values.All(v => v.Count > 0)) ? output : null; } - public unsafe Dictionary>? GetBoneIndicesFromPap(string hash) + public unsafe Dictionary>? GetBoneIndicesFromPap(string hash, bool persistToConfig = true) { - if (_configService.Current.BonesDictionary.TryGetValue(hash, out var cached)) + if (string.IsNullOrWhiteSpace(hash)) + return null; + + if (_configService.Current.BonesDictionary.TryGetValue(hash, out var cached) && cached is not null) return cached; var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash); @@ -138,47 +142,49 @@ public sealed class XivDataAnalyzer using var fs = File.Open(cacheEntity.ResolvedFilepath, FileMode.Open, FileAccess.Read, FileShare.Read); using var reader = new BinaryReader(fs); - // most of this is from vfxeditor - reader.ReadInt32(); // ignore - reader.ReadInt32(); // ignore - reader.ReadInt16(); // num animations - reader.ReadInt16(); // modelid + // PAP header (mostly from vfxeditor) + _ = reader.ReadInt32(); // ignore + _ = reader.ReadInt32(); // ignore + _ = reader.ReadInt16(); // num animations + _ = reader.ReadInt16(); // modelid + var type = reader.ReadByte(); // type if (type != 0) return null; // not human - reader.ReadByte(); // variant - reader.ReadInt32(); // ignore + _ = reader.ReadByte(); // variant + _ = reader.ReadInt32(); // ignore + var havokPosition = reader.ReadInt32(); var footerPosition = reader.ReadInt32(); + // sanity checks if (havokPosition <= 0 || footerPosition <= havokPosition || footerPosition > fs.Length) return null; - var havokDataSize = footerPosition - havokPosition; - reader.BaseStream.Position = havokPosition; + var havokDataSizeLong = (long)footerPosition - havokPosition; + if (havokDataSizeLong <= 8 || havokDataSizeLong > int.MaxValue) + return null; + var havokDataSize = (int)havokDataSizeLong; + + reader.BaseStream.Position = havokPosition; var havokData = reader.ReadBytes(havokDataSize); if (havokData.Length <= 8) return null; - var output = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var tempSets = new Dictionary>(StringComparer.OrdinalIgnoreCase); - // write to temp file var tempHavokDataPath = Path.Combine(Path.GetTempPath(), $"lightless_{Guid.NewGuid():N}.hkx"); - var tempHavokDataPathAnsi = IntPtr.Zero; + IntPtr tempHavokDataPathAnsi = IntPtr.Zero; try { - using (var tempFs = new FileStream(tempHavokDataPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, 4096, FileOptions.DeleteOnClose)) - { - tempFs.Write(havokData, 0, havokData.Length); - tempFs.Flush(true); - } + File.WriteAllBytes(tempHavokDataPath, havokData); if (!File.Exists(tempHavokDataPath)) { - _logger.LogTrace("Temporary havok file was deleted before it could be loaded: {path}", tempHavokDataPath); + _logger.LogTrace("Temporary havok file did not exist when attempting to load: {path}", tempHavokDataPath); return null; } @@ -228,26 +234,21 @@ public sealed class XivDataAnalyzer if (boneTransform.Length <= 0) continue; - if (!output.TryGetValue(skeletonKey, out var list)) + if (!tempSets.TryGetValue(skeletonKey, out var set)) { - list = new List(boneTransform.Length); - output[skeletonKey] = list; + set = new HashSet(); + tempSets[skeletonKey] = set; } for (int boneIdx = 0; boneIdx < boneTransform.Length; boneIdx++) { - list.Add((ushort)boneTransform[boneIdx]); + var v = boneTransform[boneIdx]; + if (v < 0) continue; + set.Add((ushort)v); } } } } - - foreach (var key in output.Keys.ToList()) - { - output[key] = [.. output[key] - .Distinct() - .Order()]; - } } catch (Exception ex) { @@ -270,20 +271,30 @@ public sealed class XivDataAnalyzer } } + if (tempSets.Count == 0) + return null; + + var output = new Dictionary>(tempSets.Count, StringComparer.OrdinalIgnoreCase); + foreach (var (key, set) in tempSets) + { + if (set.Count == 0) continue; + + var list = set.ToList(); + list.Sort(); + output[key] = list; + } + + if (output.Count == 0) + return null; + _configService.Current.BonesDictionary[hash] = output; - _configService.Save(); + + if (persistToConfig) + _configService.Save(); return output; } - private static readonly Regex _bucketPathRegex = - new(@"(?i)(?:^|/)(?c\d{4})(?:/|$)", RegexOptions.Compiled); - - private static readonly Regex _bucketSklRegex = - new(@"(?i)\bskl_(?c\d{4})[a-z]\d{4}\b", RegexOptions.Compiled); - - private static readonly Regex _bucketLooseRegex = - new(@"(?i)(?c\d{4})(?!\d)", RegexOptions.Compiled); public static string CanonicalizeSkeletonKey(string? raw) { @@ -314,6 +325,159 @@ public sealed class XivDataAnalyzer return string.Empty; } + public static bool ContainsIndexCompat(HashSet available, ushort idx, bool papLikelyOneBased) + { + if (available.Contains(idx)) + return true; + + if (papLikelyOneBased && idx > 0 && available.Contains((ushort)(idx - 1))) + return true; + + return false; + } + + public static bool IsPapCompatible( + IReadOnlyDictionary> localBoneSets, + IReadOnlyDictionary> papBoneIndices, + AnimationValidationMode mode, + out string reason) + { + if (mode == AnimationValidationMode.Unsafe) + { + reason = string.Empty; + return true; + } + + // Group PAP bindings by canonical skeleton key (with raw as fallback) + var groups = papBoneIndices + .Select(kvp => new + { + Raw = kvp.Key, + Key = CanonicalizeSkeletonKey(kvp.Key), + Indices = kvp.Value + }) + .Where(x => x.Indices is { Count: > 0 }) + .GroupBy( + x => string.IsNullOrEmpty(x.Key) ? x.Raw : x.Key, + StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (groups.Count == 0) + { + reason = "No bindings found in the PAP"; + return false; + } + + // Determine relevant groups based on mode + var relevantGroups = groups.AsEnumerable(); + + if (mode == AnimationValidationMode.Safest) + { + relevantGroups = groups.Where(g => localBoneSets.ContainsKey(g.Key)); + + if (!relevantGroups.Any()) + { + var papKeys = string.Join(", ", groups.Select(g => g.Key).Distinct(StringComparer.OrdinalIgnoreCase)); + var localKeys = string.Join(", ", localBoneSets.Keys.Order(StringComparer.OrdinalIgnoreCase)); + reason = $"No matching skeleton bucket between PAP [{papKeys}] and local [{localKeys}]."; + return false; + } + } + + foreach (var g in relevantGroups) + { + // Each group may have multiple variants (different raw names mapping to same canonical key) + bool anyVariantOk = false; + + foreach (var variant in g) + { + // Check this variant against local skeleton(s) + var min = variant.Indices.Min(); + var papLikelyOneBased = min == 1 && !variant.Indices.Contains(0); + + bool variantOk; + + if (mode == AnimationValidationMode.Safest) + { + var available = localBoneSets[g.Key]; + + variantOk = true; + foreach (var idx in variant.Indices) + { + if (!ContainsIndexCompat(available, idx, papLikelyOneBased)) + { + variantOk = false; + break; + } + } + } + else + { + // Safe mode: any local skeleton matching this canonical key + variantOk = false; + + foreach (var available in localBoneSets.Values) + { + bool ok = true; + foreach (var idx in variant.Indices) + { + if (!ContainsIndexCompat(available, idx, papLikelyOneBased)) + { + ok = false; + break; + } + } + + if (ok) + { + variantOk = true; + break; + } + } + } + + if (variantOk) + { + anyVariantOk = true; + break; + } + } + + if (!anyVariantOk) + { + // No variant was compatible for this skeleton key + var first = g.First(); + ushort? missing = null; + + HashSet best; + if (mode == AnimationValidationMode.Safest && localBoneSets.TryGetValue(g.Key, out var exact)) + best = exact; + else + best = localBoneSets.Values.OrderByDescending(s => s.Count).First(); + + var min = first.Indices.Min(); + var papLikelyOneBased = min == 1 && !first.Indices.Contains(0); + + foreach (var idx in first.Indices) + { + if (!ContainsIndexCompat(best, idx, papLikelyOneBased)) + { + missing = idx; + break; + } + } + + reason = missing.HasValue + ? $"Skeleton '{g.Key}' missing bone index {missing.Value}. (raw '{first.Raw}')" + : $"Skeleton '{g.Key}' missing required bone indices. (raw '{first.Raw}')"; + return false; + } + } + + reason = string.Empty; + return true; + } + public void DumpLocalSkeletonIndices(GameObjectHandler handler, string? filter = null) { var skels = GetSkeletonBoneIndices(handler); @@ -408,4 +572,23 @@ public sealed class XivDataAnalyzer return 0; } } + + // Regexes for canonicalizing skeleton keys + private static readonly Regex _bucketPathRegex = + BucketRegex(); + + private static readonly Regex _bucketSklRegex = + SklRegex(); + + private static readonly Regex _bucketLooseRegex = + LooseBucketRegex(); + + [GeneratedRegex(@"(?i)(?:^|/)(?c\d{4})(?:/|$)", RegexOptions.Compiled, "en-NL")] + private static partial Regex BucketRegex(); + + [GeneratedRegex(@"(?i)\bskl_(?c\d{4})[a-z]\d{4}\b", RegexOptions.Compiled, "en-NL")] + private static partial Regex SklRegex(); + + [GeneratedRegex(@"(?i)(?c\d{4})(?!\d)", RegexOptions.Compiled, "en-NL")] + private static partial Regex LooseBucketRegex(); } diff --git a/LightlessSync/UI/SettingsUi.cs b/LightlessSync/UI/SettingsUi.cs index 0efc91a..1ce6753 100644 --- a/LightlessSync/UI/SettingsUi.cs +++ b/LightlessSync/UI/SettingsUi.cs @@ -14,6 +14,7 @@ using LightlessSync.Interop.Ipc; using LightlessSync.LightlessConfiguration; using LightlessSync.LightlessConfiguration.Configurations; using LightlessSync.LightlessConfiguration.Models; +using LightlessSync.PlayerData.Factories; using LightlessSync.PlayerData.Handlers; using LightlessSync.PlayerData.Pairs; using LightlessSync.Services; @@ -40,6 +41,7 @@ using System.Globalization; using System.Net.Http.Headers; using System.Net.Http.Json; using System.Numerics; +using System.Runtime.InteropServices; using System.Text; using System.Text.Json; @@ -105,8 +107,8 @@ public class SettingsUi : WindowMediatorSubscriberBase }; private readonly UiSharedService.TabOption[] _transferTabOptions = new UiSharedService.TabOption[2]; private readonly List> _serverTabOptions = new(4); - private readonly string[] _generalTreeNavOrder = new[] - { + private readonly string[] _generalTreeNavOrder = + [ "Import & Export", "Popup & Auto Fill", "Behavior", @@ -116,7 +118,8 @@ public class SettingsUi : WindowMediatorSubscriberBase "Colors", "Server Info Bar", "Nameplate", - }; + "Animation & Bones" + ]; private static readonly HashSet _generalNavSeparatorAfter = new(StringComparer.Ordinal) { "Popup & Auto Fill", @@ -1141,7 +1144,7 @@ public class SettingsUi : WindowMediatorSubscriberBase private async Task?> RunSpeedTest(List servers, CancellationToken token) { - List speedTestResults = new(); + List speedTestResults = []; foreach (var server in servers) { HttpResponseMessage? result = null; @@ -3085,10 +3088,81 @@ public class SettingsUi : WindowMediatorSubscriberBase } ImGui.Separator(); + ImGui.Dummy(new Vector2(10)); + _uiShared.BigText("Animation"); + + using (var animationTree = BeginGeneralTree("Animation & Bones", UIColors.Get("LightlessPurple"))) + { + if (animationTree.Visible) + { + ImGui.TextUnformatted("Animation Options"); + + var modes = new[] + { + AnimationValidationMode.Unsafe, + AnimationValidationMode.Safe, + AnimationValidationMode.Safest, + }; + + var labels = new[] + { + "Unsafe", + "Safe (Bones)", + "Safest (Race + Bones)", + }; + + var tooltips = new[] + { + "No validation. Fastest, but may allow incompatible animations (riskier).", + "Validates bone indices against your current skeleton (recommended).", + "Requires matching skeleton race + bone compatibility (strictest).", + }; + + + var currentMode = _configService.Current.AnimationValidationMode; + int selectedIndex = Array.IndexOf(modes, currentMode); + if (selectedIndex < 0) selectedIndex = 1; + + ImGui.SetNextItemWidth(140 * ImGuiHelpers.GlobalScale); + + bool open = ImGui.BeginCombo("Animation validation", labels[selectedIndex]); + + if (ImGui.IsItemHovered()) + ImGui.SetTooltip(tooltips[selectedIndex]); + + if (open) + { + for (int i = 0; i < modes.Length; i++) + { + bool isSelected = (i == selectedIndex); + + if (ImGui.Selectable(labels[i], isSelected)) + { + selectedIndex = i; + _configService.Current.AnimationValidationMode = modes[i]; + _configService.Save(); + } + + if (ImGui.IsItemHovered()) + ImGui.SetTooltip(tooltips[i]); + + if (isSelected) + ImGui.SetItemDefaultFocus(); + } + + ImGui.EndCombo(); + } + + UiSharedService.ColoredSeparator(UIColors.Get("LightlessPurple"), 1.5f); + ImGui.TreePop(); + animationTree.MarkContentEnd(); + } + } ImGui.EndChild(); ImGui.EndGroup(); + ImGui.Separator(); generalSelune.DrawHighlightOnly(ImGui.GetIO().DeltaTime); } } @@ -3178,6 +3252,7 @@ public class SettingsUi : WindowMediatorSubscriberBase return 1f - (elapsed / GeneralTreeHighlightDuration); } + [StructLayout(LayoutKind.Auto)] private struct GeneralTreeScope : IDisposable { private readonly bool _visible; @@ -3485,7 +3560,7 @@ public class SettingsUi : WindowMediatorSubscriberBase _uiShared.DrawHelpText("Controls whether Lightless reduces index textures that exceed the size limit."); var dimensionOptions = new[] { 512, 1024, 2048, 4096 }; - var optionLabels = dimensionOptions.Select(static value => value.ToString()).ToArray(); + var optionLabels = dimensionOptions.Select(selector: static value => value.ToString()).ToArray(); var currentDimension = textureConfig.TextureDownscaleMaxDimension; var selectedIndex = Array.IndexOf(dimensionOptions, currentDimension); if (selectedIndex < 0) -- 2.49.1 From 02d091eefa3e9168eca4659affcedfe30ebb6a81 Mon Sep 17 00:00:00 2001 From: cake Date: Sat, 3 Jan 2026 15:59:10 +0100 Subject: [PATCH 11/13] Added more loose matching options, fixed some race issues --- .../Configurations/LightlessConfig.cs | 3 + .../PlayerData/Factories/PlayerDataFactory.cs | 14 +- .../PlayerData/Pairs/PairHandlerAdapter.cs | 8 +- LightlessSync/Services/XivDataAnalyzer.cs | 208 +++++++----------- LightlessSync/UI/SettingsUi.cs | 23 +- 5 files changed, 122 insertions(+), 134 deletions(-) diff --git a/LightlessSync/LightlessConfiguration/Configurations/LightlessConfig.cs b/LightlessSync/LightlessConfiguration/Configurations/LightlessConfig.cs index f38e100..bf6dde1 100644 --- a/LightlessSync/LightlessConfiguration/Configurations/LightlessConfig.cs +++ b/LightlessSync/LightlessConfiguration/Configurations/LightlessConfig.cs @@ -158,4 +158,7 @@ public class LightlessConfig : ILightlessConfiguration public string LastSeenVersion { get; set; } = string.Empty; public HashSet OrphanableTempCollections { get; set; } = []; public AnimationValidationMode AnimationValidationMode { get; set; } = AnimationValidationMode.Safe; + public bool AnimationAllowOneBasedShift { get; set; } = true; + + public bool AnimationAllowNeighborIndexTolerance { get; set; } = false; } diff --git a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs index 6dc46ba..4ef77c9 100644 --- a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs +++ b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs @@ -470,6 +470,8 @@ public class PlayerDataFactory CancellationToken ct) { var mode = _configService.Current.AnimationValidationMode; + var allowBasedShift = _configService.Current.AnimationAllowOneBasedShift; + var allownNightIndex = _configService.Current.AnimationAllowNeighborIndexTolerance; if (mode == AnimationValidationMode.Unsafe) return; @@ -528,12 +530,12 @@ public class PlayerDataFactory var hash = g.Key; - Dictionary>? papSkeletonIndices = null; + Dictionary>? papIndices = null; await _papParseLimiter.WaitAsync(ct).ConfigureAwait(false); try { - papSkeletonIndices = await Task.Run(() => _modelAnalyzer.GetBoneIndicesFromPap(hash), ct) + papIndices = await Task.Run(() => _modelAnalyzer.GetBoneIndicesFromPap(hash), ct) .ConfigureAwait(false); } finally @@ -541,15 +543,15 @@ public class PlayerDataFactory _papParseLimiter.Release(); } - if (papSkeletonIndices == null || papSkeletonIndices.Count == 0) + if (papIndices == null || papIndices.Count == 0) continue; - if (papSkeletonIndices.All(k => k.Value.DefaultIfEmpty().Max() <= 105)) + if (papIndices.All(k => k.Value.DefaultIfEmpty().Max() <= 105)) continue; if (_logger.IsEnabled(LogLevel.Debug)) { - var papBuckets = papSkeletonIndices + var papBuckets = papIndices .Select(kvp => new { Raw = kvp.Key, @@ -573,7 +575,7 @@ public class PlayerDataFactory string.Join(" | ", papBuckets)); } - if (XivDataAnalyzer.IsPapCompatible(localBoneSets, papSkeletonIndices, mode, out var reason)) + if (XivDataAnalyzer.IsPapCompatible(localBoneSets, papIndices, mode, allowBasedShift, allownNightIndex, out var reason)) continue; noValidationFailed++; diff --git a/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs b/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs index cdfeaf0..97ba71a 100644 --- a/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs +++ b/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs @@ -93,7 +93,10 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa ".avfx", ".scd" }; + private readonly ConcurrentDictionary _blockedPapHashes = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _dumpedRemoteSkeletonForHash = new(StringComparer.OrdinalIgnoreCase); + private DateTime? _invisibleSinceUtc; private DateTime? _visibilityEvictionDueAtUtc; private DateTime _nextActorLookupUtc = DateTime.MinValue; @@ -2389,6 +2392,9 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa CancellationToken token) { var mode = _configService.Current.AnimationValidationMode; + var allowBasedShift = _configService.Current.AnimationAllowOneBasedShift; + var allownNightIndex = _configService.Current.AnimationAllowNeighborIndexTolerance; + if (mode == AnimationValidationMode.Unsafe || papOnly.Count == 0) return 0; @@ -2432,7 +2438,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa if (papIndices.All(k => k.Value.DefaultIfEmpty().Max() <= 105)) continue; - if (XivDataAnalyzer.IsPapCompatible(localBoneSets, papIndices, mode, out var reason)) + if (XivDataAnalyzer.IsPapCompatible(localBoneSets, papIndices, mode, allowBasedShift, allownNightIndex, out var reason)) continue; var keysToRemove = papOnly.Keys.Where(k => string.Equals(k.Hash, hash, StringComparison.OrdinalIgnoreCase)).ToList(); diff --git a/LightlessSync/Services/XivDataAnalyzer.cs b/LightlessSync/Services/XivDataAnalyzer.cs index 848fda3..03b5ee9 100644 --- a/LightlessSync/Services/XivDataAnalyzer.cs +++ b/LightlessSync/Services/XivDataAnalyzer.cs @@ -236,7 +236,7 @@ public sealed partial class XivDataAnalyzer if (!tempSets.TryGetValue(skeletonKey, out var set)) { - set = new HashSet(); + set = []; tempSets[skeletonKey] = set; } @@ -325,13 +325,37 @@ public sealed partial class XivDataAnalyzer return string.Empty; } - public static bool ContainsIndexCompat(HashSet available, ushort idx, bool papLikelyOneBased) + public static bool ContainsIndexCompat( + HashSet available, + ushort idx, + bool papLikelyOneBased, + bool allowOneBasedShift, + bool allowNeighborTolerance) { - if (available.Contains(idx)) - return true; + Span candidates = stackalloc ushort[2]; + int count = 0; - if (papLikelyOneBased && idx > 0 && available.Contains((ushort)(idx - 1))) - return true; + candidates[count++] = idx; + + if (allowOneBasedShift && papLikelyOneBased && idx > 0) + candidates[count++] = (ushort)(idx - 1); + + for (int i = 0; i < count; i++) + { + var c = candidates[i]; + + if (available.Contains(c)) + return true; + + if (allowNeighborTolerance) + { + if (c > 0 && available.Contains((ushort)(c - 1))) + return true; + + if (c < ushort.MaxValue && available.Contains((ushort)(c + 1))) + return true; + } + } return false; } @@ -340,141 +364,73 @@ public sealed partial class XivDataAnalyzer IReadOnlyDictionary> localBoneSets, IReadOnlyDictionary> papBoneIndices, AnimationValidationMode mode, + bool allowOneBasedShift, + bool allowNeighborTolerance, out string reason) { - if (mode == AnimationValidationMode.Unsafe) - { - reason = string.Empty; - return true; - } + reason = string.Empty; - // Group PAP bindings by canonical skeleton key (with raw as fallback) - var groups = papBoneIndices - .Select(kvp => new - { - Raw = kvp.Key, - Key = CanonicalizeSkeletonKey(kvp.Key), - Indices = kvp.Value - }) - .Where(x => x.Indices is { Count: > 0 }) - .GroupBy( - x => string.IsNullOrEmpty(x.Key) ? x.Raw : x.Key, - StringComparer.OrdinalIgnoreCase) + if (mode == AnimationValidationMode.Unsafe) + return true; + + var papBuckets = papBoneIndices.Keys + .Select(CanonicalizeSkeletonKey) + .Where(k => !string.IsNullOrEmpty(k)) + .Distinct(StringComparer.OrdinalIgnoreCase) .ToList(); - if (groups.Count == 0) + if (papBuckets.Count == 0) { - reason = "No bindings found in the PAP"; + reason = "No skeleton bucket bindings found in the PAP"; return false; } - // Determine relevant groups based on mode - var relevantGroups = groups.AsEnumerable(); - - if (mode == AnimationValidationMode.Safest) + if (mode == AnimationValidationMode.Safe) { - relevantGroups = groups.Where(g => localBoneSets.ContainsKey(g.Key)); + if (papBuckets.Any(b => localBoneSets.ContainsKey(b))) + return true; - if (!relevantGroups.Any()) + reason = $"No matching skeleton bucket between PAP [{string.Join(", ", papBuckets)}] and local [{string.Join(", ", localBoneSets.Keys.Order())}]."; + return false; + } + + foreach (var bucket in papBuckets) + { + if (!localBoneSets.TryGetValue(bucket, out var available)) { - var papKeys = string.Join(", ", groups.Select(g => g.Key).Distinct(StringComparer.OrdinalIgnoreCase)); - var localKeys = string.Join(", ", localBoneSets.Keys.Order(StringComparer.OrdinalIgnoreCase)); - reason = $"No matching skeleton bucket between PAP [{papKeys}] and local [{localKeys}]."; + reason = $"Missing skeleton bucket '{bucket}' on local actor."; return false; } + + var indices = papBoneIndices + .Where(kvp => string.Equals(CanonicalizeSkeletonKey(kvp.Key), bucket, StringComparison.OrdinalIgnoreCase)) + .SelectMany(kvp => kvp.Value ?? Enumerable.Empty()) + .Distinct() + .ToList(); + + if (indices.Count == 0) + continue; + + bool has0 = false, has1 = false; + ushort min = ushort.MaxValue; + foreach (var v in indices) + { + if (v == 0) has0 = true; + if (v == 1) has1 = true; + if (v < min) min = v; + } + bool papLikelyOneBased = allowOneBasedShift && (min == 1) && has1 && !has0; + + foreach (var idx in indices) + { + if (!ContainsIndexCompat(available, idx, papLikelyOneBased, allowOneBasedShift, allowNeighborTolerance)) + { + reason = $"No compatible local skeleton for PAP '{bucket}': missing bone index {idx}."; + return false; + } + } } - foreach (var g in relevantGroups) - { - // Each group may have multiple variants (different raw names mapping to same canonical key) - bool anyVariantOk = false; - - foreach (var variant in g) - { - // Check this variant against local skeleton(s) - var min = variant.Indices.Min(); - var papLikelyOneBased = min == 1 && !variant.Indices.Contains(0); - - bool variantOk; - - if (mode == AnimationValidationMode.Safest) - { - var available = localBoneSets[g.Key]; - - variantOk = true; - foreach (var idx in variant.Indices) - { - if (!ContainsIndexCompat(available, idx, papLikelyOneBased)) - { - variantOk = false; - break; - } - } - } - else - { - // Safe mode: any local skeleton matching this canonical key - variantOk = false; - - foreach (var available in localBoneSets.Values) - { - bool ok = true; - foreach (var idx in variant.Indices) - { - if (!ContainsIndexCompat(available, idx, papLikelyOneBased)) - { - ok = false; - break; - } - } - - if (ok) - { - variantOk = true; - break; - } - } - } - - if (variantOk) - { - anyVariantOk = true; - break; - } - } - - if (!anyVariantOk) - { - // No variant was compatible for this skeleton key - var first = g.First(); - ushort? missing = null; - - HashSet best; - if (mode == AnimationValidationMode.Safest && localBoneSets.TryGetValue(g.Key, out var exact)) - best = exact; - else - best = localBoneSets.Values.OrderByDescending(s => s.Count).First(); - - var min = first.Indices.Min(); - var papLikelyOneBased = min == 1 && !first.Indices.Contains(0); - - foreach (var idx in first.Indices) - { - if (!ContainsIndexCompat(best, idx, papLikelyOneBased)) - { - missing = idx; - break; - } - } - - reason = missing.HasValue - ? $"Skeleton '{g.Key}' missing bone index {missing.Value}. (raw '{first.Raw}')" - : $"Skeleton '{g.Key}' missing required bone indices. (raw '{first.Raw}')"; - return false; - } - } - - reason = string.Empty; return true; } @@ -486,7 +442,7 @@ public sealed partial class XivDataAnalyzer _logger.LogTrace("DumpLocalSkeletonIndices: local skeleton indices are null or not found"); return; } - + var keys = skels.Keys .Order(StringComparer.OrdinalIgnoreCase) .ToArray(); diff --git a/LightlessSync/UI/SettingsUi.cs b/LightlessSync/UI/SettingsUi.cs index 1ce6753..3102e13 100644 --- a/LightlessSync/UI/SettingsUi.cs +++ b/LightlessSync/UI/SettingsUi.cs @@ -3114,7 +3114,7 @@ public class SettingsUi : WindowMediatorSubscriberBase var tooltips = new[] { "No validation. Fastest, but may allow incompatible animations (riskier).", - "Validates bone indices against your current skeleton (recommended).", + "Validates skeleton race (recommended).", "Requires matching skeleton race + bone compatibility (strictest).", }; @@ -3154,6 +3154,27 @@ public class SettingsUi : WindowMediatorSubscriberBase } UiSharedService.ColoredSeparator(UIColors.Get("LightlessPurple"), 1.5f); + + var cfg = _configService.Current; + + bool oneBased = cfg.AnimationAllowOneBasedShift; + if (ImGui.Checkbox("Treat 1-based PAP indices as compatible", ref oneBased)) + { + cfg.AnimationAllowOneBasedShift = oneBased; + _configService.Save(); + } + if (ImGui.IsItemHovered()) + ImGui.SetTooltip("Fixes off-by-one PAPs (one bone differance in bones and PAP). Can also increase crashing, toggle off if alot of crashing is happening"); + + bool neighbor = cfg.AnimationAllowNeighborIndexTolerance; + if (ImGui.Checkbox("Allow 1+- bone index tolerance", ref neighbor)) + { + cfg.AnimationAllowNeighborIndexTolerance = neighbor; + _configService.Save(); + } + if (ImGui.IsItemHovered()) + ImGui.SetTooltip("Looser matching on bone matching. Can reduce false blocks happening but also reduces safety and more prone to crashing."); + ImGui.TreePop(); animationTree.MarkContentEnd(); } -- 2.49.1 From 6af61451dca311ad1b3e5f26d917549c9f54dd8e Mon Sep 17 00:00:00 2001 From: cake Date: Sat, 3 Jan 2026 16:32:39 +0100 Subject: [PATCH 12/13] Fixed naming of setting --- LightlessSync/UI/SettingsUi.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LightlessSync/UI/SettingsUi.cs b/LightlessSync/UI/SettingsUi.cs index 3102e13..d80bb35 100644 --- a/LightlessSync/UI/SettingsUi.cs +++ b/LightlessSync/UI/SettingsUi.cs @@ -3107,14 +3107,14 @@ public class SettingsUi : WindowMediatorSubscriberBase var labels = new[] { "Unsafe", - "Safe (Bones)", + "Safe (Race)", "Safest (Race + Bones)", }; var tooltips = new[] { "No validation. Fastest, but may allow incompatible animations (riskier).", - "Validates skeleton race (recommended).", + "Validates skeleton race + modded skeleton check (recommended).", "Requires matching skeleton race + bone compatibility (strictest).", }; -- 2.49.1 From 11099c05ffb5f45da8639c0b87a504339f8f68ab Mon Sep 17 00:00:00 2001 From: defnotken Date: Sat, 3 Jan 2026 17:28:05 -0600 Subject: [PATCH 13/13] Throwing analysis in background task so dalamud can breath --- LightlessSync/Services/CharacterAnalyzer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LightlessSync/Services/CharacterAnalyzer.cs b/LightlessSync/Services/CharacterAnalyzer.cs index 959ece3..58388ae 100644 --- a/LightlessSync/Services/CharacterAnalyzer.cs +++ b/LightlessSync/Services/CharacterAnalyzer.cs @@ -28,7 +28,7 @@ public sealed class CharacterAnalyzer : MediatorSubscriberBase, IDisposable { _baseAnalysisCts = _baseAnalysisCts.CancelRecreate(); var token = _baseAnalysisCts.Token; - _ = BaseAnalysis(msg.CharacterData, token); + _ = Task.Run(async () => await BaseAnalysis(msg.CharacterData, token).ConfigureAwait(false), token); }); _fileCacheManager = fileCacheManager; _xivDataAnalyzer = modelAnalyzer; -- 2.49.1