using FFXIVClientStructs.FFXIV.Client.Game.Character; using LightlessSync.API.Data.Enum; using LightlessSync.FileCache; using LightlessSync.Interop.Ipc; using LightlessSync.LightlessConfiguration.Models; using LightlessSync.PlayerData.Data; using LightlessSync.PlayerData.Handlers; using LightlessSync.Services; using LightlessSync.Services.Mediator; using Microsoft.Extensions.Logging; namespace LightlessSync.PlayerData.Factories; public class PlayerDataFactory { private readonly DalamudUtilService _dalamudUtil; private readonly FileCacheManager _fileCacheManager; private readonly IpcManager _ipcManager; private readonly ILogger _logger; private readonly PerformanceCollectorService _performanceCollector; private readonly XivDataAnalyzer _modelAnalyzer; private readonly LightlessMediator _lightlessMediator; private readonly TransientResourceManager _transientResourceManager; private static readonly SemaphoreSlim _papParseLimiter = new(1, 1); public PlayerDataFactory(ILogger logger, DalamudUtilService dalamudUtil, IpcManager ipcManager, TransientResourceManager transientResourceManager, FileCacheManager fileReplacementFactory, PerformanceCollectorService performanceCollector, XivDataAnalyzer modelAnalyzer, LightlessMediator lightlessMediator) { _logger = logger; _dalamudUtil = dalamudUtil; _ipcManager = ipcManager; _transientResourceManager = transientResourceManager; _fileCacheManager = fileReplacementFactory; _performanceCollector = performanceCollector; _modelAnalyzer = modelAnalyzer; _lightlessMediator = lightlessMediator; _logger.LogTrace("Creating {this}", nameof(PlayerDataFactory)); } public async Task BuildCharacterData(GameObjectHandler playerRelatedObject, CancellationToken token) { if (!_ipcManager.Initialized) { throw new InvalidOperationException("Penumbra or Glamourer is not connected"); } if (playerRelatedObject == null) return null; bool pointerIsZero = true; try { pointerIsZero = playerRelatedObject.Address == IntPtr.Zero; try { pointerIsZero = await CheckForNullDrawObject(playerRelatedObject.Address).ConfigureAwait(false); } catch { pointerIsZero = true; _logger.LogDebug("NullRef for {object}", playerRelatedObject); } } catch (Exception ex) { _logger.LogWarning(ex, "Could not create data for {object}", playerRelatedObject); } if (pointerIsZero) { _logger.LogTrace("Pointer was zero for {objectKind}", playerRelatedObject.ObjectKind); return null; } try { return await _performanceCollector.LogPerformance(this, $"CreateCharacterData>{playerRelatedObject.ObjectKind}", async () => { return await CreateCharacterData(playerRelatedObject, token).ConfigureAwait(false); }).ConfigureAwait(true); } catch (OperationCanceledException) { _logger.LogDebug("Cancelled creating Character data for {object}", playerRelatedObject); throw; } catch (Exception e) { _logger.LogWarning(e, "Failed to create {object} data", playerRelatedObject); } return null; } private async Task CheckForNullDrawObject(IntPtr playerPointer) { return await _dalamudUtil.RunOnFrameworkThread(() => CheckForNullDrawObjectUnsafe(playerPointer)).ConfigureAwait(false); } private unsafe bool CheckForNullDrawObjectUnsafe(IntPtr playerPointer) { if (playerPointer == IntPtr.Zero) return true; var character = (Character*)playerPointer; if (character == null) return true; var gameObject = &character->GameObject; if (gameObject == null) return true; return gameObject->DrawObject == null; } private async Task CreateCharacterData(GameObjectHandler playerRelatedObject, CancellationToken ct) { var objectKind = playerRelatedObject.ObjectKind; CharacterDataFragment fragment = objectKind == ObjectKind.Player ? new CharacterDataFragmentPlayer() : new(); _logger.LogDebug("Building character data for {obj}", playerRelatedObject); var logDebug = _logger.IsEnabled(LogLevel.Debug); await _dalamudUtil.WaitWhileCharacterIsDrawing(_logger, playerRelatedObject, Guid.NewGuid(), 30000, ct: ct).ConfigureAwait(false); int totalWaitTime = 10000; while (!await _dalamudUtil.IsObjectPresentAsync(await _dalamudUtil.CreateGameObjectAsync(playerRelatedObject.Address).ConfigureAwait(false)).ConfigureAwait(false) && totalWaitTime > 0) { _logger.LogTrace("Character is null but it shouldn't be, waiting"); await Task.Delay(50, ct).ConfigureAwait(false); totalWaitTime -= 50; } ct.ThrowIfCancellationRequested(); DateTime start = DateTime.UtcNow; Dictionary>? resolvedPaths; resolvedPaths = (await _ipcManager.Penumbra.GetCharacterData(_logger, playerRelatedObject).ConfigureAwait(false)); if (resolvedPaths == null) throw new InvalidOperationException("Penumbra returned null data"); ct.ThrowIfCancellationRequested(); fragment.FileReplacements = [.. new HashSet(resolvedPaths.Select(c => new FileReplacement([.. c.Value], c.Key)), FileReplacementComparer.Instance).Where(p => p.HasFileReplacement)]; fragment.FileReplacements.RemoveWhere(c => c.GamePaths.Any(g => !CacheMonitor.AllowedFileExtensions.Any(e => g.EndsWith(e, StringComparison.OrdinalIgnoreCase)))); ct.ThrowIfCancellationRequested(); if (logDebug) { _logger.LogDebug("== Static Replacements =="); foreach (var replacement in fragment.FileReplacements.Where(i => i.HasFileReplacement).OrderBy(i => i.GamePaths.First(), StringComparer.OrdinalIgnoreCase)) { _logger.LogDebug("=> {repl}", replacement); ct.ThrowIfCancellationRequested(); } } else { foreach (var replacement in fragment.FileReplacements.Where(i => i.HasFileReplacement)) { ct.ThrowIfCancellationRequested(); } } await _transientResourceManager.WaitForRecording(ct).ConfigureAwait(false); if (objectKind == ObjectKind.Pet) { foreach (var item in fragment.FileReplacements.Where(i => i.HasFileReplacement).SelectMany(p => p.GamePaths)) { if (_transientResourceManager.AddTransientResource(objectKind, item)) { _logger.LogDebug("Marking static {item} for Pet as transient", item); } } _logger.LogTrace("Clearing {count} Static Replacements for Pet", fragment.FileReplacements.Count); fragment.FileReplacements.Clear(); } ct.ThrowIfCancellationRequested(); _logger.LogDebug("Handling transient update for {obj}", playerRelatedObject); _transientResourceManager.ClearTransientPaths(objectKind, [.. fragment.FileReplacements.SelectMany(c => c.GamePaths)]); var transientPaths = ManageSemiTransientData(objectKind); var resolvedTransientPaths = await GetFileReplacementsFromPaths(playerRelatedObject, transientPaths, new HashSet(StringComparer.Ordinal)).ConfigureAwait(false); if (logDebug) { _logger.LogDebug("== Transient Replacements =="); foreach (var replacement in resolvedTransientPaths.Select(c => new FileReplacement([.. c.Value], c.Key)).OrderBy(f => f.ResolvedPath, StringComparer.Ordinal)) { _logger.LogDebug("=> {repl}", replacement); fragment.FileReplacements.Add(replacement); } } else { foreach (var replacement in resolvedTransientPaths.Select(c => new FileReplacement([.. c.Value], c.Key))) { fragment.FileReplacements.Add(replacement); } } _transientResourceManager.CleanUpSemiTransientResources(objectKind, [.. fragment.FileReplacements]); ct.ThrowIfCancellationRequested(); fragment.FileReplacements = new HashSet(fragment.FileReplacements.Where(v => v.HasFileReplacement).OrderBy(v => v.ResolvedPath, StringComparer.Ordinal), FileReplacementComparer.Instance); // gather up data from ipc Task getHeelsOffset = _ipcManager.Heels.GetOffsetAsync(); Task getGlamourerData = _ipcManager.Glamourer.GetCharacterCustomizationAsync(playerRelatedObject.Address); Task getCustomizeData = _ipcManager.CustomizePlus.GetScaleAsync(playerRelatedObject.Address); Task getHonorificTitle = _ipcManager.Honorific.GetTitle(); fragment.GlamourerString = await getGlamourerData.ConfigureAwait(false); _logger.LogDebug("Glamourer is now: {data}", fragment.GlamourerString); var customizeScale = await getCustomizeData.ConfigureAwait(false); fragment.CustomizePlusScale = customizeScale ?? string.Empty; _logger.LogDebug("Customize is now: {data}", fragment.CustomizePlusScale); if (objectKind == ObjectKind.Player) { var playerFragment = (fragment as CharacterDataFragmentPlayer)!; playerFragment.ManipulationString = _ipcManager.Penumbra.GetMetaManipulations(); playerFragment!.HonorificData = await getHonorificTitle.ConfigureAwait(false); _logger.LogDebug("Honorific is now: {data}", playerFragment!.HonorificData); playerFragment!.HeelsData = await getHeelsOffset.ConfigureAwait(false); _logger.LogDebug("Heels is now: {heels}", playerFragment!.HeelsData); playerFragment!.MoodlesData = await _ipcManager.Moodles.GetStatusAsync(playerRelatedObject.Address).ConfigureAwait(false) ?? string.Empty; _logger.LogDebug("Moodles is now: {moodles}", playerFragment!.MoodlesData); playerFragment!.PetNamesData = _ipcManager.PetNames.GetLocalNames(); _logger.LogDebug("Pet Nicknames is now: {petnames}", playerFragment!.PetNamesData); } ct.ThrowIfCancellationRequested(); var toCompute = fragment.FileReplacements.Where(f => !f.IsFileSwap).ToArray(); _logger.LogDebug("Getting Hashes for {amount} Files", toCompute.Length); var computedPaths = _fileCacheManager.GetFileCachesByPaths(toCompute.Select(c => c.ResolvedPath).ToArray()); foreach (var file in toCompute) { ct.ThrowIfCancellationRequested(); file.Hash = computedPaths[file.ResolvedPath]?.Hash ?? string.Empty; } var removed = fragment.FileReplacements.RemoveWhere(f => !f.IsFileSwap && string.IsNullOrEmpty(f.Hash)); if (removed > 0) { _logger.LogDebug("Removed {amount} of invalid files", removed); } ct.ThrowIfCancellationRequested(); Dictionary>? boneIndices = null; var hasPapFiles = false; if (objectKind == ObjectKind.Player) { hasPapFiles = fragment.FileReplacements.Any(f => !f.IsFileSwap && f.GamePaths.First().EndsWith("pap", StringComparison.OrdinalIgnoreCase)); if (hasPapFiles) { boneIndices = await _dalamudUtil .RunOnFrameworkThread(() => _modelAnalyzer.GetSkeletonBoneIndices(playerRelatedObject)) .ConfigureAwait(false); } } if (objectKind == ObjectKind.Player) { try { #if DEBUG if (hasPapFiles && boneIndices != null) { _modelAnalyzer.DumpLocalSkeletonIndices(playerRelatedObject); } #endif if (hasPapFiles) { await VerifyPlayerAnimationBones(boneIndices, (fragment as CharacterDataFragmentPlayer)!, ct) .ConfigureAwait(false); } } catch (OperationCanceledException e) { _logger.LogDebug(e, "Cancelled during player animation verification"); throw; } catch (Exception e) { _logger.LogWarning(e, "Failed to verify player animations, continuing without further verification"); } } _logger.LogInformation("Building character data for {obj} took {time}ms", objectKind, TimeSpan.FromTicks(DateTime.UtcNow.Ticks - start.Ticks).TotalMilliseconds); return fragment; } private async Task VerifyPlayerAnimationBones( Dictionary>? playerBoneIndices, CharacterDataFragmentPlayer fragment, CancellationToken ct) { if (playerBoneIndices == null || playerBoneIndices.Count == 0) return; var playerBoneSets = new Dictionary>(StringComparer.OrdinalIgnoreCase); foreach (var (rawLocalKey, indices) in playerBoneIndices) { if (indices == null || indices.Count == 0) continue; var key = XivDataAnalyzer.CanonicalizeSkeletonKey(rawLocalKey); if (string.IsNullOrEmpty(key)) continue; if (!playerBoneSets.TryGetValue(key, out var set)) playerBoneSets[key] = set = new HashSet(); foreach (var idx in indices) set.Add(idx); } if (playerBoneSets.Count == 0) return; if (_logger.IsEnabled(LogLevel.Debug)) { foreach (var kvp in playerBoneSets) { _logger.LogDebug( "Found local skeleton bucket '{bucket}' ({count} indices, max {max})", kvp.Key, kvp.Value.Count, kvp.Value.Count > 0 ? kvp.Value.Max() : 0); } } var papFiles = fragment.FileReplacements .Where(f => !f.IsFileSwap && f.GamePaths.First().EndsWith("pap", StringComparison.OrdinalIgnoreCase)) .ToList(); if (papFiles.Count == 0) return; var papGroupsByHash = papFiles .Where(f => !string.IsNullOrEmpty(f.Hash)) .GroupBy(f => f.Hash, StringComparer.OrdinalIgnoreCase) .ToList(); int noValidationFailed = 0; static ushort MaxIndex(List list) { if (list == null || list.Count == 0) return 0; ushort max = 0; for (int i = 0; i < list.Count; i++) if (list[i] > max) max = list[i]; return max; } static bool ShouldIgnorePap(Dictionary> pap) { foreach (var kv in pap) { if (kv.Value == null || kv.Value.Count == 0) continue; if (MaxIndex(kv.Value) > 105) return false; } return true; } foreach (var group in papGroupsByHash) { ct.ThrowIfCancellationRequested(); var hash = group.Key; Dictionary>? papSkeletonIndices; await _papParseLimiter.WaitAsync(ct).ConfigureAwait(false); try { papSkeletonIndices = await Task.Run(() => _modelAnalyzer.GetBoneIndicesFromPap(hash), ct) .ConfigureAwait(false); } finally { _papParseLimiter.Release(); } if (papSkeletonIndices == null || papSkeletonIndices.Count == 0) continue; if (ShouldIgnorePap(papSkeletonIndices)) { _logger.LogTrace("All indices of PAP hash {hash} are <= 105, ignoring", hash); continue; } bool invalid = false; string? reason = null; foreach (var (rawPapName, usedIndices) in papSkeletonIndices) { var papKey = XivDataAnalyzer.CanonicalizeSkeletonKey(rawPapName); if (string.IsNullOrEmpty(papKey)) continue; if (!playerBoneSets.TryGetValue(papKey, out var available)) { invalid = true; reason = $"Missing skeleton bucket '{papKey}' (raw '{rawPapName}') on local player."; break; } for (int i = 0; i < usedIndices.Count; i++) { var idx = usedIndices[i]; if (!available.Contains(idx)) { invalid = true; reason = $"Skeleton '{papKey}' missing bone index {idx} (raw '{rawPapName}')."; break; } } if (invalid) break; } if (!invalid) continue; noValidationFailed++; _logger.LogWarning( "Animation PAP hash {hash} is not compatible with local skeletons; dropping all mappings for this hash. Reason: {reason}", hash, reason); foreach (var file in group.ToList()) { _logger.LogDebug("Removing {file} from sent file replacements and transient data", file.ResolvedPath); fragment.FileReplacements.Remove(file); foreach (var gamePath in file.GamePaths) _transientResourceManager.RemoveTransientResource(ObjectKind.Player, gamePath); } } if (noValidationFailed > 0) { _lightlessMediator.Publish(new NotificationMessage( "Invalid Skeleton Setup", $"Your client is attempting to send {noValidationFailed} animation file groups with bone indices not present on your current skeleton. " + "Those animation files have been removed from your sent data. Verify that you are using the correct skeleton for those animations " + "(Check /xllog for more information).", NotificationType.Warning, TimeSpan.FromSeconds(10))); } } private async Task> GetFileReplacementsFromPaths(GameObjectHandler handler, HashSet forwardResolve, HashSet reverseResolve) { var forwardPaths = forwardResolve.ToArray(); var reversePaths = reverseResolve.ToArray(); Dictionary> resolvedPaths = new(StringComparer.Ordinal); if (handler.ObjectKind != ObjectKind.Player) { var (objectIndex, forwardResolved, reverseResolved) = await _dalamudUtil.RunOnFrameworkThread(() => { var idx = handler.GetGameObject()?.ObjectIndex; if (!idx.HasValue) { return ((int?)null, Array.Empty(), Array.Empty()); } var resolvedForward = new string[forwardPaths.Length]; for (int i = 0; i < forwardPaths.Length; i++) { resolvedForward[i] = _ipcManager.Penumbra.ResolveGameObjectPath(forwardPaths[i], idx.Value); } var resolvedReverse = new string[reversePaths.Length][]; for (int i = 0; i < reversePaths.Length; i++) { resolvedReverse[i] = _ipcManager.Penumbra.ReverseResolveGameObjectPath(reversePaths[i], idx.Value); } return (idx, resolvedForward, resolvedReverse); }).ConfigureAwait(false); if (objectIndex.HasValue) { for (int i = 0; i < forwardPaths.Length; i++) { var filePath = forwardResolved[i]?.ToLowerInvariant(); if (string.IsNullOrEmpty(filePath)) { continue; } if (resolvedPaths.TryGetValue(filePath, out var list)) { list.Add(forwardPaths[i].ToLowerInvariant()); } else { resolvedPaths[filePath] = [forwardPaths[i].ToLowerInvariant()]; } } for (int i = 0; i < reversePaths.Length; i++) { var filePath = reversePaths[i].ToLowerInvariant(); if (resolvedPaths.TryGetValue(filePath, out var list)) { list.AddRange(reverseResolved[i].Select(c => c.ToLowerInvariant())); } else { resolvedPaths[filePath] = new List(reverseResolved[i].Select(c => c.ToLowerInvariant()).ToList()); } } return resolvedPaths.ToDictionary(k => k.Key, k => k.Value.ToArray(), StringComparer.OrdinalIgnoreCase).AsReadOnly(); } } var (forward, reverse) = await _ipcManager.Penumbra.ResolvePathsAsync(forwardPaths, reversePaths).ConfigureAwait(false); for (int i = 0; i < forwardPaths.Length; i++) { var filePath = forward[i].ToLowerInvariant(); if (resolvedPaths.TryGetValue(filePath, out var list)) { list.Add(forwardPaths[i].ToLowerInvariant()); } else { resolvedPaths[filePath] = [forwardPaths[i].ToLowerInvariant()]; } } for (int i = 0; i < reversePaths.Length; i++) { var filePath = reversePaths[i].ToLowerInvariant(); if (resolvedPaths.TryGetValue(filePath, out var list)) { list.AddRange(reverse[i].Select(c => c.ToLowerInvariant())); } else { resolvedPaths[filePath] = new List(reverse[i].Select(c => c.ToLowerInvariant()).ToList()); } } return resolvedPaths.ToDictionary(k => k.Key, k => k.Value.ToArray(), StringComparer.OrdinalIgnoreCase).AsReadOnly(); } private HashSet ManageSemiTransientData(ObjectKind objectKind) { _transientResourceManager.PersistTransientResources(objectKind); HashSet pathsToResolve = new(StringComparer.Ordinal); foreach (var path in _transientResourceManager.GetSemiTransientResources(objectKind).Where(path => !string.IsNullOrEmpty(path))) { pathsToResolve.Add(path); } return pathsToResolve; } }