using FFXIVClientStructs.FFXIV.Client.Game.Character; using FFXIVClientStructs.FFXIV.Client.Graphics.Scene; using FFXIVClientStructs.Havok.Common.Serialize.Resource; using FFXIVClientStructs.Havok.Animation; using FFXIVClientStructs.Havok.Common.Base.Types; using FFXIVClientStructs.Havok.Common.Serialize.Util; using LightlessSync.FileCache; using LightlessSync.Interop.GameModel; using LightlessSync.LightlessConfiguration; using LightlessSync.PlayerData.Factories; using LightlessSync.PlayerData.Handlers; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Runtime.InteropServices; using System.Text.RegularExpressions; namespace LightlessSync.Services; public sealed partial class XivDataAnalyzer { private readonly ILogger _logger; private readonly FileCacheManager _fileCacheManager; private readonly XivDataStorageService _configService; private readonly List _failedCalculatedTris = []; private readonly List _failedCalculatedEffectiveTris = []; public XivDataAnalyzer(ILogger logger, FileCacheManager fileCacheManager, XivDataStorageService configService) { _logger = logger; _fileCacheManager = fileCacheManager; _configService = configService; } public unsafe Dictionary>? GetSkeletonBoneIndices(GameObjectHandler handler) { if (handler is null || handler.Address == nint.Zero) return null; Dictionary> sets = new(StringComparer.OrdinalIgnoreCase); try { var drawObject = ((Character*)handler.Address)->GameObject.DrawObject; if (drawObject == null) return null; var chara = (CharacterBase*)drawObject; if (chara->GetModelType() != CharacterBase.ModelType.Human) return null; var skeleton = chara->Skeleton; if (skeleton == null) return null; var resHandles = skeleton->SkeletonResourceHandles; var partialCount = skeleton->PartialSkeletonCount; if (partialCount <= 0) return null; for (int i = 0; i < partialCount; i++) { var handle = *(resHandles + i); if ((nint)handle == nint.Zero) continue; if (handle->FileName.Length > 1024) continue; var rawName = handle->FileName.ToString(); if (string.IsNullOrWhiteSpace(rawName)) continue; var skeletonKey = CanonicalizeSkeletonKey(rawName); if (string.IsNullOrEmpty(skeletonKey)) continue; var boneCount = handle->BoneCount; if (boneCount == 0) continue; var havokSkel = handle->HavokSkeleton; if ((nint)havokSkel == nint.Zero) continue; if (!sets.TryGetValue(skeletonKey, out var set)) { set = []; sets[skeletonKey] = set; } uint maxExclusive = boneCount; uint ushortExclusive = (uint)ushort.MaxValue + 1u; if (maxExclusive > ushortExclusive) maxExclusive = ushortExclusive; for (uint boneIdx = 0; boneIdx < maxExclusive; boneIdx++) { var name = havokSkel->Bones[boneIdx].Name.String; if (name == null) continue; set.Add((ushort)boneIdx); } _logger.LogTrace("Local skeleton raw file='{raw}', key='{key}', boneCount={count}", rawName, skeletonKey, boneCount); } } catch (Exception ex) { _logger.LogWarning(ex, "Could not process skeleton data"); return null; } if (sets.Count == 0) return null; var output = new Dictionary>(sets.Count, StringComparer.OrdinalIgnoreCase); foreach (var (key, set) in sets) { if (set.Count == 0) continue; var list = set.ToList(); list.Sort(); output[key] = list; } return (output.Count != 0 && output.Values.All(v => v.Count > 0)) ? output : null; } public unsafe Dictionary>? GetBoneIndicesFromPap(string hash, bool persistToConfig = true) { if (string.IsNullOrWhiteSpace(hash)) return null; if (_configService.Current.BonesDictionary.TryGetValue(hash, out var cached) && cached is not null) return cached; var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash); if (cacheEntity == null || string.IsNullOrEmpty(cacheEntity.ResolvedFilepath) || !File.Exists(cacheEntity.ResolvedFilepath)) return null; using var fs = File.Open(cacheEntity.ResolvedFilepath, FileMode.Open, FileAccess.Read, FileShare.Read); using var reader = new BinaryReader(fs); // PAP header (mostly from vfxeditor) try { _ = reader.ReadInt32(); // ignore _ = reader.ReadInt32(); // ignore var numAnimations = reader.ReadInt16(); // num animations var modelId = reader.ReadInt16(); // modelid if (numAnimations < 0 || numAnimations > 1000) { _logger.LogWarning("PAP file {hash} has invalid animation count {count}, skipping", hash, numAnimations); return null; } var type = reader.ReadByte(); // type if (type != 0) return null; // not human _ = reader.ReadByte(); // variant _ = reader.ReadInt32(); // ignore var havokPosition = reader.ReadInt32(); var footerPosition = reader.ReadInt32(); if (havokPosition <= 0 || footerPosition <= havokPosition || footerPosition > fs.Length || havokPosition >= fs.Length) { _logger.LogWarning("PAP file {hash} has invalid offsets (havok={havok}, footer={footer}, length={length})", hash, havokPosition, footerPosition, fs.Length); return null; } var havokDataSizeLong = (long)footerPosition - havokPosition; if (havokDataSizeLong <= 8 || havokDataSizeLong > int.MaxValue) { _logger.LogWarning("PAP file {hash} has invalid Havok data size {size}", hash, havokDataSizeLong); return null; } var havokDataSize = (int)havokDataSizeLong; reader.BaseStream.Position = havokPosition; var havokData = new byte[havokDataSize]; var bytesRead = reader.Read(havokData, 0, havokDataSize); if (bytesRead != havokDataSize) { _logger.LogWarning("PAP file {hash}: Expected to read {expected} bytes but got {actual}", hash, havokDataSize, bytesRead); return null; } if (havokData.Length < 8) return null; var tempSets = new Dictionary>(StringComparer.OrdinalIgnoreCase); var tempFileName = $"lightless_pap_{Guid.NewGuid():N}_{hash.Substring(0, Math.Min(8, hash.Length))}.hkx"; var tempHavokDataPath = Path.Combine(Path.GetTempPath(), tempFileName); IntPtr tempHavokDataPathAnsi = IntPtr.Zero; try { var tempDir = Path.GetDirectoryName(tempHavokDataPath); if (!Directory.Exists(tempDir)) { _logger.LogWarning("Temp directory {dir} doesn't exist", tempDir); return null; } File.WriteAllBytes(tempHavokDataPath, havokData); if (!File.Exists(tempHavokDataPath)) { _logger.LogWarning("Temporary havok file was not created at {path}", tempHavokDataPath); return null; } var writtenFileInfo = new FileInfo(tempHavokDataPath); if (writtenFileInfo.Length != havokData.Length) { _logger.LogWarning("Written temp file size mismatch: expected {expected}, got {actual}", havokData.Length, writtenFileInfo.Length); File.Delete(tempHavokDataPath); return null; } tempHavokDataPathAnsi = Marshal.StringToHGlobalAnsi(tempHavokDataPath); var loadoptions = stackalloc hkSerializeUtil.LoadOptions[1]; loadoptions->TypeInfoRegistry = hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry(); loadoptions->ClassNameRegistry = hkBuiltinTypeRegistry.Instance()->GetClassNameRegistry(); loadoptions->Flags = new hkFlags { Storage = (int)hkSerializeUtil.LoadOptionBits.Default }; hkResource* resource = null; try { resource = hkSerializeUtil.LoadFromFile((byte*)tempHavokDataPathAnsi, null, loadoptions); } catch (SEHException ex) { _logger.LogError(ex, "SEH exception loading Havok file from {path} (hash={hash}). Native error code: 0x{code:X}", tempHavokDataPath, hash, ex.ErrorCode); return null; } if (resource == null) { _logger.LogDebug("Havok resource was null after loading from {path} (hash={hash})", tempHavokDataPath, hash); return null; } if ((nint)resource == nint.Zero || !IsValidPointer((IntPtr)resource)) { _logger.LogDebug("Havok resource pointer is invalid (hash={hash})", hash); return null; } var rootLevelName = @"hkRootLevelContainer"u8; fixed (byte* n1 = rootLevelName) { var container = (hkRootLevelContainer*)resource->GetContentsPointer(n1, hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry()); if (container == null) { _logger.LogDebug("hkRootLevelContainer is null (hash={hash})", hash); return null; } if ((nint)container == nint.Zero || !IsValidPointer((IntPtr)container)) { _logger.LogDebug("hkRootLevelContainer pointer is invalid (hash={hash})", hash); return null; } var animationName = @"hkaAnimationContainer"u8; fixed (byte* n2 = animationName) { var animContainer = (hkaAnimationContainer*)container->findObjectByName(n2, null); if (animContainer == null) { _logger.LogDebug("hkaAnimationContainer is null (hash={hash})", hash); return null; } if ((nint)animContainer == nint.Zero || !IsValidPointer((IntPtr)animContainer)) { _logger.LogDebug("hkaAnimationContainer pointer is invalid (hash={hash})", hash); return null; } if (animContainer->Bindings.Length < 0 || animContainer->Bindings.Length > 10000) { _logger.LogDebug("Invalid bindings count {count} (hash={hash})", animContainer->Bindings.Length, hash); return null; } for (int i = 0; i < animContainer->Bindings.Length; i++) { var binding = animContainer->Bindings[i].ptr; if (binding == null) continue; if ((nint)binding == nint.Zero || !IsValidPointer((IntPtr)binding)) { _logger.LogDebug("Skipping invalid binding at index {index} (hash={hash})", i, hash); continue; } var rawSkel = binding->OriginalSkeletonName.String; var skeletonKey = CanonicalizeSkeletonKey(rawSkel); if (string.IsNullOrEmpty(skeletonKey)) continue; var boneTransform = binding->TransformTrackToBoneIndices; if (boneTransform.Length <= 0 || boneTransform.Length > 10000) { _logger.LogDebug("Invalid bone transform length {length} for skeleton {skel} (hash={hash})", boneTransform.Length, skeletonKey, hash); continue; } if (!tempSets.TryGetValue(skeletonKey, out var set)) { set = []; tempSets[skeletonKey] = set; } for (int boneIdx = 0; boneIdx < boneTransform.Length; boneIdx++) { var v = boneTransform[boneIdx]; if (v < 0 || v > ushort.MaxValue) continue; set.Add((ushort)v); } } } } } catch (SEHException ex) { _logger.LogError(ex, "SEH exception processing PAP file {hash} from {path}. Error code: 0x{code:X}", hash, tempHavokDataPath, ex.ErrorCode); return null; } catch (Exception ex) { _logger.LogError(ex, "Managed exception loading havok file {hash} from {path}", hash, tempHavokDataPath); return null; } finally { if (tempHavokDataPathAnsi != IntPtr.Zero) Marshal.FreeHGlobal(tempHavokDataPathAnsi); int retryCount = 3; while (retryCount > 0 && File.Exists(tempHavokDataPath)) { try { File.Delete(tempHavokDataPath); break; } catch (IOException ex) { retryCount--; if (retryCount == 0) { _logger.LogDebug(ex, "Failed to delete temporary havok file after retries: {path}", tempHavokDataPath); } else { Thread.Sleep(50); } } catch (Exception ex) { _logger.LogDebug(ex, "Unexpected error deleting temporary havok file: {path}", tempHavokDataPath); break; } } } if (tempSets.Count == 0) { _logger.LogDebug("No bone sets found in PAP file (hash={hash})", hash); return null; } var output = new Dictionary>(tempSets.Count, StringComparer.OrdinalIgnoreCase); foreach (var (key, set) in tempSets) { if (set.Count == 0) continue; var list = set.ToList(); list.Sort(); output[key] = list; } if (output.Count == 0) return null; _configService.Current.BonesDictionary[hash] = output; if (persistToConfig) _configService.Save(); return output; } catch (Exception ex) { _logger.LogError(ex, "Outer exception reading PAP file (hash={hash})", hash); return null; } } private static bool IsValidPointer(IntPtr ptr) { if (ptr == IntPtr.Zero) return false; try { _ = Marshal.ReadByte(ptr); return true; } catch { return false; } } public static string CanonicalizeSkeletonKey(string? raw) { if (string.IsNullOrWhiteSpace(raw)) return string.Empty; var s = raw.Replace('\\', '/').Trim(); var underscore = s.LastIndexOf('_'); if (underscore > 0 && underscore + 1 < s.Length && char.IsDigit(s[underscore + 1])) s = s[..underscore]; if (s.StartsWith("skeleton", StringComparison.OrdinalIgnoreCase)) return "skeleton"; var m = _bucketPathRegex.Match(s); if (m.Success) return m.Groups["bucket"].Value.ToLowerInvariant(); m = _bucketSklRegex.Match(s); if (m.Success) return m.Groups["bucket"].Value.ToLowerInvariant(); m = _bucketLooseRegex.Match(s); if (m.Success) return m.Groups["bucket"].Value.ToLowerInvariant(); return string.Empty; } public static bool ContainsIndexCompat( HashSet available, ushort idx, bool papLikelyOneBased, bool allowOneBasedShift, bool allowNeighborTolerance) { Span candidates = stackalloc ushort[2]; int count = 0; candidates[count++] = idx; if (allowOneBasedShift && papLikelyOneBased && idx > 0) candidates[count++] = (ushort)(idx - 1); for (int i = 0; i < count; i++) { var c = candidates[i]; if (available.Contains(c)) return true; if (allowNeighborTolerance) { if (c > 0 && available.Contains((ushort)(c - 1))) return true; if (c < ushort.MaxValue && available.Contains((ushort)(c + 1))) return true; } } return false; } public static bool IsPapCompatible( IReadOnlyDictionary> localBoneSets, IReadOnlyDictionary> papBoneIndices, AnimationValidationMode mode, bool allowOneBasedShift, bool allowNeighborTolerance, out string reason) { reason = string.Empty; if (mode == AnimationValidationMode.Unsafe) return true; var papBuckets = papBoneIndices.Keys .Select(CanonicalizeSkeletonKey) .Where(k => !string.IsNullOrEmpty(k)) .Distinct(StringComparer.OrdinalIgnoreCase) .ToList(); if (papBuckets.Count == 0) { reason = "No skeleton bucket bindings found in the PAP"; return false; } if (mode == AnimationValidationMode.Safe) { if (papBuckets.Any(b => localBoneSets.ContainsKey(b))) return true; reason = $"No matching skeleton bucket between PAP [{string.Join(", ", papBuckets)}] and local [{string.Join(", ", localBoneSets.Keys.Order())}]."; return false; } foreach (var bucket in papBuckets) { if (!localBoneSets.TryGetValue(bucket, out var available)) { reason = $"Missing skeleton bucket '{bucket}' on local actor."; return false; } var indices = papBoneIndices .Where(kvp => string.Equals(CanonicalizeSkeletonKey(kvp.Key), bucket, StringComparison.OrdinalIgnoreCase)) .SelectMany(kvp => kvp.Value ?? Enumerable.Empty()) .Distinct() .ToList(); if (indices.Count == 0) continue; bool has0 = false, has1 = false; ushort min = ushort.MaxValue; foreach (var v in indices) { if (v == 0) has0 = true; if (v == 1) has1 = true; if (v < min) min = v; } bool papLikelyOneBased = allowOneBasedShift && (min == 1) && has1 && !has0; foreach (var idx in indices) { if (!ContainsIndexCompat(available, idx, papLikelyOneBased, allowOneBasedShift, allowNeighborTolerance)) { reason = $"No compatible local skeleton for PAP '{bucket}': missing bone index {idx}."; return false; } } } return true; } public void DumpLocalSkeletonIndices(GameObjectHandler handler, string? filter = null) { var skels = GetSkeletonBoneIndices(handler); if (skels == null) { _logger.LogTrace("DumpLocalSkeletonIndices: local skeleton indices are null or not found"); return; } var keys = skels.Keys .Order(StringComparer.OrdinalIgnoreCase) .ToArray(); _logger.LogTrace("Local skeleton indices found ({count}): {keys}", keys.Length, string.Join(", ", keys)); if (!string.IsNullOrWhiteSpace(filter)) { var hits = keys.Where(k => k.Equals(filter, StringComparison.OrdinalIgnoreCase) || k.StartsWith(filter + "_", StringComparison.OrdinalIgnoreCase) || filter.StartsWith(k + "_", StringComparison.OrdinalIgnoreCase) || k.Contains(filter, StringComparison.OrdinalIgnoreCase)) .ToArray(); _logger.LogTrace("Matches found for '{filter}': {hits}", filter, hits.Length == 0 ? "" : string.Join(", ", hits)); } } public async Task GetTrianglesByHash(string hash) { if (_configService.Current.TriangleDictionary.TryGetValue(hash, out var cachedTris) && cachedTris > 0) return cachedTris; if (_failedCalculatedTris.Contains(hash, StringComparer.Ordinal)) return 0; var path = _fileCacheManager.GetFileCacheByHash(hash); if (path == null || !path.ResolvedFilepath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase)) return 0; return CalculateTrianglesFromPath(hash, path.ResolvedFilepath, _configService.Current.TriangleDictionary, _failedCalculatedTris); } public async Task GetEffectiveTrianglesByHash(string hash, string filePath) { if (_configService.Current.EffectiveTriangleDictionary.TryGetValue(hash, out var cachedTris) && cachedTris > 0) return cachedTris; if (_failedCalculatedEffectiveTris.Contains(hash, StringComparer.Ordinal)) return 0; if (string.IsNullOrEmpty(filePath) || !filePath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase) || !File.Exists(filePath)) { return 0; } return CalculateTrianglesFromPath(hash, filePath, _configService.Current.EffectiveTriangleDictionary, _failedCalculatedEffectiveTris); } private long CalculateTrianglesFromPath( string hash, string filePath, ConcurrentDictionary cache, List failedList) { try { _logger.LogDebug("Detected Model File {path}, calculating Tris", filePath); var file = new MdlFile(filePath); if (file.LodCount <= 0) { failedList.Add(hash); cache[hash] = 0; _configService.Save(); return 0; } long tris = 0; foreach (var lod in file.Lods) { try { var meshIdx = lod.MeshIndex; var meshCnt = lod.MeshCount; tris = file.Meshes.Skip(meshIdx).Take(meshCnt).Sum(p => p.IndexCount) / 3; } catch (Exception ex) { _logger.LogDebug(ex, "Could not load lod mesh {mesh} from path {path}", lod.MeshIndex, filePath); continue; } if (tris > 0) { _logger.LogDebug("TriAnalysis: {filePath} => {tris} triangles", filePath, tris); cache[hash] = tris; _configService.Save(); break; } } return tris; } catch (Exception e) { failedList.Add(hash); cache[hash] = 0; _configService.Save(); _logger.LogWarning(e, "Could not parse file {file}", filePath); return 0; } } // Regexes for canonicalizing skeleton keys private static readonly Regex _bucketPathRegex = BucketRegex(); private static readonly Regex _bucketSklRegex = SklRegex(); private static readonly Regex _bucketLooseRegex = LooseBucketRegex(); [GeneratedRegex(@"(?i)(?:^|/)(?c\d{4})(?:/|$)", RegexOptions.Compiled, "en-NL")] private static partial Regex BucketRegex(); [GeneratedRegex(@"(?i)\bskl_(?c\d{4})[a-z]\d{4}\b", RegexOptions.Compiled, "en-NL")] private static partial Regex SklRegex(); [GeneratedRegex(@"(?i)(?c\d{4})(?!\d)", RegexOptions.Compiled, "en-NL")] private static partial Regex LooseBucketRegex(); }