Merged Cake and Abel branched into 2.0.3 (#131)

Co-authored-by: azyges <aaaaaa@aaa.aaa>
Co-authored-by: cake <admin@cakeandbanana.nl>
Co-authored-by: defnotken <itsdefnotken@gmail.com>
Reviewed-on: #131

LightlessSync/Services/XivDataAnalyzer.cs

@@ -6,18 +6,22 @@ using FFXIVClientStructs.Havok.Common.Serialize.Util;
 using LightlessSync.FileCache;
 using LightlessSync.Interop.GameModel;
 using LightlessSync.LightlessConfiguration;
+using LightlessSync.PlayerData.Factories;
 using LightlessSync.PlayerData.Handlers;
 using Microsoft.Extensions.Logging;
+using System.Collections.Concurrent;
 using System.Runtime.InteropServices;
+using System.Text.RegularExpressions;
 
 namespace LightlessSync.Services;
 
-public sealed class XivDataAnalyzer
+public sealed partial class XivDataAnalyzer
 {
     private readonly ILogger<XivDataAnalyzer> _logger;
     private readonly FileCacheManager _fileCacheManager;
     private readonly XivDataStorageService _configService;
     private readonly List<string> _failedCalculatedTris = [];
+    private readonly List<string> _failedCalculatedEffectiveTris = [];
 
     public XivDataAnalyzer(ILogger<XivDataAnalyzer> logger, FileCacheManager fileCacheManager,
         XivDataStorageService configService)
@@ -29,127 +33,441 @@ public sealed class XivDataAnalyzer
 
     public unsafe Dictionary<string, List<ushort>>? GetSkeletonBoneIndices(GameObjectHandler handler)
     {
-        if (handler.Address == nint.Zero) return null;
-        var chara = (CharacterBase*)(((Character*)handler.Address)->GameObject.DrawObject);
-        if (chara->GetModelType() != CharacterBase.ModelType.Human) return null;
-        var resHandles = chara->Skeleton->SkeletonResourceHandles;
-        Dictionary<string, List<ushort>> outputIndices = [];
+        if (handler is null || handler.Address == nint.Zero)
+            return null;
+
+        Dictionary<string, HashSet<ushort>> sets = new(StringComparer.OrdinalIgnoreCase);
+
         try
         {
-            for (int i = 0; i < chara->Skeleton->PartialSkeletonCount; i++)
+            var drawObject = ((Character*)handler.Address)->GameObject.DrawObject;
+            if (drawObject == null)
+                return null;
+
+            var chara = (CharacterBase*)drawObject;
+            if (chara->GetModelType() != CharacterBase.ModelType.Human)
+                return null;
+
+            var skeleton = chara->Skeleton;
+            if (skeleton == null)
+                return null;
+
+            var resHandles = skeleton->SkeletonResourceHandles;
+            var partialCount = skeleton->PartialSkeletonCount;
+            if (partialCount <= 0)
+                return null;
+
+            for (int i = 0; i < partialCount; i++)
             {
                 var handle = *(resHandles + i);
-                _logger.LogTrace("Iterating over SkeletonResourceHandle #{i}:{x}", i, ((nint)handle).ToString("X"));
-                if ((nint)handle == nint.Zero) continue;
-                var curBones = handle->BoneCount;
-                // this is unrealistic, the filename shouldn't ever be that long
-                if (handle->FileName.Length > 1024) continue;
-                var skeletonName = handle->FileName.ToString();
-                if (string.IsNullOrEmpty(skeletonName)) continue;
-                outputIndices[skeletonName] = [];
-                for (ushort boneIdx = 0; boneIdx < curBones; boneIdx++)
+                if ((nint)handle == nint.Zero)
+                    continue;
+
+                if (handle->FileName.Length > 1024)
+                    continue;
+
+                var rawName = handle->FileName.ToString();
+                if (string.IsNullOrWhiteSpace(rawName))
+                    continue;
+
+                var skeletonKey = CanonicalizeSkeletonKey(rawName);
+                if (string.IsNullOrEmpty(skeletonKey))
+                    continue;
+
+                var boneCount = handle->BoneCount;
+                if (boneCount == 0)
+                    continue;
+
+                var havokSkel = handle->HavokSkeleton;
+                if ((nint)havokSkel == nint.Zero)
+                    continue;
+
+                if (!sets.TryGetValue(skeletonKey, out var set))
                 {
-                    var boneName = handle->HavokSkeleton->Bones[boneIdx].Name.String;
-                    if (boneName == null) continue;
-                    outputIndices[skeletonName].Add((ushort)(boneIdx + 1));
+                    set = [];
+                    sets[skeletonKey] = set;
                 }
+
+                uint maxExclusive = boneCount;
+                uint ushortExclusive = (uint)ushort.MaxValue + 1u;
+                if (maxExclusive > ushortExclusive)
+                    maxExclusive = ushortExclusive;
+
+                for (uint boneIdx = 0; boneIdx < maxExclusive; boneIdx++)
+                {
+                    var name = havokSkel->Bones[boneIdx].Name.String;
+                    if (name == null)
+                        continue;
+
+                    set.Add((ushort)boneIdx);
+                }
+
+                _logger.LogTrace("Local skeleton raw file='{raw}', key='{key}', boneCount={count}",
+                    rawName, skeletonKey, boneCount);
             }
         }
         catch (Exception ex)
         {
             _logger.LogWarning(ex, "Could not process skeleton data");
+            return null;
         }
 
-        return (outputIndices.Count != 0 && outputIndices.Values.All(u => u.Count > 0)) ? outputIndices : null;
+        if (sets.Count == 0)
+            return null;
+
+        var output = new Dictionary<string, List<ushort>>(sets.Count, StringComparer.OrdinalIgnoreCase);
+        foreach (var (key, set) in sets)
+        {
+            if (set.Count == 0)
+                continue;
+
+            var list = set.ToList();
+            list.Sort();
+            output[key] = list;
+        }
+
+        return (output.Count != 0 && output.Values.All(v => v.Count > 0)) ? output : null;
     }
 
-    public unsafe Dictionary<string, List<ushort>>? GetBoneIndicesFromPap(string hash)
+    public unsafe Dictionary<string, List<ushort>>? GetBoneIndicesFromPap(string hash, bool persistToConfig = true)
     {
-        if (_configService.Current.BonesDictionary.TryGetValue(hash, out var bones)) return bones;
+        if (string.IsNullOrWhiteSpace(hash))
+            return null;
+
+        if (_configService.Current.BonesDictionary.TryGetValue(hash, out var cached) && cached is not null)
+            return cached;
 
         var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash);
-        if (cacheEntity == null) return null;
+        if (cacheEntity == null || string.IsNullOrEmpty(cacheEntity.ResolvedFilepath) || !File.Exists(cacheEntity.ResolvedFilepath))
+            return null;
 
-        using BinaryReader reader = new(File.Open(cacheEntity.ResolvedFilepath, FileMode.Open, FileAccess.Read, FileShare.Read));
+        using var fs = File.Open(cacheEntity.ResolvedFilepath, FileMode.Open, FileAccess.Read, FileShare.Read);
+        using var reader = new BinaryReader(fs);
 
-        // most of this shit is from vfxeditor, surely nothing will change in the pap format :copium:
-        reader.ReadInt32(); // ignore
-        reader.ReadInt32(); // ignore
-        reader.ReadInt16(); // read 2 (num animations)
-        reader.ReadInt16(); // read 2 (modelid)
-        var type = reader.ReadByte();// read 1 (type)
-        if (type != 0) return null; // it's not human, just ignore it, whatever
+        // PAP header (mostly from vfxeditor)
+        _ = reader.ReadInt32(); // ignore
+        _ = reader.ReadInt32(); // ignore
+        _ = reader.ReadInt16(); // num animations
+        _ = reader.ReadInt16(); // modelid
+
+        var type = reader.ReadByte(); // type
+        if (type != 0)
+            return null; // not human
+
+        _ = reader.ReadByte();  // variant
+        _ = reader.ReadInt32(); // ignore
 
-        reader.ReadByte(); // read 1 (variant)
-        reader.ReadInt32(); // ignore
         var havokPosition = reader.ReadInt32();
         var footerPosition = reader.ReadInt32();
-        var havokDataSize = footerPosition - havokPosition;
+
+        // sanity checks
+        if (havokPosition <= 0 || footerPosition <= havokPosition || footerPosition > fs.Length)
+            return null;
+
+        var havokDataSizeLong = (long)footerPosition - havokPosition;
+        if (havokDataSizeLong <= 8 || havokDataSizeLong > int.MaxValue)
+            return null;
+
+        var havokDataSize = (int)havokDataSizeLong;
+
         reader.BaseStream.Position = havokPosition;
         var havokData = reader.ReadBytes(havokDataSize);
-        if (havokData.Length <= 8) return null; // no havok data
+        if (havokData.Length <= 8)
+            return null;
 
-        var output = new Dictionary<string, List<ushort>>(StringComparer.OrdinalIgnoreCase);
-        var tempHavokDataPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()) + ".hkx";
-        var tempHavokDataPathAnsi = Marshal.StringToHGlobalAnsi(tempHavokDataPath);
+        var tempSets = new Dictionary<string, HashSet<ushort>>(StringComparer.OrdinalIgnoreCase);
+
+        var tempHavokDataPath = Path.Combine(Path.GetTempPath(), $"lightless_{Guid.NewGuid():N}.hkx");
+        IntPtr tempHavokDataPathAnsi = IntPtr.Zero;
 
         try
         {
             File.WriteAllBytes(tempHavokDataPath, havokData);
 
+            if (!File.Exists(tempHavokDataPath))
+            {
+                _logger.LogTrace("Temporary havok file did not exist when attempting to load: {path}", tempHavokDataPath);
+                return null;
+            }
+
+            tempHavokDataPathAnsi = Marshal.StringToHGlobalAnsi(tempHavokDataPath);
+
             var loadoptions = stackalloc hkSerializeUtil.LoadOptions[1];
             loadoptions->TypeInfoRegistry = hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry();
             loadoptions->ClassNameRegistry = hkBuiltinTypeRegistry.Instance()->GetClassNameRegistry();
             loadoptions->Flags = new hkFlags<hkSerializeUtil.LoadOptionBits, int>
             {
-                Storage = (int)(hkSerializeUtil.LoadOptionBits.Default)
+                Storage = (int)hkSerializeUtil.LoadOptionBits.Default
             };
 
             var resource = hkSerializeUtil.LoadFromFile((byte*)tempHavokDataPathAnsi, null, loadoptions);
             if (resource == null)
             {
-                throw new InvalidOperationException("Resource was null after loading");
+                _logger.LogWarning("Havok resource was null after loading from {path}", tempHavokDataPath);
+                return null;
             }
 
             var rootLevelName = @"hkRootLevelContainer"u8;
             fixed (byte* n1 = rootLevelName)
             {
                 var container = (hkRootLevelContainer*)resource->GetContentsPointer(n1, hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry());
+                if (container == null)
+                    return null;
+
                 var animationName = @"hkaAnimationContainer"u8;
                 fixed (byte* n2 = animationName)
                 {
                     var animContainer = (hkaAnimationContainer*)container->findObjectByName(n2, null);
+                    if (animContainer == null)
+                        return null;
+
                     for (int i = 0; i < animContainer->Bindings.Length; i++)
                     {
                         var binding = animContainer->Bindings[i].ptr;
+                        if (binding == null)
+                            continue;
+
+                        var rawSkel = binding->OriginalSkeletonName.String;
+                        var skeletonKey = CanonicalizeSkeletonKey(rawSkel);
+                        if (string.IsNullOrEmpty(skeletonKey))
+                            continue;
+
                         var boneTransform = binding->TransformTrackToBoneIndices;
-                        string name = binding->OriginalSkeletonName.String! + "_" + i;
-                        output[name] = [];
+                        if (boneTransform.Length <= 0)
+                            continue;
+
+                        if (!tempSets.TryGetValue(skeletonKey, out var set))
+                        {
+                            set = [];
+                            tempSets[skeletonKey] = set;
+                        }
+
                         for (int boneIdx = 0; boneIdx < boneTransform.Length; boneIdx++)
                         {
-                            output[name].Add((ushort)boneTransform[boneIdx]);
+                            var v = boneTransform[boneIdx];
+                            if (v < 0) continue;
+                            set.Add((ushort)v);
                         }
-                        output[name].Sort();
                     }
-
                 }
             }
         }
         catch (Exception ex)
         {
             _logger.LogWarning(ex, "Could not load havok file in {path}", tempHavokDataPath);
+            return null;
         }
         finally
         {
-            Marshal.FreeHGlobal(tempHavokDataPathAnsi);
-            File.Delete(tempHavokDataPath);
+            if (tempHavokDataPathAnsi != IntPtr.Zero)
+                Marshal.FreeHGlobal(tempHavokDataPathAnsi);
+
+            try
+            {
+                if (File.Exists(tempHavokDataPath))
+                    File.Delete(tempHavokDataPath);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogTrace(ex, "Could not delete temporary havok file: {path}", tempHavokDataPath);
+            }
         }
 
+        if (tempSets.Count == 0)
+            return null;
+
+        var output = new Dictionary<string, List<ushort>>(tempSets.Count, StringComparer.OrdinalIgnoreCase);
+        foreach (var (key, set) in tempSets)
+        {
+            if (set.Count == 0) continue;
+
+            var list = set.ToList();
+            list.Sort();
+            output[key] = list;
+        }
+
+        if (output.Count == 0)
+            return null;
+
         _configService.Current.BonesDictionary[hash] = output;
-        _configService.Save();
+
+        if (persistToConfig)
+            _configService.Save();
+
         return output;
     }
 
+
+    public static string CanonicalizeSkeletonKey(string? raw)
+    {
+        if (string.IsNullOrWhiteSpace(raw))
+            return string.Empty;
+
+        var s = raw.Replace('\\', '/').Trim();
+
+        var underscore = s.LastIndexOf('_');
+        if (underscore > 0 && underscore + 1 < s.Length && char.IsDigit(s[underscore + 1]))
+            s = s[..underscore];
+
+        if (s.StartsWith("skeleton", StringComparison.OrdinalIgnoreCase))
+            return "skeleton";
+
+        var m = _bucketPathRegex.Match(s);
+        if (m.Success)
+            return m.Groups["bucket"].Value.ToLowerInvariant();
+
+        m = _bucketSklRegex.Match(s);
+        if (m.Success)
+            return m.Groups["bucket"].Value.ToLowerInvariant();
+
+        m = _bucketLooseRegex.Match(s);
+        if (m.Success)
+            return m.Groups["bucket"].Value.ToLowerInvariant();
+
+        return string.Empty;
+    }
+
+    public static bool ContainsIndexCompat(
+        HashSet<ushort> available,
+        ushort idx,
+        bool papLikelyOneBased,
+        bool allowOneBasedShift,
+        bool allowNeighborTolerance)
+    {
+        Span<ushort> candidates = stackalloc ushort[2];
+        int count = 0;
+
+        candidates[count++] = idx;
+
+        if (allowOneBasedShift && papLikelyOneBased && idx > 0)
+            candidates[count++] = (ushort)(idx - 1);
+
+        for (int i = 0; i < count; i++)
+        {
+            var c = candidates[i];
+
+            if (available.Contains(c))
+                return true;
+
+            if (allowNeighborTolerance)
+            {
+                if (c > 0 && available.Contains((ushort)(c - 1)))
+                    return true;
+
+                if (c < ushort.MaxValue && available.Contains((ushort)(c + 1)))
+                    return true;
+            }
+        }
+
+        return false;
+    }
+
+    public static bool IsPapCompatible(
+        IReadOnlyDictionary<string, HashSet<ushort>> localBoneSets,
+        IReadOnlyDictionary<string, List<ushort>> papBoneIndices,
+        AnimationValidationMode mode,
+        bool allowOneBasedShift,
+        bool allowNeighborTolerance,
+        out string reason)
+    {
+        reason = string.Empty;
+
+        if (mode == AnimationValidationMode.Unsafe)
+            return true;
+
+        var papBuckets = papBoneIndices.Keys
+            .Select(CanonicalizeSkeletonKey)
+            .Where(k => !string.IsNullOrEmpty(k))
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToList();
+
+        if (papBuckets.Count == 0)
+        {
+            reason = "No skeleton bucket bindings found in the PAP";
+            return false;
+        }
+
+        if (mode == AnimationValidationMode.Safe)
+        {
+            if (papBuckets.Any(b => localBoneSets.ContainsKey(b)))
+                return true;
+
+            reason = $"No matching skeleton bucket between PAP [{string.Join(", ", papBuckets)}] and local [{string.Join(", ", localBoneSets.Keys.Order())}].";
+            return false;
+        }
+
+        foreach (var bucket in papBuckets)
+        {
+            if (!localBoneSets.TryGetValue(bucket, out var available))
+            {
+                reason = $"Missing skeleton bucket '{bucket}' on local actor.";
+                return false;
+            }
+
+            var indices = papBoneIndices
+                .Where(kvp => string.Equals(CanonicalizeSkeletonKey(kvp.Key), bucket, StringComparison.OrdinalIgnoreCase))
+                .SelectMany(kvp => kvp.Value ?? Enumerable.Empty<ushort>())
+                .Distinct()
+                .ToList();
+
+            if (indices.Count == 0)
+                continue;
+
+            bool has0 = false, has1 = false;
+            ushort min = ushort.MaxValue;
+            foreach (var v in indices)
+            {
+                if (v == 0) has0 = true;
+                if (v == 1) has1 = true;
+                if (v < min) min = v;
+            }
+            bool papLikelyOneBased = allowOneBasedShift && (min == 1) && has1 && !has0;
+
+            foreach (var idx in indices)
+            {
+                if (!ContainsIndexCompat(available, idx, papLikelyOneBased, allowOneBasedShift, allowNeighborTolerance))
+                {
+                    reason = $"No compatible local skeleton for PAP '{bucket}': missing bone index {idx}.";
+                    return false;
+                }
+            }
+        }
+
+        return true;
+    }
+
+    public void DumpLocalSkeletonIndices(GameObjectHandler handler, string? filter = null)
+    {
+        var skels = GetSkeletonBoneIndices(handler);
+        if (skels == null)
+        {
+            _logger.LogTrace("DumpLocalSkeletonIndices: local skeleton indices are null or not found");
+            return;
+        }
+        
+        var keys = skels.Keys
+            .Order(StringComparer.OrdinalIgnoreCase)
+            .ToArray();
+
+        _logger.LogTrace("Local skeleton indices found ({count}): {keys}",
+            keys.Length,
+            string.Join(", ", keys));
+
+        if (!string.IsNullOrWhiteSpace(filter))
+        {
+            var hits = keys.Where(k =>
+                k.Equals(filter, StringComparison.OrdinalIgnoreCase) ||
+                k.StartsWith(filter + "_", StringComparison.OrdinalIgnoreCase) ||
+                 filter.StartsWith(k + "_", StringComparison.OrdinalIgnoreCase) ||
+                k.Contains(filter, StringComparison.OrdinalIgnoreCase))
+                .ToArray();
+
+            _logger.LogTrace("Matches found for '{filter}': {hits}",
+                filter,
+                hits.Length == 0 ? "<none>" : string.Join(", ", hits));
+        }
+    }
+
     public async Task<long> GetTrianglesByHash(string hash)
     {
         if (_configService.Current.TriangleDictionary.TryGetValue(hash, out var cachedTris) && cachedTris > 0)
@@ -162,16 +480,41 @@ public sealed class XivDataAnalyzer
         if (path == null || !path.ResolvedFilepath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase))
             return 0;
 
-        var filePath = path.ResolvedFilepath;
+        return CalculateTrianglesFromPath(hash, path.ResolvedFilepath, _configService.Current.TriangleDictionary, _failedCalculatedTris);
+    }
 
+    public async Task<long> GetEffectiveTrianglesByHash(string hash, string filePath)
+    {
+        if (_configService.Current.EffectiveTriangleDictionary.TryGetValue(hash, out var cachedTris) && cachedTris > 0)
+            return cachedTris;
+
+        if (_failedCalculatedEffectiveTris.Contains(hash, StringComparer.Ordinal))
+            return 0;
+
+        if (string.IsNullOrEmpty(filePath)
+            || !filePath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase)
+            || !File.Exists(filePath))
+        {
+            return 0;
+        }
+
+        return CalculateTrianglesFromPath(hash, filePath, _configService.Current.EffectiveTriangleDictionary, _failedCalculatedEffectiveTris);
+    }
+
+    private long CalculateTrianglesFromPath(
+        string hash,
+        string filePath,
+        ConcurrentDictionary<string, long> cache,
+        List<string> failedList)
+    {
         try
         {
             _logger.LogDebug("Detected Model File {path}, calculating Tris", filePath);
             var file = new MdlFile(filePath);
             if (file.LodCount <= 0)
             {
-                _failedCalculatedTris.Add(hash);
-                _configService.Current.TriangleDictionary[hash] = 0;
+                failedList.Add(hash);
+                cache[hash] = 0;
                 _configService.Save();
                 return 0;
             }
@@ -195,7 +538,7 @@ public sealed class XivDataAnalyzer
                 if (tris > 0)
                 {
                     _logger.LogDebug("TriAnalysis: {filePath} => {tris} triangles", filePath, tris);
-                    _configService.Current.TriangleDictionary[hash] = tris;
+                    cache[hash] = tris;
                     _configService.Save();
                     break;
                 }
@@ -205,11 +548,30 @@ public sealed class XivDataAnalyzer
         }
         catch (Exception e)
         {
-            _failedCalculatedTris.Add(hash);
-            _configService.Current.TriangleDictionary[hash] = 0;
+            failedList.Add(hash);
+            cache[hash] = 0;
             _configService.Save();
             _logger.LogWarning(e, "Could not parse file {file}", filePath);
             return 0;
         }
     }
+
+    // Regexes for canonicalizing skeleton keys
+    private static readonly Regex _bucketPathRegex =
+    BucketRegex();
+
+    private static readonly Regex _bucketSklRegex =
+        SklRegex();
+
+    private static readonly Regex _bucketLooseRegex =
+        LooseBucketRegex();
+
+    [GeneratedRegex(@"(?i)(?:^|/)(?<bucket>c\d{4})(?:/|$)", RegexOptions.Compiled, "en-NL")]
+    private static partial Regex BucketRegex();
+
+    [GeneratedRegex(@"(?i)\bskl_(?<bucket>c\d{4})[a-z]\d{4}\b", RegexOptions.Compiled, "en-NL")]
+    private static partial Regex SklRegex();
+
+    [GeneratedRegex(@"(?i)(?<![a-z0-9])(?<bucket>c\d{4})(?!\d)", RegexOptions.Compiled, "en-NL")]
+    private static partial Regex LooseBucketRegex();
 }