LightlessClient/LightlessSync/Services/XivDataAnalyzer.cs

using FFXIVClientStructs.FFXIV.Client.Game.Character;
using FFXIVClientStructs.FFXIV.Client.Graphics.Scene;
using FFXIVClientStructs.Havok.Common.Serialize.Resource;
using FFXIVClientStructs.Havok.Animation;
using FFXIVClientStructs.Havok.Common.Base.Types;
using FFXIVClientStructs.Havok.Common.Serialize.Resource;
using FFXIVClientStructs.Havok.Common.Serialize.Util;
using LightlessSync.FileCache;
using LightlessSync.Interop.GameModel;
using LightlessSync.LightlessConfiguration;
using LightlessSync.PlayerData.Factories;
using LightlessSync.PlayerData.Handlers;
using Microsoft.Extensions.Logging;
using OtterGui.Text.EndObjects;
using System.Collections.Concurrent;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;

namespace LightlessSync.Services;

public sealed partial class XivDataAnalyzer
{
    private readonly ILogger<XivDataAnalyzer> _logger;
    private readonly FileCacheManager _fileCacheManager;
    private readonly XivDataStorageService _configService;
    private readonly List<string> _failedCalculatedTris = [];
    private readonly List<string> _failedCalculatedEffectiveTris = [];

    public XivDataAnalyzer(ILogger<XivDataAnalyzer> logger, FileCacheManager fileCacheManager,
        XivDataStorageService configService)
    {
        _logger = logger;
        _fileCacheManager = fileCacheManager;
        _configService = configService;
    }

    public unsafe Dictionary<string, List<ushort>>? GetSkeletonBoneIndices(GameObjectHandler handler)
    {
        if (handler is null || handler.Address == nint.Zero)
            return null;

        Dictionary<string, HashSet<ushort>> sets = new(StringComparer.OrdinalIgnoreCase);

        try
        {
            var drawObject = ((Character*)handler.Address)->GameObject.DrawObject;
            if (drawObject == null)
                return null;

            var chara = (CharacterBase*)drawObject;
            if (chara->GetModelType() != CharacterBase.ModelType.Human)
                return null;

            var skeleton = chara->Skeleton;
            if (skeleton == null)
                return null;

            var resHandles = skeleton->SkeletonResourceHandles;
            var partialCount = skeleton->PartialSkeletonCount;
            if (partialCount <= 0)
                return null;

            for (int i = 0; i < partialCount; i++)
            {
                var handle = *(resHandles + i);
                if ((nint)handle == nint.Zero)
                    continue;

                if (handle->FileName.Length > 1024)
                    continue;

                var rawName = handle->FileName.ToString();
                if (string.IsNullOrWhiteSpace(rawName))
                    continue;

                var skeletonKey = CanonicalizeSkeletonKey(rawName);
                if (string.IsNullOrEmpty(skeletonKey))
                    continue;

                var boneCount = handle->BoneCount;
                if (boneCount == 0)
                    continue;

                var havokSkel = handle->HavokSkeleton;
                if ((nint)havokSkel == nint.Zero)
                    continue;

                if (!sets.TryGetValue(skeletonKey, out var set))
                {
                    set = [];
                    sets[skeletonKey] = set;
                }

                uint maxExclusive = boneCount;
                uint ushortExclusive = (uint)ushort.MaxValue + 1u;
                if (maxExclusive > ushortExclusive)
                    maxExclusive = ushortExclusive;

                for (uint boneIdx = 0; boneIdx < maxExclusive; boneIdx++)
                {
                    var name = havokSkel->Bones[boneIdx].Name.String;
                    if (name == null)
                        continue;

                    set.Add((ushort)boneIdx);
                }

                _logger.LogTrace("Local skeleton raw file='{raw}', key='{key}', boneCount={count}",
                    rawName, skeletonKey, boneCount);
            }
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex, "Could not process skeleton data");
            return null;
        }

        if (sets.Count == 0)
            return null;

        var output = new Dictionary<string, List<ushort>>(sets.Count, StringComparer.OrdinalIgnoreCase);
        foreach (var (key, set) in sets)
        {
            if (set.Count == 0)
                continue;

            var list = set.ToList();
            list.Sort();
            output[key] = list;
        }

        return (output.Count != 0 && output.Values.All(v => v.Count > 0)) ? output : null;
    }

    public static byte[]? ReadHavokBytesFromPap(string papPath)
    {
        using var fs = File.Open(papPath, FileMode.Open, FileAccess.Read, FileShare.Read);
        using var reader = new BinaryReader(fs);

        _ = reader.ReadInt32();
        _ = reader.ReadInt32();
        _ = reader.ReadInt16();
        _ = reader.ReadInt16();

        var type = reader.ReadByte();
        if (type != 0) return null;

        _ = reader.ReadByte();
        _ = reader.ReadInt32();

        var havokPosition = reader.ReadInt32();
        var footerPosition = reader.ReadInt32();

        if (havokPosition <= 0 || footerPosition <= havokPosition || footerPosition > fs.Length)
            return null;

        var sizeLong = (long)footerPosition - havokPosition;
        if (sizeLong <= 8 || sizeLong > int.MaxValue)
            return null;

        var size = (int)sizeLong;

        fs.Position = havokPosition;
        var bytes = reader.ReadBytes(size);
        return bytes.Length > 8 ? bytes : null;
    }

    public unsafe Dictionary<string, List<ushort>>? ParseHavokBytesOnFrameworkThread(
    byte[] havokData,
    string hash,
    bool persistToConfig)
    {
        var tempSets = new Dictionary<string, HashSet<ushort>>(StringComparer.OrdinalIgnoreCase);

        var tempHkxPath = Path.Combine(Path.GetTempPath(), $"lightless_{Guid.NewGuid():N}.hkx");
        IntPtr pathAnsi = IntPtr.Zero;

        try
        {
            File.WriteAllBytes(tempHkxPath, havokData);

            pathAnsi = Marshal.StringToHGlobalAnsi(tempHkxPath);

            hkSerializeUtil.LoadOptions loadOptions = default;
            loadOptions.TypeInfoRegistry = hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry();
            loadOptions.ClassNameRegistry = hkBuiltinTypeRegistry.Instance()->GetClassNameRegistry();
            loadOptions.Flags = new hkFlags<hkSerializeUtil.LoadOptionBits, int>
            {
                Storage = (int)hkSerializeUtil.LoadOptionBits.Default
            };

            hkSerializeUtil.LoadOptions* pOpts = &loadOptions;

            var resource = hkSerializeUtil.LoadFromFile((byte*)pathAnsi, errorResult: null, pOpts);
            if (resource == null)
                return null;

            var rootLevelName = @"hkRootLevelContainer"u8;
            fixed (byte* n1 = rootLevelName)
            {
                var container = (hkRootLevelContainer*)resource->GetContentsPointer(
                    n1, hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry());

                if (container == null) return null;

                var animationName = @"hkaAnimationContainer"u8;
                fixed (byte* n2 = animationName)
                {
                    var animContainer = (hkaAnimationContainer*)container->findObjectByName(n2, null);
                    if (animContainer == null) return null;

                    for (int i = 0; i < animContainer->Bindings.Length; i++)
                    {
                        var binding = animContainer->Bindings[i].ptr;
                        if (binding == null) continue;

                        var rawSkel = binding->OriginalSkeletonName.String;
                        var skeletonKey = CanonicalizeSkeletonKey(rawSkel);
                        if (string.IsNullOrEmpty(skeletonKey)) continue;

                        var boneTransform = binding->TransformTrackToBoneIndices;
                        if (boneTransform.Length <= 0) continue;

                        if (!tempSets.TryGetValue(skeletonKey, out var set))
                            tempSets[skeletonKey] = set = [];

                        for (int boneIdx = 0; boneIdx < boneTransform.Length; boneIdx++)
                        {
                            var v = boneTransform[boneIdx];
                            if (v < 0) continue;
                            set.Add((ushort)v);
                        }
                    }
                }
            }
        }
        finally
        {
            if (pathAnsi != IntPtr.Zero)
                Marshal.FreeHGlobal(pathAnsi);

            try { if (File.Exists(tempHkxPath)) File.Delete(tempHkxPath); }
            catch { /* ignore */ }
        }

        if (tempSets.Count == 0) return null;

        var output = new Dictionary<string, List<ushort>>(tempSets.Count, StringComparer.OrdinalIgnoreCase);
        foreach (var (key, set) in tempSets)
        {
            if (set.Count == 0) continue;
            var list = set.ToList();
            list.Sort();
            output[key] = list;
        }

        if (output.Count == 0) return null;

        _configService.Current.BonesDictionary[hash] = output;
        if (persistToConfig) _configService.Save();

        return output;
    }

    public static string CanonicalizeSkeletonKey(string? raw)
    {
        if (string.IsNullOrWhiteSpace(raw))
            return string.Empty;

        var s = raw.Replace('\\', '/').Trim();

        var underscore = s.LastIndexOf('_');
        if (underscore > 0 && underscore + 1 < s.Length && char.IsDigit(s[underscore + 1]))
            s = s[..underscore];

        if (s.StartsWith("skeleton", StringComparison.OrdinalIgnoreCase))
            return "skeleton";

        var m = _bucketPathRegex.Match(s);
        if (m.Success)
            return m.Groups["bucket"].Value.ToLowerInvariant();

        m = _bucketSklRegex.Match(s);
        if (m.Success)
            return m.Groups["bucket"].Value.ToLowerInvariant();

        m = _bucketLooseRegex.Match(s);
        if (m.Success)
            return m.Groups["bucket"].Value.ToLowerInvariant();

        return string.Empty;
    }

    public static bool ContainsIndexCompat(
        HashSet<ushort> available,
        ushort idx,
        bool papLikelyOneBased,
        bool allowOneBasedShift,
        bool allowNeighborTolerance)
    {
        Span<ushort> candidates = stackalloc ushort[2];
        int count = 0;

        candidates[count++] = idx;

        if (allowOneBasedShift && papLikelyOneBased && idx > 0)
            candidates[count++] = (ushort)(idx - 1);

        for (int i = 0; i < count; i++)
        {
            var c = candidates[i];

            if (available.Contains(c))
                return true;

            if (allowNeighborTolerance)
            {
                if (c > 0 && available.Contains((ushort)(c - 1)))
                    return true;

                if (c < ushort.MaxValue && available.Contains((ushort)(c + 1)))
                    return true;
            }
        }

        return false;
    }

    public static bool IsPapCompatible(
        IReadOnlyDictionary<string, HashSet<ushort>> localBoneSets,
        IReadOnlyDictionary<string, List<ushort>> papBoneIndices,
        AnimationValidationMode mode,
        bool allowOneBasedShift,
        bool allowNeighborTolerance,
        out string reason)
    {
        reason = string.Empty;

        if (mode == AnimationValidationMode.Unsafe)
            return true;

        var papByBucket = new Dictionary<string, List<ushort>>(StringComparer.OrdinalIgnoreCase);

        foreach (var (rawKey, list) in papBoneIndices)
        {
            var key = CanonicalizeSkeletonKey(rawKey);
            if (string.IsNullOrEmpty(key))
                continue;

            if (string.Equals(key, "skeleton", StringComparison.OrdinalIgnoreCase))
                key = "__any__";

            if (!papByBucket.TryGetValue(key, out var acc))
                papByBucket[key] = acc = [];

            if (list is { Count: > 0 })
                acc.AddRange(list);
        }

        foreach (var k in papByBucket.Keys.ToList())
            papByBucket[k] = papByBucket[k].Distinct().ToList();

        if (papByBucket.Count == 0)
        {
            reason = "No skeleton bucket bindings found in the PAP";
            return false;
        }

        static bool AllIndicesOk(
            HashSet<ushort> available,
            List<ushort> indices,
            bool papLikelyOneBased,
            bool allowOneBasedShift,
            bool allowNeighborTolerance,
            out ushort missing)
        {
            foreach (var idx in indices)
            {
                if (!ContainsIndexCompat(available, idx, papLikelyOneBased, allowOneBasedShift, allowNeighborTolerance))
                {
                    missing = idx;
                    return false;
                }
            }

            missing = 0;
            return true;
        }

        foreach (var (bucket, indices) in papByBucket)
        {
            if (indices.Count == 0)
                continue;

            bool has0 = false, has1 = false;
            ushort min = ushort.MaxValue;
            foreach (var v in indices)
            {
                if (v == 0) has0 = true;
                if (v == 1) has1 = true;
                if (v < min) min = v;
            }
            bool papLikelyOneBased = allowOneBasedShift && (min == 1) && has1 && !has0;

            if (string.Equals(bucket, "__any__", StringComparison.OrdinalIgnoreCase))
            {
                foreach (var (lk, ls) in localBoneSets)
                {
                    if (AllIndicesOk(ls, indices, papLikelyOneBased, allowOneBasedShift, allowNeighborTolerance, out _))
                        goto nextBucket; 
                }

                reason = $"No compatible local skeleton bucket for generic PAP skeleton '{bucket}'. Local buckets: {string.Join(", ", localBoneSets.Keys)}";
                return false;
            }

            if (!localBoneSets.TryGetValue(bucket, out var available))
            {
                reason = $"Missing skeleton bucket '{bucket}' on local actor.";
                return false;
            }

            if (!AllIndicesOk(available, indices, papLikelyOneBased, allowOneBasedShift, allowNeighborTolerance, out var missing))
            {
                reason = $"No compatible local skeleton for PAP '{bucket}': missing bone index {missing}.";
                return false;
            }

        nextBucket:
            ;
        }

        return true;
    }

    public void DumpLocalSkeletonIndices(GameObjectHandler handler, string? filter = null)
    {
        var skels = GetSkeletonBoneIndices(handler);
        if (skels == null)
        {
            _logger.LogTrace("DumpLocalSkeletonIndices: local skeleton indices are null or not found");
            return;
        }
        
        var keys = skels.Keys
            .Order(StringComparer.OrdinalIgnoreCase)
            .ToArray();

        _logger.LogTrace("Local skeleton indices found ({count}): {keys}",
            keys.Length,
            string.Join(", ", keys));

        if (!string.IsNullOrWhiteSpace(filter))
        {
            var hits = keys.Where(k =>
                k.Equals(filter, StringComparison.OrdinalIgnoreCase) ||
                k.StartsWith(filter + "_", StringComparison.OrdinalIgnoreCase) ||
                 filter.StartsWith(k + "_", StringComparison.OrdinalIgnoreCase) ||
                k.Contains(filter, StringComparison.OrdinalIgnoreCase))
                .ToArray();

            _logger.LogTrace("Matches found for '{filter}': {hits}",
                filter,
                hits.Length == 0 ? "<none>" : string.Join(", ", hits));
        }
    }

    public async Task<long> GetTrianglesByHash(string hash)
    {
        if (_configService.Current.TriangleDictionary.TryGetValue(hash, out var cachedTris) && cachedTris > 0)
            return cachedTris;

        if (_failedCalculatedTris.Contains(hash, StringComparer.Ordinal))
            return 0;

        var path = _fileCacheManager.GetFileCacheByHash(hash);
        if (path == null || !path.ResolvedFilepath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase))
            return 0;

        return CalculateTrianglesFromPath(hash, path.ResolvedFilepath, _configService.Current.TriangleDictionary, _failedCalculatedTris);
    }

    public long RefreshTrianglesForPath(string hash, string filePath)
    {
        if (string.IsNullOrEmpty(filePath)
            || !filePath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase)
            || !File.Exists(filePath))
        {
            return 0;
        }

        _failedCalculatedTris.RemoveAll(entry => entry.Equals(hash, StringComparison.Ordinal));
        _configService.Current.TriangleDictionary.TryRemove(hash, out _);
        return CalculateTrianglesFromPath(hash, filePath, _configService.Current.TriangleDictionary, _failedCalculatedTris);
    }

    public async Task<long> GetEffectiveTrianglesByHash(string hash, string filePath)
    {
        if (_configService.Current.EffectiveTriangleDictionary.TryGetValue(hash, out var cachedTris) && cachedTris > 0)
            return cachedTris;

        if (_failedCalculatedEffectiveTris.Contains(hash, StringComparer.Ordinal))
            return 0;

        if (string.IsNullOrEmpty(filePath)
            || !filePath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase)
            || !File.Exists(filePath))
        {
            return 0;
        }

        return CalculateTrianglesFromPath(hash, filePath, _configService.Current.EffectiveTriangleDictionary, _failedCalculatedEffectiveTris);
    }

    private long CalculateTrianglesFromPath(
        string hash,
        string filePath,
        ConcurrentDictionary<string, long> cache,
        List<string> failedList)
    {
        try
        {
            _logger.LogDebug("Detected Model File {path}, calculating Tris", filePath);
            var file = new MdlFile(filePath);
            if (file.LodCount <= 0)
            {
                failedList.Add(hash);
                cache[hash] = 0;
                _configService.Save();
                return 0;
            }

            long tris = 0;
            foreach (var lod in file.Lods)
            {
                try
                {
                    var meshIdx = lod.MeshIndex;
                    var meshCnt = lod.MeshCount;

                    tris = file.Meshes.Skip(meshIdx).Take(meshCnt).Sum(p => p.IndexCount) / 3;
                }
                catch (Exception ex)
                {
                    _logger.LogDebug(ex, "Could not load lod mesh {mesh} from path {path}", lod.MeshIndex, filePath);
                    continue;
                }

                if (tris > 0)
                {
                    _logger.LogDebug("TriAnalysis: {filePath} => {tris} triangles", filePath, tris);
                    cache[hash] = tris;
                    _configService.Save();
                    break;
                }
            }

            return tris;
        }
        catch (Exception e)
        {
            failedList.Add(hash);
            cache[hash] = 0;
            _configService.Save();
            _logger.LogWarning(e, "Could not parse file {file}", filePath);
            return 0;
        }
    }

    // Regexes for canonicalizing skeleton keys
    private static readonly Regex _bucketPathRegex =
    BucketRegex();

    private static readonly Regex _bucketSklRegex =
        SklRegex();

    private static readonly Regex _bucketLooseRegex =
        LooseBucketRegex();

    [GeneratedRegex(@"(?i)(?:^|/)(?<bucket>c\d{4})(?:/|$)", RegexOptions.Compiled, "en-NL")]
    private static partial Regex BucketRegex();

    [GeneratedRegex(@"(?i)\bskl_(?<bucket>c\d{4})[a-z]\d{4}\b", RegexOptions.Compiled, "en-NL")]
    private static partial Regex SklRegex();

    [GeneratedRegex(@"(?i)(?<![a-z0-9])(?<bucket>c\d{4})(?!\d)", RegexOptions.Compiled, "en-NL")]
    private static partial Regex LooseBucketRegex();
}