Merge branch '2.0.3' into dev

LightlessSync/PlayerData/Factories/PlayerDataFactory.cs

@@ -566,8 +566,21 @@ public class PlayerDataFactory
             await _papParseLimiter.WaitAsync(ct).ConfigureAwait(false);
             try
             {
-                papIndices = await Task.Run(() => _modelAnalyzer.GetBoneIndicesFromPap(hash), ct)
+                try
-                    .ConfigureAwait(false);
+                {
+                    papIndices = await Task.Run(() => _modelAnalyzer.GetBoneIndicesFromPap(hash, persistToConfig: false), ct)
+                        .ConfigureAwait(false);
+                }
+                catch (SEHException ex)
+                {
+                    _logger.LogError(ex, "SEH exception while parsing PAP file (hash={hash}, path={path}). Error code: 0x{code:X}. Skipping this animation.", hash, papPathSummary, ex.ErrorCode);
+                    continue;
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError(ex, "Unexpected error parsing PAP file (hash={hash}, path={path}). Skipping this animation.", hash, papPathSummary);
+                    continue;
+                }
             }
             finally
             {
@@ -577,36 +590,68 @@ public class PlayerDataFactory
             if (papIndices == null || papIndices.Count == 0)
                 continue;
 
-            if (papIndices.All(k => k.Value.DefaultIfEmpty().Max() <= 105))
+            bool hasValidIndices = false;
+            try
+            {
+                hasValidIndices = papIndices.All(k => k.Value != null && k.Value.DefaultIfEmpty().Max() <= 105);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Error validating bone indices for PAP (hash={hash}, path={path}). Skipping.", hash, papPathSummary);
+                continue;
+            }
+
+            if (hasValidIndices)
                 continue;
 
             if (_logger.IsEnabled(LogLevel.Debug))
             {
-                var papBuckets = papIndices
+                try
-                    .Select(kvp => new
+                {
-                    {
+                    var papBuckets = papIndices
-                        Raw = kvp.Key,
+                        .Where(kvp => kvp.Value is { Count: > 0 })
-                        Key = XivDataAnalyzer.CanonicalizeSkeletonKey(kvp.Key),
+                        .Select(kvp => new
-                        Indices = kvp.Value
+                        {
-                    })
+                            Raw = kvp.Key,
-                    .Where(x => x.Indices is { Count: > 0 })
+                            Key = XivDataAnalyzer.CanonicalizeSkeletonKey(kvp.Key),
-                    .GroupBy(x => string.IsNullOrEmpty(x.Key) ? x.Raw : x.Key!, StringComparer.OrdinalIgnoreCase)
+                            Indices = kvp.Value
-                    .Select(grp =>
+                        })
-                    {
+                        .Where(x => x.Indices is { Count: > 0 })
-                        var all = grp.SelectMany(v => v.Indices).ToList();
+                        .GroupBy(x => string.IsNullOrEmpty(x.Key) ? x.Raw : x.Key!, StringComparer.OrdinalIgnoreCase)
-                        var min = all.Count > 0 ? all.Min() : 0;
+                        .Select(grp =>
-                        var max = all.Count > 0 ? all.Max() : 0;
+                        {
-                        var raws = string.Join(',', grp.Select(v => v.Raw).Distinct(StringComparer.OrdinalIgnoreCase));
+                            var all = grp.SelectMany(v => v.Indices).ToList();
-                        return $"{grp.Key}(min={min},max={max},raw=[{raws}])";
+                            var min = all.Count > 0 ? all.Min() : 0;
-                    })
+                            var max = all.Count > 0 ? all.Max() : 0;
-                    .ToList();
+                            var raws = string.Join(',', grp.Select(v => v.Raw).Distinct(StringComparer.OrdinalIgnoreCase));
+                            return $"{grp.Key}(min={min},max={max},raw=[{raws}])";
+                        })
+                        .ToList();
 
-                _logger.LogDebug("SEND pap buckets for hash={hash}: {b}",
+                    _logger.LogDebug("SEND pap buckets for hash={hash}: {b}",
-                    hash,
+                        hash,
-                    string.Join(" | ", papBuckets));
+                        string.Join(" | ", papBuckets));
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogDebug(ex, "Error logging PAP bucket details for hash={hash}", hash);
+                }
             }
 
-            if (XivDataAnalyzer.IsPapCompatible(localBoneSets, papIndices, mode, allowBasedShift, allownNightIndex, out var reason))
+            bool isCompatible = false;
+            string reason = string.Empty;
+            try
+            {
+                isCompatible = XivDataAnalyzer.IsPapCompatible(localBoneSets, papIndices, mode, allowBasedShift, allownNightIndex, out reason);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Error checking PAP compatibility for hash={hash}, path={path}. Treating as incompatible.", hash, papPathSummary);
+                reason = $"Exception during compatibility check: {ex.Message}";
+                isCompatible = false;
+            }
+
+            if (isCompatible)
                 continue;
 
             noValidationFailed++;

LightlessSync/Services/XivDataAnalyzer.cs

@@ -1,5 +1,6 @@
 using FFXIVClientStructs.FFXIV.Client.Game.Character;
 using FFXIVClientStructs.FFXIV.Client.Graphics.Scene;
+using FFXIVClientStructs.Havok.Common.Serialize.Resource;
 using FFXIVClientStructs.Havok.Animation;
 using FFXIVClientStructs.Havok.Common.Base.Types;
 using FFXIVClientStructs.Havok.Common.Serialize.Util;
@@ -145,156 +146,297 @@ public sealed partial class XivDataAnalyzer
         using var reader = new BinaryReader(fs);
 
         // PAP header (mostly from vfxeditor)
-        _ = reader.ReadInt32(); // ignore
-        _ = reader.ReadInt32(); // ignore
-        _ = reader.ReadInt16(); // num animations
-        _ = reader.ReadInt16(); // modelid
-
-        var type = reader.ReadByte(); // type
-        if (type != 0)
-            return null; // not human
-
-        _ = reader.ReadByte();  // variant
-        _ = reader.ReadInt32(); // ignore
-
-        var havokPosition = reader.ReadInt32();
-        var footerPosition = reader.ReadInt32();
-
-        // sanity checks
-        if (havokPosition <= 0 || footerPosition <= havokPosition || footerPosition > fs.Length)
-            return null;
-
-        var havokDataSizeLong = (long)footerPosition - havokPosition;
-        if (havokDataSizeLong <= 8 || havokDataSizeLong > int.MaxValue)
-            return null;
-
-        var havokDataSize = (int)havokDataSizeLong;
-
-        reader.BaseStream.Position = havokPosition;
-        var havokData = reader.ReadBytes(havokDataSize);
-        if (havokData.Length <= 8)
-            return null;
-
-        var tempSets = new Dictionary<string, HashSet<ushort>>(StringComparer.OrdinalIgnoreCase);
-
-        var tempHavokDataPath = Path.Combine(Path.GetTempPath(), $"lightless_{Guid.NewGuid():N}.hkx");
-        IntPtr tempHavokDataPathAnsi = IntPtr.Zero;
-
         try
         {
-            File.WriteAllBytes(tempHavokDataPath, havokData);
+            _ = reader.ReadInt32(); // ignore
+            _ = reader.ReadInt32(); // ignore
+            var numAnimations = reader.ReadInt16(); // num animations
+            var modelId = reader.ReadInt16(); // modelid
 
-            if (!File.Exists(tempHavokDataPath))
+            if (numAnimations < 0 || numAnimations > 1000)
             {
-                _logger.LogTrace("Temporary havok file did not exist when attempting to load: {path}", tempHavokDataPath);
+                _logger.LogWarning("PAP file {hash} has invalid animation count {count}, skipping", hash, numAnimations);
                 return null;
             }
 
-            tempHavokDataPathAnsi = Marshal.StringToHGlobalAnsi(tempHavokDataPath);
+            var type = reader.ReadByte(); // type
+            if (type != 0)
+                return null; // not human
 
-            var loadoptions = stackalloc hkSerializeUtil.LoadOptions[1];
+            _ = reader.ReadByte();  // variant
-            loadoptions->TypeInfoRegistry = hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry();
+            _ = reader.ReadInt32(); // ignore
-            loadoptions->ClassNameRegistry = hkBuiltinTypeRegistry.Instance()->GetClassNameRegistry();
-            loadoptions->Flags = new hkFlags<hkSerializeUtil.LoadOptionBits, int>
-            {
-                Storage = (int)hkSerializeUtil.LoadOptionBits.Default
-            };
 
-            var resource = hkSerializeUtil.LoadFromFile((byte*)tempHavokDataPathAnsi, null, loadoptions);
+            var havokPosition = reader.ReadInt32();
-            if (resource == null)
+            var footerPosition = reader.ReadInt32();
+
+            if (havokPosition <= 0 || footerPosition <= havokPosition ||
+                footerPosition > fs.Length || havokPosition >= fs.Length)
             {
-                _logger.LogWarning("Havok resource was null after loading from {path}", tempHavokDataPath);
+                _logger.LogWarning("PAP file {hash} has invalid offsets (havok={havok}, footer={footer}, length={length})",
+                    hash, havokPosition, footerPosition, fs.Length);
                 return null;
             }
 
-            var rootLevelName = @"hkRootLevelContainer"u8;
+            var havokDataSizeLong = (long)footerPosition - havokPosition;
-            fixed (byte* n1 = rootLevelName)
+            if (havokDataSizeLong <= 8 || havokDataSizeLong > int.MaxValue) 
             {
-                var container = (hkRootLevelContainer*)resource->GetContentsPointer(n1, hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry());
+                _logger.LogWarning("PAP file {hash} has invalid Havok data size {size}", hash, havokDataSizeLong);
-                if (container == null)
+                return null;
-                    return null;
+            }
 
-                var animationName = @"hkaAnimationContainer"u8;
+            var havokDataSize = (int)havokDataSizeLong;
-                fixed (byte* n2 = animationName)
+
+            reader.BaseStream.Position = havokPosition;
+
+            var havokData = new byte[havokDataSize];
+            var bytesRead = reader.Read(havokData, 0, havokDataSize);
+            if (bytesRead != havokDataSize)
+            {
+                _logger.LogWarning("PAP file {hash}: Expected to read {expected} bytes but got {actual}",
+                    hash, havokDataSize, bytesRead);
+                return null;
+            }
+
+            if (havokData.Length < 8)
+                return null;
+
+            var tempSets = new Dictionary<string, HashSet<ushort>>(StringComparer.OrdinalIgnoreCase);
+
+            var tempFileName = $"lightless_pap_{Guid.NewGuid():N}_{hash.Substring(0, Math.Min(8, hash.Length))}.hkx";
+            var tempHavokDataPath = Path.Combine(Path.GetTempPath(), tempFileName);
+            IntPtr tempHavokDataPathAnsi = IntPtr.Zero;
+
+            try
+            {
+                var tempDir = Path.GetDirectoryName(tempHavokDataPath);
+                if (!Directory.Exists(tempDir))
                 {
-                    var animContainer = (hkaAnimationContainer*)container->findObjectByName(n2, null);
+                    _logger.LogWarning("Temp directory {dir} doesn't exist", tempDir);
-                    if (animContainer == null)
+                    return null;
-                        return null;
+                }
 
-                    for (int i = 0; i < animContainer->Bindings.Length; i++)
+                File.WriteAllBytes(tempHavokDataPath, havokData);
+
+                if (!File.Exists(tempHavokDataPath))
+                {
+                    _logger.LogWarning("Temporary havok file was not created at {path}", tempHavokDataPath);
+                    return null;
+                }
+
+                var writtenFileInfo = new FileInfo(tempHavokDataPath);
+                if (writtenFileInfo.Length != havokData.Length)
+                {
+                    _logger.LogWarning("Written temp file size mismatch: expected {expected}, got {actual}",
+                        havokData.Length, writtenFileInfo.Length);
+                    File.Delete(tempHavokDataPath);
+                    return null;
+                }
+
+                tempHavokDataPathAnsi = Marshal.StringToHGlobalAnsi(tempHavokDataPath);
+
+                var loadoptions = stackalloc hkSerializeUtil.LoadOptions[1];
+                loadoptions->TypeInfoRegistry = hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry();
+                loadoptions->ClassNameRegistry = hkBuiltinTypeRegistry.Instance()->GetClassNameRegistry();
+                loadoptions->Flags = new hkFlags<hkSerializeUtil.LoadOptionBits, int>
+                {
+                    Storage = (int)hkSerializeUtil.LoadOptionBits.Default
+                };
+
+                hkResource* resource = null;
+                try
+                {
+                    resource = hkSerializeUtil.LoadFromFile((byte*)tempHavokDataPathAnsi, null, loadoptions);
+                }
+                catch (SEHException ex)
+                {
+                    _logger.LogError(ex, "SEH exception loading Havok file from {path} (hash={hash}). Native error code: 0x{code:X}",
+                        tempHavokDataPath, hash, ex.ErrorCode);
+                    return null;
+                }
+
+                if (resource == null)
+                {
+                    _logger.LogDebug("Havok resource was null after loading from {path} (hash={hash})", tempHavokDataPath, hash);
+                    return null;
+                }
+
+                if ((nint)resource == nint.Zero || !IsValidPointer((IntPtr)resource))
+                {
+                    _logger.LogDebug("Havok resource pointer is invalid (hash={hash})", hash);
+                    return null;
+                }
+
+                var rootLevelName = @"hkRootLevelContainer"u8;
+                fixed (byte* n1 = rootLevelName)
+                {
+                    var container = (hkRootLevelContainer*)resource->GetContentsPointer(n1, hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry());
+                    if (container == null)
                     {
-                        var binding = animContainer->Bindings[i].ptr;
+                        _logger.LogDebug("hkRootLevelContainer is null (hash={hash})", hash);
-                        if (binding == null)
+                        return null;
-                            continue;
+                    }
 
-                        var rawSkel = binding->OriginalSkeletonName.String;
+                    if ((nint)container == nint.Zero || !IsValidPointer((IntPtr)container))
-                        var skeletonKey = CanonicalizeSkeletonKey(rawSkel);
+                    {
-                        if (string.IsNullOrEmpty(skeletonKey))
+                        _logger.LogDebug("hkRootLevelContainer pointer is invalid (hash={hash})", hash);
-                            continue;
+                        return null;
+                    }
 
-                        var boneTransform = binding->TransformTrackToBoneIndices;
+                    var animationName = @"hkaAnimationContainer"u8;
-                        if (boneTransform.Length <= 0)
+                    fixed (byte* n2 = animationName)
-                            continue;
+                    {
-
+                        var animContainer = (hkaAnimationContainer*)container->findObjectByName(n2, null);
-                        if (!tempSets.TryGetValue(skeletonKey, out var set))
+                        if (animContainer == null)
                         {
-                            set = [];
+                            _logger.LogDebug("hkaAnimationContainer is null (hash={hash})", hash);
-                            tempSets[skeletonKey] = set;
+                            return null;
                         }
 
-                        for (int boneIdx = 0; boneIdx < boneTransform.Length; boneIdx++)
+                        if ((nint)animContainer == nint.Zero || !IsValidPointer((IntPtr)animContainer))
                         {
-                            var v = boneTransform[boneIdx];
+                            _logger.LogDebug("hkaAnimationContainer pointer is invalid (hash={hash})", hash);
-                            if (v < 0) continue;
+                            return null;
-                            set.Add((ushort)v);
+                        }
+
+                        if (animContainer->Bindings.Length < 0 || animContainer->Bindings.Length > 10000)
+                        {
+                            _logger.LogDebug("Invalid bindings count {count} (hash={hash})", animContainer->Bindings.Length, hash);
+                            return null;
+                        }
+
+                        for (int i = 0; i < animContainer->Bindings.Length; i++)
+                        {
+                            var binding = animContainer->Bindings[i].ptr;
+                            if (binding == null)
+                                continue;
+
+                            if ((nint)binding == nint.Zero || !IsValidPointer((IntPtr)binding))
+                            {
+                                _logger.LogDebug("Skipping invalid binding at index {index} (hash={hash})", i, hash);
+                                continue;
+                            }
+
+                            var rawSkel = binding->OriginalSkeletonName.String;
+                            var skeletonKey = CanonicalizeSkeletonKey(rawSkel);
+                            if (string.IsNullOrEmpty(skeletonKey))
+                                continue;
+
+                            var boneTransform = binding->TransformTrackToBoneIndices;
+                            if (boneTransform.Length <= 0 || boneTransform.Length > 10000)
+                            {
+                                _logger.LogDebug("Invalid bone transform length {length} for skeleton {skel} (hash={hash})",
+                                    boneTransform.Length, skeletonKey, hash);
+                                continue;
+                            }
+
+                            if (!tempSets.TryGetValue(skeletonKey, out var set))
+                            {
+                                set = [];
+                                tempSets[skeletonKey] = set;
+                            }
+
+                            for (int boneIdx = 0; boneIdx < boneTransform.Length; boneIdx++)
+                            {
+                                var v = boneTransform[boneIdx];
+                                if (v < 0 || v > ushort.MaxValue)
+                                    continue;
+                                set.Add((ushort)v);
+                            }
                         }
                     }
                 }
             }
-        }
+            catch (SEHException ex)
-        catch (Exception ex)
-        {
-            _logger.LogWarning(ex, "Could not load havok file in {path}", tempHavokDataPath);
-            return null;
-        }
-        finally
-        {
-            if (tempHavokDataPathAnsi != IntPtr.Zero)
-                Marshal.FreeHGlobal(tempHavokDataPathAnsi);
-
-            try
             {
-                if (File.Exists(tempHavokDataPath))
+                _logger.LogError(ex, "SEH exception processing PAP file {hash} from {path}. Error code: 0x{code:X}",
-                    File.Delete(tempHavokDataPath);
+                    hash, tempHavokDataPath, ex.ErrorCode);
+                return null;
             }
             catch (Exception ex)
             {
-                _logger.LogTrace(ex, "Could not delete temporary havok file: {path}", tempHavokDataPath);
+                _logger.LogError(ex, "Managed exception loading havok file {hash} from {path}", hash, tempHavokDataPath);
+                return null;
             }
+            finally
+            {
+                if (tempHavokDataPathAnsi != IntPtr.Zero)
+                    Marshal.FreeHGlobal(tempHavokDataPathAnsi);
+
+                int retryCount = 3;
+                while (retryCount > 0 && File.Exists(tempHavokDataPath))
+                {
+                    try
+                    {
+                        File.Delete(tempHavokDataPath);
+                        break;
+                    }
+                    catch (IOException ex)
+                    {
+                        retryCount--;
+                        if (retryCount == 0)
+                        {
+                            _logger.LogDebug(ex, "Failed to delete temporary havok file after retries: {path}", tempHavokDataPath);
+                        }
+                        else
+                        {
+                            Thread.Sleep(50);
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogDebug(ex, "Unexpected error deleting temporary havok file: {path}", tempHavokDataPath);
+                        break;
+                    }
+                }
+            }
+
+            if (tempSets.Count == 0)
+            {
+                _logger.LogDebug("No bone sets found in PAP file (hash={hash})", hash);
+                return null;
+            }
+
+            var output = new Dictionary<string, List<ushort>>(tempSets.Count, StringComparer.OrdinalIgnoreCase);
+            foreach (var (key, set) in tempSets)
+            {
+                if (set.Count == 0) continue;
+
+                var list = set.ToList();
+                list.Sort();
+                output[key] = list;
+            }
+
+            if (output.Count == 0)
+                return null;
+
+            _configService.Current.BonesDictionary[hash] = output;
+
+            if (persistToConfig)
+                _configService.Save();
+
+            return output;
         }
-
+        catch (Exception ex)
-        if (tempSets.Count == 0)
-            return null;
-
-        var output = new Dictionary<string, List<ushort>>(tempSets.Count, StringComparer.OrdinalIgnoreCase);
-        foreach (var (key, set) in tempSets)
         {
-            if (set.Count == 0) continue;
+            _logger.LogError(ex, "Outer exception reading PAP file (hash={hash})", hash);
-
-            var list = set.ToList();
-            list.Sort();
-            output[key] = list;
-        }
-
-        if (output.Count == 0)
             return null;
+        }
+    }
 
-        _configService.Current.BonesDictionary[hash] = output;
+    private static bool IsValidPointer(IntPtr ptr)
+    {
+        if (ptr == IntPtr.Zero)
+            return false;
 
-        if (persistToConfig)
+        try
-            _configService.Save();
+        {
-
+            _ = Marshal.ReadByte(ptr);
-        return output;
+            return true;
+        }
+        catch
+        {
+            return false;
+        }
     }