From d8b9e9cf19d8253d2d11b67a5b66b60a24e95222 Mon Sep 17 00:00:00 2001 From: cake Date: Tue, 6 Jan 2026 14:27:01 +0100 Subject: [PATCH] Splitting havok tasks. --- .../PlayerData/Factories/PlayerDataFactory.cs | 18 +- .../PlayerData/Pairs/PairHandlerAdapter.cs | 71 +++- LightlessSync/Services/XivDataAnalyzer.cs | 309 +++++------------- 3 files changed, 156 insertions(+), 242 deletions(-) diff --git a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs index 3111b82..e8f3459 100644 --- a/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs +++ b/LightlessSync/PlayerData/Factories/PlayerDataFactory.cs @@ -566,9 +566,21 @@ public class PlayerDataFactory await _papParseLimiter.WaitAsync(ct).ConfigureAwait(false); try { - papIndices = await _dalamudUtil - .RunOnFrameworkThread(() => _modelAnalyzer.GetBoneIndicesFromPap(hash, persistToConfig: false)) - .ConfigureAwait(false); + var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash); + var papPath = cacheEntity?.ResolvedFilepath; + + if (!string.IsNullOrEmpty(papPath) && File.Exists(papPath)) + { + var havokBytes = await Task.Run(() => XivDataAnalyzer.ReadHavokBytesFromPap(papPath), ct) + .ConfigureAwait(false); + + if (havokBytes is { Length: > 8 }) + { + papIndices = await _dalamudUtil.RunOnFrameworkThread( + () => _modelAnalyzer.ParseHavokBytesOnFrameworkThread(havokBytes, hash, persistToConfig: false)) + .ConfigureAwait(false); + } + } } finally { diff --git a/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs b/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs index fd1db53..eded176 100644 --- a/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs +++ b/LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs @@ -121,6 +121,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa private DateTime _nextActorLookupUtc = DateTime.MinValue; private static readonly TimeSpan ActorLookupInterval = TimeSpan.FromSeconds(1); private static readonly SemaphoreSlim ActorInitializationLimiter = new(1, 1); + private static readonly SemaphoreSlim _papParseLimiter = new(1, 1); private const int FullyLoadedTimeoutMsPlayer = 30000; private const int FullyLoadedTimeoutMsOther = 5000; private readonly object _actorInitializationGate = new(); @@ -2910,13 +2911,13 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa var mode = _configService.Current.AnimationValidationMode; var allowBasedShift = _configService.Current.AnimationAllowOneBasedShift; - var allownNightIndex = _configService.Current.AnimationAllowNeighborIndexTolerance; + var allowNeighborIndex = _configService.Current.AnimationAllowNeighborIndexTolerance; if (mode == AnimationValidationMode.Unsafe || papOnly.Count == 0) return 0; var boneIndices = await _dalamudUtil.RunOnFrameworkThread( - () => _modelAnalyzer.GetSkeletonBoneIndices(handlerForApply)) + () => _modelAnalyzer.GetSkeletonBoneIndices(handlerForApply)) .ConfigureAwait(false); if (boneIndices == null || boneIndices.Count == 0) @@ -2930,47 +2931,86 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa foreach (var (rawKey, list) in boneIndices) { var key = XivDataAnalyzer.CanonicalizeSkeletonKey(rawKey); - if (string.IsNullOrEmpty(key)) continue; + if (string.IsNullOrEmpty(key) || list == null || list.Count == 0) + continue; if (!localBoneSets.TryGetValue(key, out var set)) - localBoneSets[key] = set = []; + localBoneSets[key] = set = new HashSet(); foreach (var v in list) set.Add(v); } + if (localBoneSets.Count == 0) + { + var removedCount = papOnly.Count; + papOnly.Clear(); + return removedCount; + } + int removed = 0; - foreach (var hash in papOnly.Keys.Select(k => k.Hash).Where(h => !string.IsNullOrEmpty(h)).Distinct(StringComparer.OrdinalIgnoreCase).ToList()) + var groups = papOnly + .Where(kvp => !string.IsNullOrEmpty(kvp.Key.Hash)) + .GroupBy(kvp => kvp.Key.Hash!, StringComparer.OrdinalIgnoreCase) + .ToList(); + + foreach (var grp in groups) { token.ThrowIfCancellationRequested(); - var papIndices = await _dalamudUtil.RunOnFrameworkThread( - () => _modelAnalyzer.GetBoneIndicesFromPap(hash!)) - .ConfigureAwait(false); + var hash = grp.Key; + + var papPath = grp.Select(x => x.Value) + .FirstOrDefault(p => !string.IsNullOrEmpty(p) && File.Exists(p)); + + if (string.IsNullOrEmpty(papPath)) + continue; + + var havokBytes = await Task.Run(() => XivDataAnalyzer.ReadHavokBytesFromPap(papPath), token) + .ConfigureAwait(false); + + if (havokBytes is not { Length: > 8 }) + continue; + + Dictionary>? papIndices; + + await _papParseLimiter.WaitAsync(token).ConfigureAwait(false); + try + { + papIndices = await _dalamudUtil.RunOnFrameworkThread( + () => _modelAnalyzer.ParseHavokBytesOnFrameworkThread(havokBytes, hash, persistToConfig: false)) + .ConfigureAwait(false); + } + finally + { + _papParseLimiter.Release(); + } if (papIndices == null || papIndices.Count == 0) continue; - if (papIndices.All(k => k.Value.DefaultIfEmpty().Max() <= 105)) + if (papIndices.All(k => k.Value == null || k.Value.Count == 0 || k.Value.Max() <= 105)) continue; - if (XivDataAnalyzer.IsPapCompatible(localBoneSets, papIndices, mode, allowBasedShift, allownNightIndex, out var reason)) + if (XivDataAnalyzer.IsPapCompatible(localBoneSets, papIndices, mode, allowBasedShift, allowNeighborIndex, out var reason)) continue; - var keysToRemove = papOnly.Keys.Where(k => string.Equals(k.Hash, hash, StringComparison.OrdinalIgnoreCase)).ToList(); + var keysToRemove = grp.Select(x => x.Key).ToList(); foreach (var k in keysToRemove) papOnly.Remove(k); removed += keysToRemove.Count; - if (_blockedPapHashes.TryAdd(hash!, 0)) - Logger.LogWarning("Blocked remote object PAP (hash {hash}) for {handler}: {reason}", hash, GetLogIdentifier(), reason); + if (_blockedPapHashes.TryAdd(hash, 0)) + Logger.LogWarning("Blocked remote object PAP {papPath} (hash {hash}) for {handler}: {reason}", + papPath, hash, GetLogIdentifier(), reason); if (charaData.FileReplacements.TryGetValue(ObjectKind.Player, out var list)) { - list.RemoveAll(r => string.Equals(r.Hash, hash, StringComparison.OrdinalIgnoreCase) - && r.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase))); + list.RemoveAll(r => + string.Equals(r.Hash, hash, StringComparison.OrdinalIgnoreCase) && + r.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase))); } } @@ -2984,6 +3024,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa return removed; } + private async Task ApplyCustomizeAsync(nint address, string customizeData, ObjectKind kind) { _customizeIds[kind] = await _ipcManager.CustomizePlus.SetBodyScaleAsync(address, customizeData).ConfigureAwait(false); diff --git a/LightlessSync/Services/XivDataAnalyzer.cs b/LightlessSync/Services/XivDataAnalyzer.cs index 0959c80..c15ac5c 100644 --- a/LightlessSync/Services/XivDataAnalyzer.cs +++ b/LightlessSync/Services/XivDataAnalyzer.cs @@ -132,127 +132,54 @@ public sealed partial class XivDataAnalyzer return (output.Count != 0 && output.Values.All(v => v.Count > 0)) ? output : null; } - public unsafe Dictionary>? GetBoneIndicesFromPap(string hash, bool persistToConfig = true) + public static byte[]? ReadHavokBytesFromPap(string papPath) { - if (string.IsNullOrWhiteSpace(hash)) - return null; - - if (_configService.Current.BonesDictionary.TryGetValue(hash, out var cached) && cached is not null) - return cached; - - var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash); - if (cacheEntity == null || string.IsNullOrEmpty(cacheEntity.ResolvedFilepath) || !File.Exists(cacheEntity.ResolvedFilepath)) - return null; - - using var fs = File.Open(cacheEntity.ResolvedFilepath, FileMode.Open, FileAccess.Read, FileShare.Read); + using var fs = File.Open(papPath, FileMode.Open, FileAccess.Read, FileShare.Read); using var reader = new BinaryReader(fs); - // PAP header (mostly from vfxeditor) - try - { - _ = reader.ReadInt32(); // ignore - _ = reader.ReadInt32(); // ignore - var numAnimations = reader.ReadInt16(); // num animations - var modelId = reader.ReadInt16(); // modelid + _ = reader.ReadInt32(); + _ = reader.ReadInt32(); + _ = reader.ReadInt16(); + _ = reader.ReadInt16(); - if (numAnimations < 0 || numAnimations > 1000) - { - _logger.LogWarning("PAP file {hash} has invalid animation count {count}, skipping", hash, numAnimations); - return null; - } + var type = reader.ReadByte(); + if (type != 0) return null; - var type = reader.ReadByte(); // type - if (type != 0) - return null; // not human + _ = reader.ReadByte(); + _ = reader.ReadInt32(); - _ = reader.ReadByte(); // variant - _ = reader.ReadInt32(); // ignore + var havokPosition = reader.ReadInt32(); + var footerPosition = reader.ReadInt32(); - var havokPosition = reader.ReadInt32(); - var footerPosition = reader.ReadInt32(); - - if (havokPosition <= 0 || footerPosition <= havokPosition || - footerPosition > fs.Length || havokPosition >= fs.Length) - { - _logger.LogWarning("PAP file {hash} has invalid offsets (havok={havok}, footer={footer}, length={length})", - hash, havokPosition, footerPosition, fs.Length); - return null; - } - - var havokDataSizeLong = (long)footerPosition - havokPosition; - if (havokDataSizeLong <= 8 || havokDataSizeLong > int.MaxValue) - { - _logger.LogWarning("PAP file {hash} has invalid Havok data size {size}", hash, havokDataSizeLong); - return null; - } - - var havokDataSize = (int)havokDataSizeLong; - - reader.BaseStream.Position = havokPosition; - var havokData = reader.ReadBytes(havokDataSize); - if (havokData.Length != havokDataSize) + if (havokPosition <= 0 || footerPosition <= havokPosition || footerPosition > fs.Length) return null; + var sizeLong = (long)footerPosition - havokPosition; + if (sizeLong <= 8 || sizeLong > int.MaxValue) + return null; + + var size = (int)sizeLong; + + fs.Position = havokPosition; + var bytes = reader.ReadBytes(size); + return bytes.Length > 8 ? bytes : null; + } + + public unsafe Dictionary>? ParseHavokBytesOnFrameworkThread( + byte[] havokData, + string hash, + bool persistToConfig) + { var tempSets = new Dictionary>(StringComparer.OrdinalIgnoreCase); - var tempHavokDataPath = Path.Combine(Path.GetTempPath(), $"lightless_{Guid.NewGuid():N}.hkx"); - var tempHavokDataPathAnsi = Marshal.StringToHGlobalAnsi(tempHavokDataPath); + var tempHkxPath = Path.Combine(Path.GetTempPath(), $"lightless_{Guid.NewGuid():N}.hkx"); + IntPtr pathAnsi = IntPtr.Zero; - try - { - var tempDir = Path.GetDirectoryName(tempHavokDataPath); - if (!Directory.Exists(tempDir)) - { - _logger.LogWarning("Temp directory {dir} doesn't exist", tempDir); - return null; - } + try + { + File.WriteAllBytes(tempHkxPath, havokData); - // Write the file with explicit error handling - try - { - File.WriteAllBytes(tempHavokDataPath, havokData); - } - catch (Exception writeEx) - { - _logger.LogError(writeEx, "Failed to write temporary Havok file to {path}", tempHavokDataPath); - return null; - } - - if (!File.Exists(tempHavokDataPath)) - { - _logger.LogWarning("Temporary havok file was not created at {path}", tempHavokDataPath); - return null; - } - - var writtenFileInfo = new FileInfo(tempHavokDataPath); - if (writtenFileInfo.Length != havokData.Length) - { - _logger.LogWarning("Written temp file size mismatch: expected {expected}, got {actual}", - havokData.Length, writtenFileInfo.Length); - try { File.Delete(tempHavokDataPath); } catch { } - return null; - } - - Thread.Sleep(10); // stabilize file system - - try - { - using var testStream = File.OpenRead(tempHavokDataPath); - if (testStream.Length != havokData.Length) - { - _logger.LogWarning("File verification failed: length mismatch after write"); - try { File.Delete(tempHavokDataPath); } catch { } - return null; - } - } - catch (Exception readEx) - { - _logger.LogError(readEx, "Cannot read back temporary file at {path}", tempHavokDataPath); - try { File.Delete(tempHavokDataPath); } catch { } - return null; - } - - var pathBytes = System.Text.Encoding.ASCII.GetBytes(tempHavokDataPath + "\0"); + pathAnsi = Marshal.StringToHGlobalAnsi(tempHkxPath); hkSerializeUtil.LoadOptions loadOptions = default; loadOptions.TypeInfoRegistry = hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry(); @@ -262,143 +189,77 @@ public sealed partial class XivDataAnalyzer Storage = (int)hkSerializeUtil.LoadOptionBits.Default }; - fixed (byte* pPath = pathBytes) - { - var resource = hkSerializeUtil.LoadFromFile(pPath, errorResult: null, &loadOptions); - if (resource == null) - return null; + hkSerializeUtil.LoadOptions* pOpts = &loadOptions; - var rootLevelName = @"hkRootLevelContainer"u8; - fixed (byte* n1 = rootLevelName) - { - var container = (hkRootLevelContainer*)resource->GetContentsPointer(n1, hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry()); - if (container == null) - return null; + var resource = hkSerializeUtil.LoadFromFile((byte*)pathAnsi, errorResult: null, pOpts); + if (resource == null) + return null; + + var rootLevelName = @"hkRootLevelContainer"u8; + fixed (byte* n1 = rootLevelName) + { + var container = (hkRootLevelContainer*)resource->GetContentsPointer( + n1, hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry()); + + if (container == null) return null; var animationName = @"hkaAnimationContainer"u8; fixed (byte* n2 = animationName) { var animContainer = (hkaAnimationContainer*)container->findObjectByName(n2, null); - if (animContainer == null) - return null; + if (animContainer == null) return null; - for (int i = 0; i < animContainer->Bindings.Length; i++) + for (int i = 0; i < animContainer->Bindings.Length; i++) + { + var binding = animContainer->Bindings[i].ptr; + if (binding == null) continue; + + var rawSkel = binding->OriginalSkeletonName.String; + var skeletonKey = CanonicalizeSkeletonKey(rawSkel); + if (string.IsNullOrEmpty(skeletonKey)) continue; + + var boneTransform = binding->TransformTrackToBoneIndices; + if (boneTransform.Length <= 0) continue; + + if (!tempSets.TryGetValue(skeletonKey, out var set)) + tempSets[skeletonKey] = set = []; + + for (int boneIdx = 0; boneIdx < boneTransform.Length; boneIdx++) { - var binding = animContainer->Bindings[i].ptr; - if (binding == null) - continue; - - var rawSkel = binding->OriginalSkeletonName.String; - var skeletonKey = CanonicalizeSkeletonKey(rawSkel); - if (string.IsNullOrEmpty(skeletonKey) || string.Equals(skeletonKey, "skeleton", StringComparison.OrdinalIgnoreCase)) - skeletonKey = "__any__"; - - var boneTransform = binding->TransformTrackToBoneIndices; - if (boneTransform.Length <= 0) - continue; - - if (!tempSets.TryGetValue(skeletonKey, out var set)) - { - set = []; - tempSets[skeletonKey] = set; - } - - for (int boneIdx = 0; boneIdx < boneTransform.Length; boneIdx++) - { - var v = boneTransform[boneIdx]; - if (v < 0) continue; - set.Add((ushort)v); - } + var v = boneTransform[boneIdx]; + if (v < 0) continue; + set.Add((ushort)v); } } } } } - catch (Exception ex) - { - _logger.LogWarning(ex, "Could not load havok file in {path}", tempHavokDataPath); - return null; - } finally { - if (tempHavokDataPathAnsi != IntPtr.Zero) - Marshal.FreeHGlobal(tempHavokDataPathAnsi); + if (pathAnsi != IntPtr.Zero) + Marshal.FreeHGlobal(pathAnsi); - int retryCount = 3; - while (retryCount > 0 && File.Exists(tempHavokDataPath)) - { - try - { - File.Delete(tempHavokDataPath); - break; - } - catch (IOException ex) - { - retryCount--; - if (retryCount == 0) - { - _logger.LogDebug(ex, "Failed to delete temporary havok file after retries: {path}", tempHavokDataPath); - } - else - { - Thread.Sleep(50); - } - } - catch (Exception ex) - { - _logger.LogDebug(ex, "Unexpected error deleting temporary havok file: {path}", tempHavokDataPath); - break; - } - } - } - - if (tempSets.Count == 0) - { - _logger.LogDebug("No bone sets found in PAP file (hash={hash})", hash); - return null; - } - - var output = new Dictionary>(tempSets.Count, StringComparer.OrdinalIgnoreCase); - foreach (var (key, set) in tempSets) - { - if (set.Count == 0) continue; - - var list = set.ToList(); - list.Sort(); - output[key] = list; - } - - if (output.Count == 0) - return null; - - _configService.Current.BonesDictionary[hash] = output; - - if (persistToConfig) - _configService.Save(); - - return output; + try { if (File.Exists(tempHkxPath)) File.Delete(tempHkxPath); } + catch { /* ignore */ } } - catch (Exception ex) + + if (tempSets.Count == 0) return null; + + var output = new Dictionary>(tempSets.Count, StringComparer.OrdinalIgnoreCase); + foreach (var (key, set) in tempSets) { - _logger.LogError(ex, "Outer exception reading PAP file (hash={hash})", hash); - return null; + if (set.Count == 0) continue; + var list = set.ToList(); + list.Sort(); + output[key] = list; } - } - private static bool IsValidPointer(IntPtr ptr) - { - if (ptr == IntPtr.Zero) - return false; + if (output.Count == 0) return null; - try - { - _ = Marshal.ReadByte(ptr); - return true; - } - catch - { - return false; - } + _configService.Current.BonesDictionary[hash] = output; + if (persistToConfig) _configService.Save(); + + return output; } public static string CanonicalizeSkeletonKey(string? raw)