Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 53 Wiki Activity

Splitting havok tasks.

Browse Source
This commit is contained in:
cake
2026-01-06 14:27:01 +01:00
parent ad34d88336
commit d8b9e9cf19
3 changed files with 156 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
LightlessSync/PlayerData/Factories/PlayerDataFactory.cs
View File

@@ -566,9 +566,21 @@ public class PlayerDataFactory
await _papParseLimiter.WaitAsync(ct).ConfigureAwait(false);
try
{
papIndices = await _dalamudUtil
.RunOnFrameworkThread(() => _modelAnalyzer.GetBoneIndicesFromPap(hash, persistToConfig: false))
.ConfigureAwait(false);
var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash);
var papPath = cacheEntity?.ResolvedFilepath;
if (!string.IsNullOrEmpty(papPath) && File.Exists(papPath))
{
var havokBytes = await Task.Run(() => XivDataAnalyzer.ReadHavokBytesFromPap(papPath), ct)
.ConfigureAwait(false);
if (havokBytes is { Length: > 8 })
{
papIndices = await _dalamudUtil.RunOnFrameworkThread(
() => _modelAnalyzer.ParseHavokBytesOnFrameworkThread(havokBytes, hash, persistToConfig: false))
.ConfigureAwait(false);
}
}
}
finally
{

71
LightlessSync/PlayerData/Pairs/PairHandlerAdapter.cs
View File

@@ -121,6 +121,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
private DateTime _nextActorLookupUtc = DateTime.MinValue;
private static readonly TimeSpan ActorLookupInterval = TimeSpan.FromSeconds(1);
private static readonly SemaphoreSlim ActorInitializationLimiter = new(1, 1);
private static readonly SemaphoreSlim _papParseLimiter = new(1, 1);
private const int FullyLoadedTimeoutMsPlayer = 30000;
private const int FullyLoadedTimeoutMsOther = 5000;
private readonly object _actorInitializationGate = new();
@@ -2910,13 +2911,13 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
var mode = _configService.Current.AnimationValidationMode;
var allowBasedShift = _configService.Current.AnimationAllowOneBasedShift;
var allownNightIndex = _configService.Current.AnimationAllowNeighborIndexTolerance;
var allowNeighborIndex = _configService.Current.AnimationAllowNeighborIndexTolerance;
if (mode == AnimationValidationMode.Unsafe || papOnly.Count == 0)
return 0;
var boneIndices = await _dalamudUtil.RunOnFrameworkThread(
() => _modelAnalyzer.GetSkeletonBoneIndices(handlerForApply))
() => _modelAnalyzer.GetSkeletonBoneIndices(handlerForApply))
.ConfigureAwait(false);
if (boneIndices == null || boneIndices.Count == 0)
@@ -2930,47 +2931,86 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
foreach (var (rawKey, list) in boneIndices)
{
var key = XivDataAnalyzer.CanonicalizeSkeletonKey(rawKey);
if (string.IsNullOrEmpty(key)) continue;
if (string.IsNullOrEmpty(key) || list == null || list.Count == 0)
continue;
if (!localBoneSets.TryGetValue(key, out var set))
localBoneSets[key] = set = [];
localBoneSets[key] = set = new HashSet<ushort>();
foreach (var v in list)
set.Add(v);
}
if (localBoneSets.Count == 0)
{
var removedCount = papOnly.Count;
papOnly.Clear();
return removedCount;
}
int removed = 0;
foreach (var hash in papOnly.Keys.Select(k => k.Hash).Where(h => !string.IsNullOrEmpty(h)).Distinct(StringComparer.OrdinalIgnoreCase).ToList())
var groups = papOnly
.Where(kvp => !string.IsNullOrEmpty(kvp.Key.Hash))
.GroupBy(kvp => kvp.Key.Hash!, StringComparer.OrdinalIgnoreCase)
.ToList();
foreach (var grp in groups)
{
token.ThrowIfCancellationRequested();
var papIndices = await _dalamudUtil.RunOnFrameworkThread(
() => _modelAnalyzer.GetBoneIndicesFromPap(hash!))
.ConfigureAwait(false);
var hash = grp.Key;
var papPath = grp.Select(x => x.Value)
.FirstOrDefault(p => !string.IsNullOrEmpty(p) && File.Exists(p));
if (string.IsNullOrEmpty(papPath))
continue;
var havokBytes = await Task.Run(() => XivDataAnalyzer.ReadHavokBytesFromPap(papPath), token)
.ConfigureAwait(false);
if (havokBytes is not { Length: > 8 })
continue;
Dictionary<string, List<ushort>>? papIndices;
await _papParseLimiter.WaitAsync(token).ConfigureAwait(false);
try
{
papIndices = await _dalamudUtil.RunOnFrameworkThread(
() => _modelAnalyzer.ParseHavokBytesOnFrameworkThread(havokBytes, hash, persistToConfig: false))
.ConfigureAwait(false);
}
finally
{
_papParseLimiter.Release();
}
if (papIndices == null || papIndices.Count == 0)
continue;
if (papIndices.All(k => k.Value.DefaultIfEmpty().Max() <= 105))
if (papIndices.All(k => k.Value == null || k.Value.Count == 0 || k.Value.Max() <= 105))
continue;
if (XivDataAnalyzer.IsPapCompatible(localBoneSets, papIndices, mode, allowBasedShift, allownNightIndex, out var reason))
if (XivDataAnalyzer.IsPapCompatible(localBoneSets, papIndices, mode, allowBasedShift, allowNeighborIndex, out var reason))
continue;
var keysToRemove = papOnly.Keys.Where(k => string.Equals(k.Hash, hash, StringComparison.OrdinalIgnoreCase)).ToList();
var keysToRemove = grp.Select(x => x.Key).ToList();
foreach (var k in keysToRemove)
papOnly.Remove(k);
removed += keysToRemove.Count;
if (_blockedPapHashes.TryAdd(hash!, 0))
Logger.LogWarning("Blocked remote object PAP (hash {hash}) for {handler}: {reason}", hash, GetLogIdentifier(), reason);
if (_blockedPapHashes.TryAdd(hash, 0))
Logger.LogWarning("Blocked remote object PAP {papPath} (hash {hash}) for {handler}: {reason}",
papPath, hash, GetLogIdentifier(), reason);
if (charaData.FileReplacements.TryGetValue(ObjectKind.Player, out var list))
{
list.RemoveAll(r => string.Equals(r.Hash, hash, StringComparison.OrdinalIgnoreCase)
&& r.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase)));
list.RemoveAll(r =>
string.Equals(r.Hash, hash, StringComparison.OrdinalIgnoreCase) &&
r.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase)));
}
}
@@ -2984,6 +3024,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
return removed;
}
private async Task ApplyCustomizeAsync(nint address, string customizeData, ObjectKind kind)
{
_customizeIds[kind] = await _ipcManager.CustomizePlus.SetBodyScaleAsync(address, customizeData).ConfigureAwait(false);

309
LightlessSync/Services/XivDataAnalyzer.cs
View File

@@ -132,127 +132,54 @@ public sealed partial class XivDataAnalyzer
return (output.Count != 0 && output.Values.All(v => v.Count > 0)) ? output : null;
}
public unsafe Dictionary<string, List<ushort>>? GetBoneIndicesFromPap(string hash, bool persistToConfig = true)
public static byte[]? ReadHavokBytesFromPap(string papPath)
{
if (string.IsNullOrWhiteSpace(hash))
return null;
if (_configService.Current.BonesDictionary.TryGetValue(hash, out var cached) && cached is not null)
return cached;
var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash);
if (cacheEntity == null || string.IsNullOrEmpty(cacheEntity.ResolvedFilepath) || !File.Exists(cacheEntity.ResolvedFilepath))
return null;
using var fs = File.Open(cacheEntity.ResolvedFilepath, FileMode.Open, FileAccess.Read, FileShare.Read);
using var fs = File.Open(papPath, FileMode.Open, FileAccess.Read, FileShare.Read);
using var reader = new BinaryReader(fs);
// PAP header (mostly from vfxeditor)
try
{
_ = reader.ReadInt32(); // ignore
_ = reader.ReadInt32(); // ignore
var numAnimations = reader.ReadInt16(); // num animations
var modelId = reader.ReadInt16(); // modelid
_ = reader.ReadInt32();
_ = reader.ReadInt32();
_ = reader.ReadInt16();
_ = reader.ReadInt16();
if (numAnimations < 0 || numAnimations > 1000)
{
_logger.LogWarning("PAP file {hash} has invalid animation count {count}, skipping", hash, numAnimations);
return null;
}
var type = reader.ReadByte();
if (type != 0) return null;
var type = reader.ReadByte(); // type
if (type != 0)
return null; // not human
_ = reader.ReadByte();
_ = reader.ReadInt32();
_ = reader.ReadByte(); // variant
_ = reader.ReadInt32(); // ignore
var havokPosition = reader.ReadInt32();
var footerPosition = reader.ReadInt32();
var havokPosition = reader.ReadInt32();
var footerPosition = reader.ReadInt32();
if (havokPosition <= 0 || footerPosition <= havokPosition ||
footerPosition > fs.Length || havokPosition >= fs.Length)
{
_logger.LogWarning("PAP file {hash} has invalid offsets (havok={havok}, footer={footer}, length={length})",
hash, havokPosition, footerPosition, fs.Length);
return null;
}
var havokDataSizeLong = (long)footerPosition - havokPosition;
if (havokDataSizeLong <= 8 || havokDataSizeLong > int.MaxValue)
{
_logger.LogWarning("PAP file {hash} has invalid Havok data size {size}", hash, havokDataSizeLong);
return null;
}
var havokDataSize = (int)havokDataSizeLong;
reader.BaseStream.Position = havokPosition;
var havokData = reader.ReadBytes(havokDataSize);
if (havokData.Length != havokDataSize)
if (havokPosition <= 0 || footerPosition <= havokPosition || footerPosition > fs.Length)
return null;
var sizeLong = (long)footerPosition - havokPosition;
if (sizeLong <= 8 || sizeLong > int.MaxValue)
return null;
var size = (int)sizeLong;
fs.Position = havokPosition;
var bytes = reader.ReadBytes(size);
return bytes.Length > 8 ? bytes : null;
}
public unsafe Dictionary<string, List<ushort>>? ParseHavokBytesOnFrameworkThread(
byte[] havokData,
string hash,
bool persistToConfig)
{
var tempSets = new Dictionary<string, HashSet<ushort>>(StringComparer.OrdinalIgnoreCase);
var tempHavokDataPath = Path.Combine(Path.GetTempPath(), $"lightless_{Guid.NewGuid():N}.hkx");
var tempHavokDataPathAnsi = Marshal.StringToHGlobalAnsi(tempHavokDataPath);
var tempHkxPath = Path.Combine(Path.GetTempPath(), $"lightless_{Guid.NewGuid():N}.hkx");
IntPtr pathAnsi = IntPtr.Zero;
try
{
var tempDir = Path.GetDirectoryName(tempHavokDataPath);
if (!Directory.Exists(tempDir))
{
_logger.LogWarning("Temp directory {dir} doesn't exist", tempDir);
return null;
}
try
{
File.WriteAllBytes(tempHkxPath, havokData);
// Write the file with explicit error handling
try
{
File.WriteAllBytes(tempHavokDataPath, havokData);
}
catch (Exception writeEx)
{
_logger.LogError(writeEx, "Failed to write temporary Havok file to {path}", tempHavokDataPath);
return null;
}
if (!File.Exists(tempHavokDataPath))
{
_logger.LogWarning("Temporary havok file was not created at {path}", tempHavokDataPath);
return null;
}
var writtenFileInfo = new FileInfo(tempHavokDataPath);
if (writtenFileInfo.Length != havokData.Length)
{
_logger.LogWarning("Written temp file size mismatch: expected {expected}, got {actual}",
havokData.Length, writtenFileInfo.Length);
try { File.Delete(tempHavokDataPath); } catch { }
return null;
}
Thread.Sleep(10); // stabilize file system
try
{
using var testStream = File.OpenRead(tempHavokDataPath);
if (testStream.Length != havokData.Length)
{
_logger.LogWarning("File verification failed: length mismatch after write");
try { File.Delete(tempHavokDataPath); } catch { }
return null;
}
}
catch (Exception readEx)
{
_logger.LogError(readEx, "Cannot read back temporary file at {path}", tempHavokDataPath);
try { File.Delete(tempHavokDataPath); } catch { }
return null;
}
var pathBytes = System.Text.Encoding.ASCII.GetBytes(tempHavokDataPath + "\0");
pathAnsi = Marshal.StringToHGlobalAnsi(tempHkxPath);
hkSerializeUtil.LoadOptions loadOptions = default;
loadOptions.TypeInfoRegistry = hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry();
@@ -262,143 +189,77 @@ public sealed partial class XivDataAnalyzer
Storage = (int)hkSerializeUtil.LoadOptionBits.Default
};
fixed (byte* pPath = pathBytes)
{
var resource = hkSerializeUtil.LoadFromFile(pPath, errorResult: null, &loadOptions);
if (resource == null)
return null;
hkSerializeUtil.LoadOptions* pOpts = &loadOptions;
var rootLevelName = @"hkRootLevelContainer"u8;
fixed (byte* n1 = rootLevelName)
{
var container = (hkRootLevelContainer*)resource->GetContentsPointer(n1, hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry());
if (container == null)
return null;
var resource = hkSerializeUtil.LoadFromFile((byte*)pathAnsi, errorResult: null, pOpts);
if (resource == null)
return null;
var rootLevelName = @"hkRootLevelContainer"u8;
fixed (byte* n1 = rootLevelName)
{
var container = (hkRootLevelContainer*)resource->GetContentsPointer(
n1, hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry());
if (container == null) return null;
var animationName = @"hkaAnimationContainer"u8;
fixed (byte* n2 = animationName)
{
var animContainer = (hkaAnimationContainer*)container->findObjectByName(n2, null);
if (animContainer == null)
return null;
if (animContainer == null) return null;
for (int i = 0; i < animContainer->Bindings.Length; i++)
for (int i = 0; i < animContainer->Bindings.Length; i++)
{
var binding = animContainer->Bindings[i].ptr;
if (binding == null) continue;
var rawSkel = binding->OriginalSkeletonName.String;
var skeletonKey = CanonicalizeSkeletonKey(rawSkel);
if (string.IsNullOrEmpty(skeletonKey)) continue;
var boneTransform = binding->TransformTrackToBoneIndices;
if (boneTransform.Length <= 0) continue;
if (!tempSets.TryGetValue(skeletonKey, out var set))
tempSets[skeletonKey] = set = [];
for (int boneIdx = 0; boneIdx < boneTransform.Length; boneIdx++)
{
var binding = animContainer->Bindings[i].ptr;
if (binding == null)
continue;
var rawSkel = binding->OriginalSkeletonName.String;
var skeletonKey = CanonicalizeSkeletonKey(rawSkel);
if (string.IsNullOrEmpty(skeletonKey) || string.Equals(skeletonKey, "skeleton", StringComparison.OrdinalIgnoreCase))
skeletonKey = "__any__";
var boneTransform = binding->TransformTrackToBoneIndices;
if (boneTransform.Length <= 0)
continue;
if (!tempSets.TryGetValue(skeletonKey, out var set))
{
set = [];
tempSets[skeletonKey] = set;
}
for (int boneIdx = 0; boneIdx < boneTransform.Length; boneIdx++)
{
var v = boneTransform[boneIdx];
if (v < 0) continue;
set.Add((ushort)v);
}
var v = boneTransform[boneIdx];
if (v < 0) continue;
set.Add((ushort)v);
}
}
}
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Could not load havok file in {path}", tempHavokDataPath);
return null;
}
finally
{
if (tempHavokDataPathAnsi != IntPtr.Zero)
Marshal.FreeHGlobal(tempHavokDataPathAnsi);
if (pathAnsi != IntPtr.Zero)
Marshal.FreeHGlobal(pathAnsi);
int retryCount = 3;
while (retryCount > 0 && File.Exists(tempHavokDataPath))
{
try
{
File.Delete(tempHavokDataPath);
break;
}
catch (IOException ex)
{
retryCount--;
if (retryCount == 0)
{
_logger.LogDebug(ex, "Failed to delete temporary havok file after retries: {path}", tempHavokDataPath);
}
else
{
Thread.Sleep(50);
}
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Unexpected error deleting temporary havok file: {path}", tempHavokDataPath);
break;
}
}
}
if (tempSets.Count == 0)
{
_logger.LogDebug("No bone sets found in PAP file (hash={hash})", hash);
return null;
}
var output = new Dictionary<string, List<ushort>>(tempSets.Count, StringComparer.OrdinalIgnoreCase);
foreach (var (key, set) in tempSets)
{
if (set.Count == 0) continue;
var list = set.ToList();
list.Sort();
output[key] = list;
}
if (output.Count == 0)
return null;
_configService.Current.BonesDictionary[hash] = output;
if (persistToConfig)
_configService.Save();
return output;
try { if (File.Exists(tempHkxPath)) File.Delete(tempHkxPath); }
catch { /* ignore */ }
}
catch (Exception ex)
if (tempSets.Count == 0) return null;
var output = new Dictionary<string, List<ushort>>(tempSets.Count, StringComparer.OrdinalIgnoreCase);
foreach (var (key, set) in tempSets)
{
_logger.LogError(ex, "Outer exception reading PAP file (hash={hash})", hash);
return null;
if (set.Count == 0) continue;
var list = set.ToList();
list.Sort();
output[key] = list;
}
}
private static bool IsValidPointer(IntPtr ptr)
{
if (ptr == IntPtr.Zero)
return false;
if (output.Count == 0) return null;
try
{
_ = Marshal.ReadByte(ptr);
return true;
}
catch
{
return false;
}
_configService.Current.BonesDictionary[hash] = output;
if (persistToConfig) _configService.Save();
return output;
}
public static string CanonicalizeSkeletonKey(string? raw)