Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 51 Wiki Activity

Defenses?

Browse Source
This commit is contained in:
defnotken
2026-01-05 18:57:18 -06:00
parent 032201ed9e
commit 9b9010ab8e
2 changed files with 324 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
LightlessSync/PlayerData/Factories/PlayerDataFactory.cs
View File

@@ -566,8 +566,21 @@ public class PlayerDataFactory
await _papParseLimiter.WaitAsync(ct).ConfigureAwait(false);
try
{
papIndices = await Task.Run(() => _modelAnalyzer.GetBoneIndicesFromPap(hash), ct)
.ConfigureAwait(false);
try
{
papIndices = await Task.Run(() => _modelAnalyzer.GetBoneIndicesFromPap(hash, persistToConfig: false), ct)
.ConfigureAwait(false);
}
catch (SEHException ex)
{
_logger.LogError(ex, "SEH exception while parsing PAP file (hash={hash}, path={path}). Error code: 0x{code:X}. Skipping this animation.", hash, papPathSummary, ex.ErrorCode);
continue;
}
catch (Exception ex)
{
_logger.LogError(ex, "Unexpected error parsing PAP file (hash={hash}, path={path}). Skipping this animation.", hash, papPathSummary);
continue;
}
}
finally
{
@@ -577,36 +590,68 @@ public class PlayerDataFactory
if (papIndices == null || papIndices.Count == 0)
continue;
if (papIndices.All(k => k.Value.DefaultIfEmpty().Max() <= 105))
bool hasValidIndices = false;
try
{
hasValidIndices = papIndices.All(k => k.Value != null && k.Value.DefaultIfEmpty().Max() <= 105);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error validating bone indices for PAP (hash={hash}, path={path}). Skipping.", hash, papPathSummary);
continue;
}
if (hasValidIndices)
continue;
if (_logger.IsEnabled(LogLevel.Debug))
{
var papBuckets = papIndices
.Select(kvp => new
{
Raw = kvp.Key,
Key = XivDataAnalyzer.CanonicalizeSkeletonKey(kvp.Key),
Indices = kvp.Value
})
.Where(x => x.Indices is { Count: > 0 })
.GroupBy(x => string.IsNullOrEmpty(x.Key) ? x.Raw : x.Key!, StringComparer.OrdinalIgnoreCase)
.Select(grp =>
{
var all = grp.SelectMany(v => v.Indices).ToList();
var min = all.Count > 0 ? all.Min() : 0;
var max = all.Count > 0 ? all.Max() : 0;
var raws = string.Join(',', grp.Select(v => v.Raw).Distinct(StringComparer.OrdinalIgnoreCase));
return $"{grp.Key}(min={min},max={max},raw=[{raws}])";
})
.ToList();
try
{
var papBuckets = papIndices
.Where(kvp => kvp.Value is { Count: > 0 })
.Select(kvp => new
{
Raw = kvp.Key,
Key = XivDataAnalyzer.CanonicalizeSkeletonKey(kvp.Key),
Indices = kvp.Value
})
.Where(x => x.Indices is { Count: > 0 })
.GroupBy(x => string.IsNullOrEmpty(x.Key) ? x.Raw : x.Key!, StringComparer.OrdinalIgnoreCase)
.Select(grp =>
{
var all = grp.SelectMany(v => v.Indices).ToList();
var min = all.Count > 0 ? all.Min() : 0;
var max = all.Count > 0 ? all.Max() : 0;
var raws = string.Join(',', grp.Select(v => v.Raw).Distinct(StringComparer.OrdinalIgnoreCase));
return $"{grp.Key}(min={min},max={max},raw=[{raws}])";
})
.ToList();
_logger.LogDebug("SEND pap buckets for hash={hash}: {b}",
hash,
string.Join(" | ", papBuckets));
_logger.LogDebug("SEND pap buckets for hash={hash}: {b}",
hash,
string.Join(" | ", papBuckets));
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Error logging PAP bucket details for hash={hash}", hash);
}
}
if (XivDataAnalyzer.IsPapCompatible(localBoneSets, papIndices, mode, allowBasedShift, allownNightIndex, out var reason))
bool isCompatible = false;
string reason = string.Empty;
try
{
isCompatible = XivDataAnalyzer.IsPapCompatible(localBoneSets, papIndices, mode, allowBasedShift, allownNightIndex, out reason);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error checking PAP compatibility for hash={hash}, path={path}. Treating as incompatible.", hash, papPathSummary);
reason = $"Exception during compatibility check: {ex.Message}";
isCompatible = false;
}
if (isCompatible)
continue;
noValidationFailed++;

366
LightlessSync/Services/XivDataAnalyzer.cs
View File

@@ -1,5 +1,6 @@
using FFXIVClientStructs.FFXIV.Client.Game.Character;
using FFXIVClientStructs.FFXIV.Client.Graphics.Scene;
using FFXIVClientStructs.Havok.Common.Serialize.Resource;
using FFXIVClientStructs.Havok.Animation;
using FFXIVClientStructs.Havok.Common.Base.Types;
using FFXIVClientStructs.Havok.Common.Serialize.Util;
@@ -145,156 +146,297 @@ public sealed partial class XivDataAnalyzer
using var reader = new BinaryReader(fs);
// PAP header (mostly from vfxeditor)
_ = reader.ReadInt32(); // ignore
_ = reader.ReadInt32(); // ignore
_ = reader.ReadInt16(); // num animations
_ = reader.ReadInt16(); // modelid
var type = reader.ReadByte(); // type
if (type != 0)
return null; // not human
_ = reader.ReadByte(); // variant
_ = reader.ReadInt32(); // ignore
var havokPosition = reader.ReadInt32();
var footerPosition = reader.ReadInt32();
// sanity checks
if (havokPosition <= 0 || footerPosition <= havokPosition || footerPosition > fs.Length)
return null;
var havokDataSizeLong = (long)footerPosition - havokPosition;
if (havokDataSizeLong <= 8 || havokDataSizeLong > int.MaxValue)
return null;
var havokDataSize = (int)havokDataSizeLong;
reader.BaseStream.Position = havokPosition;
var havokData = reader.ReadBytes(havokDataSize);
if (havokData.Length <= 8)
return null;
var tempSets = new Dictionary<string, HashSet<ushort>>(StringComparer.OrdinalIgnoreCase);
var tempHavokDataPath = Path.Combine(Path.GetTempPath(), $"lightless_{Guid.NewGuid():N}.hkx");
IntPtr tempHavokDataPathAnsi = IntPtr.Zero;
try
{
File.WriteAllBytes(tempHavokDataPath, havokData);
_ = reader.ReadInt32(); // ignore
_ = reader.ReadInt32(); // ignore
var numAnimations = reader.ReadInt16(); // num animations
var modelId = reader.ReadInt16(); // modelid
if (!File.Exists(tempHavokDataPath))
if (numAnimations < 0 || numAnimations > 1000)
{
_logger.LogTrace("Temporary havok file did not exist when attempting to load: {path}", tempHavokDataPath);
_logger.LogWarning("PAP file {hash} has invalid animation count {count}, skipping", hash, numAnimations);
return null;
}
tempHavokDataPathAnsi = Marshal.StringToHGlobalAnsi(tempHavokDataPath);
var type = reader.ReadByte(); // type
if (type != 0)
return null; // not human
var loadoptions = stackalloc hkSerializeUtil.LoadOptions[1];
loadoptions->TypeInfoRegistry = hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry();
loadoptions->ClassNameRegistry = hkBuiltinTypeRegistry.Instance()->GetClassNameRegistry();
loadoptions->Flags = new hkFlags<hkSerializeUtil.LoadOptionBits, int>
{
Storage = (int)hkSerializeUtil.LoadOptionBits.Default
};
_ = reader.ReadByte(); // variant
_ = reader.ReadInt32(); // ignore
var resource = hkSerializeUtil.LoadFromFile((byte*)tempHavokDataPathAnsi, null, loadoptions);
if (resource == null)
var havokPosition = reader.ReadInt32();
var footerPosition = reader.ReadInt32();
if (havokPosition <= 0 || footerPosition <= havokPosition ||
footerPosition > fs.Length || havokPosition >= fs.Length)
{
_logger.LogWarning("Havok resource was null after loading from {path}", tempHavokDataPath);
_logger.LogWarning("PAP file {hash} has invalid offsets (havok={havok}, footer={footer}, length={length})",
hash, havokPosition, footerPosition, fs.Length);
return null;
}
var rootLevelName = @"hkRootLevelContainer"u8;
fixed (byte* n1 = rootLevelName)
var havokDataSizeLong = (long)footerPosition - havokPosition;
if (havokDataSizeLong <= 8 || havokDataSizeLong > int.MaxValue)
{
var container = (hkRootLevelContainer*)resource->GetContentsPointer(n1, hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry());
if (container == null)
return null;
_logger.LogWarning("PAP file {hash} has invalid Havok data size {size}", hash, havokDataSizeLong);
return null;
}
var animationName = @"hkaAnimationContainer"u8;
fixed (byte* n2 = animationName)
var havokDataSize = (int)havokDataSizeLong;
reader.BaseStream.Position = havokPosition;
var havokData = new byte[havokDataSize];
var bytesRead = reader.Read(havokData, 0, havokDataSize);
if (bytesRead != havokDataSize)
{
_logger.LogWarning("PAP file {hash}: Expected to read {expected} bytes but got {actual}",
hash, havokDataSize, bytesRead);
return null;
}
if (havokData.Length < 8)
return null;
var tempSets = new Dictionary<string, HashSet<ushort>>(StringComparer.OrdinalIgnoreCase);
var tempFileName = $"lightless_pap_{Guid.NewGuid():N}_{hash.Substring(0, Math.Min(8, hash.Length))}.hkx";
var tempHavokDataPath = Path.Combine(Path.GetTempPath(), tempFileName);
IntPtr tempHavokDataPathAnsi = IntPtr.Zero;
try
{
var tempDir = Path.GetDirectoryName(tempHavokDataPath);
if (!Directory.Exists(tempDir))
{
var animContainer = (hkaAnimationContainer*)container->findObjectByName(n2, null);
if (animContainer == null)
return null;
_logger.LogWarning("Temp directory {dir} doesn't exist", tempDir);
return null;
}
for (int i = 0; i < animContainer->Bindings.Length; i++)
File.WriteAllBytes(tempHavokDataPath, havokData);
if (!File.Exists(tempHavokDataPath))
{
_logger.LogWarning("Temporary havok file was not created at {path}", tempHavokDataPath);
return null;
}
var writtenFileInfo = new FileInfo(tempHavokDataPath);
if (writtenFileInfo.Length != havokData.Length)
{
_logger.LogWarning("Written temp file size mismatch: expected {expected}, got {actual}",
havokData.Length, writtenFileInfo.Length);
File.Delete(tempHavokDataPath);
return null;
}
tempHavokDataPathAnsi = Marshal.StringToHGlobalAnsi(tempHavokDataPath);
var loadoptions = stackalloc hkSerializeUtil.LoadOptions[1];
loadoptions->TypeInfoRegistry = hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry();
loadoptions->ClassNameRegistry = hkBuiltinTypeRegistry.Instance()->GetClassNameRegistry();
loadoptions->Flags = new hkFlags<hkSerializeUtil.LoadOptionBits, int>
{
Storage = (int)hkSerializeUtil.LoadOptionBits.Default
};
hkResource* resource = null;
try
{
resource = hkSerializeUtil.LoadFromFile((byte*)tempHavokDataPathAnsi, null, loadoptions);
}
catch (SEHException ex)
{
_logger.LogError(ex, "SEH exception loading Havok file from {path} (hash={hash}). Native error code: 0x{code:X}",
tempHavokDataPath, hash, ex.ErrorCode);
return null;
}
if (resource == null)
{
_logger.LogDebug("Havok resource was null after loading from {path} (hash={hash})", tempHavokDataPath, hash);
return null;
}
if ((nint)resource == nint.Zero || !IsValidPointer((IntPtr)resource))
{
_logger.LogDebug("Havok resource pointer is invalid (hash={hash})", hash);
return null;
}
var rootLevelName = @"hkRootLevelContainer"u8;
fixed (byte* n1 = rootLevelName)
{
var container = (hkRootLevelContainer*)resource->GetContentsPointer(n1, hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry());
if (container == null)
{
var binding = animContainer->Bindings[i].ptr;
if (binding == null)
continue;
_logger.LogDebug("hkRootLevelContainer is null (hash={hash})", hash);
return null;
}
var rawSkel = binding->OriginalSkeletonName.String;
var skeletonKey = CanonicalizeSkeletonKey(rawSkel);
if (string.IsNullOrEmpty(skeletonKey))
continue;
if ((nint)container == nint.Zero || !IsValidPointer((IntPtr)container))
{
_logger.LogDebug("hkRootLevelContainer pointer is invalid (hash={hash})", hash);
return null;
}
var boneTransform = binding->TransformTrackToBoneIndices;
if (boneTransform.Length <= 0)
continue;
if (!tempSets.TryGetValue(skeletonKey, out var set))
var animationName = @"hkaAnimationContainer"u8;
fixed (byte* n2 = animationName)
{
var animContainer = (hkaAnimationContainer*)container->findObjectByName(n2, null);
if (animContainer == null)
{
set = [];
tempSets[skeletonKey] = set;
_logger.LogDebug("hkaAnimationContainer is null (hash={hash})", hash);
return null;
}
for (int boneIdx = 0; boneIdx < boneTransform.Length; boneIdx++)
if ((nint)animContainer == nint.Zero || !IsValidPointer((IntPtr)animContainer))
{
var v = boneTransform[boneIdx];
if (v < 0) continue;
set.Add((ushort)v);
_logger.LogDebug("hkaAnimationContainer pointer is invalid (hash={hash})", hash);
return null;
}
if (animContainer->Bindings.Length < 0 || animContainer->Bindings.Length > 10000)
{
_logger.LogDebug("Invalid bindings count {count} (hash={hash})", animContainer->Bindings.Length, hash);
return null;
}
for (int i = 0; i < animContainer->Bindings.Length; i++)
{
var binding = animContainer->Bindings[i].ptr;
if (binding == null)
continue;
if ((nint)binding == nint.Zero || !IsValidPointer((IntPtr)binding))
{
_logger.LogDebug("Skipping invalid binding at index {index} (hash={hash})", i, hash);
continue;
}
var rawSkel = binding->OriginalSkeletonName.String;
var skeletonKey = CanonicalizeSkeletonKey(rawSkel);
if (string.IsNullOrEmpty(skeletonKey))
continue;
var boneTransform = binding->TransformTrackToBoneIndices;
if (boneTransform.Length <= 0 || boneTransform.Length > 10000)
{
_logger.LogDebug("Invalid bone transform length {length} for skeleton {skel} (hash={hash})",
boneTransform.Length, skeletonKey, hash);
continue;
}
if (!tempSets.TryGetValue(skeletonKey, out var set))
{
set = [];
tempSets[skeletonKey] = set;
}
for (int boneIdx = 0; boneIdx < boneTransform.Length; boneIdx++)
{
var v = boneTransform[boneIdx];
if (v < 0 || v > ushort.MaxValue)
continue;
set.Add((ushort)v);
}
}
}
}
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Could not load havok file in {path}", tempHavokDataPath);
return null;
}
finally
{
if (tempHavokDataPathAnsi != IntPtr.Zero)
Marshal.FreeHGlobal(tempHavokDataPathAnsi);
try
catch (SEHException ex)
{
if (File.Exists(tempHavokDataPath))
File.Delete(tempHavokDataPath);
_logger.LogError(ex, "SEH exception processing PAP file {hash} from {path}. Error code: 0x{code:X}",
hash, tempHavokDataPath, ex.ErrorCode);
return null;
}
catch (Exception ex)
{
_logger.LogTrace(ex, "Could not delete temporary havok file: {path}", tempHavokDataPath);
_logger.LogError(ex, "Managed exception loading havok file {hash} from {path}", hash, tempHavokDataPath);
return null;
}
finally
{
if (tempHavokDataPathAnsi != IntPtr.Zero)
Marshal.FreeHGlobal(tempHavokDataPathAnsi);
int retryCount = 3;
while (retryCount > 0 && File.Exists(tempHavokDataPath))
{
try
{
File.Delete(tempHavokDataPath);
break;
}
catch (IOException ex)
{
retryCount--;
if (retryCount == 0)
{
_logger.LogDebug(ex, "Failed to delete temporary havok file after retries: {path}", tempHavokDataPath);
}
else
{
Thread.Sleep(50);
}
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Unexpected error deleting temporary havok file: {path}", tempHavokDataPath);
break;
}
}
}
if (tempSets.Count == 0)
{
_logger.LogDebug("No bone sets found in PAP file (hash={hash})", hash);
return null;
}
var output = new Dictionary<string, List<ushort>>(tempSets.Count, StringComparer.OrdinalIgnoreCase);
foreach (var (key, set) in tempSets)
{
if (set.Count == 0) continue;
var list = set.ToList();
list.Sort();
output[key] = list;
}
if (output.Count == 0)
return null;
_configService.Current.BonesDictionary[hash] = output;
if (persistToConfig)
_configService.Save();
return output;
}
if (tempSets.Count == 0)
return null;
var output = new Dictionary<string, List<ushort>>(tempSets.Count, StringComparer.OrdinalIgnoreCase);
foreach (var (key, set) in tempSets)
catch (Exception ex)
{
if (set.Count == 0) continue;
var list = set.ToList();
list.Sort();
output[key] = list;
}
if (output.Count == 0)
_logger.LogError(ex, "Outer exception reading PAP file (hash={hash})", hash);
return null;
}
}
_configService.Current.BonesDictionary[hash] = output;
private static bool IsValidPointer(IntPtr ptr)
{
if (ptr == IntPtr.Zero)
return false;
if (persistToConfig)
_configService.Save();
return output;
try
{
_ = Marshal.ReadByte(ptr);
return true;
}
catch
{
return false;
}
}