Files
LightlessClient/LightlessSync/Services/XivDataAnalyzer.cs
defnotken 9b9010ab8e Defenses?
2026-01-05 18:57:18 -06:00

720 lines
26 KiB
C#

using FFXIVClientStructs.FFXIV.Client.Game.Character;
using FFXIVClientStructs.FFXIV.Client.Graphics.Scene;
using FFXIVClientStructs.Havok.Common.Serialize.Resource;
using FFXIVClientStructs.Havok.Animation;
using FFXIVClientStructs.Havok.Common.Base.Types;
using FFXIVClientStructs.Havok.Common.Serialize.Util;
using LightlessSync.FileCache;
using LightlessSync.Interop.GameModel;
using LightlessSync.LightlessConfiguration;
using LightlessSync.PlayerData.Factories;
using LightlessSync.PlayerData.Handlers;
using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
namespace LightlessSync.Services;
public sealed partial class XivDataAnalyzer
{
private readonly ILogger<XivDataAnalyzer> _logger;
private readonly FileCacheManager _fileCacheManager;
private readonly XivDataStorageService _configService;
private readonly List<string> _failedCalculatedTris = [];
private readonly List<string> _failedCalculatedEffectiveTris = [];
public XivDataAnalyzer(ILogger<XivDataAnalyzer> logger, FileCacheManager fileCacheManager,
XivDataStorageService configService)
{
_logger = logger;
_fileCacheManager = fileCacheManager;
_configService = configService;
}
public unsafe Dictionary<string, List<ushort>>? GetSkeletonBoneIndices(GameObjectHandler handler)
{
if (handler is null || handler.Address == nint.Zero)
return null;
Dictionary<string, HashSet<ushort>> sets = new(StringComparer.OrdinalIgnoreCase);
try
{
var drawObject = ((Character*)handler.Address)->GameObject.DrawObject;
if (drawObject == null)
return null;
var chara = (CharacterBase*)drawObject;
if (chara->GetModelType() != CharacterBase.ModelType.Human)
return null;
var skeleton = chara->Skeleton;
if (skeleton == null)
return null;
var resHandles = skeleton->SkeletonResourceHandles;
var partialCount = skeleton->PartialSkeletonCount;
if (partialCount <= 0)
return null;
for (int i = 0; i < partialCount; i++)
{
var handle = *(resHandles + i);
if ((nint)handle == nint.Zero)
continue;
if (handle->FileName.Length > 1024)
continue;
var rawName = handle->FileName.ToString();
if (string.IsNullOrWhiteSpace(rawName))
continue;
var skeletonKey = CanonicalizeSkeletonKey(rawName);
if (string.IsNullOrEmpty(skeletonKey))
continue;
var boneCount = handle->BoneCount;
if (boneCount == 0)
continue;
var havokSkel = handle->HavokSkeleton;
if ((nint)havokSkel == nint.Zero)
continue;
if (!sets.TryGetValue(skeletonKey, out var set))
{
set = [];
sets[skeletonKey] = set;
}
uint maxExclusive = boneCount;
uint ushortExclusive = (uint)ushort.MaxValue + 1u;
if (maxExclusive > ushortExclusive)
maxExclusive = ushortExclusive;
for (uint boneIdx = 0; boneIdx < maxExclusive; boneIdx++)
{
var name = havokSkel->Bones[boneIdx].Name.String;
if (name == null)
continue;
set.Add((ushort)boneIdx);
}
_logger.LogTrace("Local skeleton raw file='{raw}', key='{key}', boneCount={count}",
rawName, skeletonKey, boneCount);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Could not process skeleton data");
return null;
}
if (sets.Count == 0)
return null;
var output = new Dictionary<string, List<ushort>>(sets.Count, StringComparer.OrdinalIgnoreCase);
foreach (var (key, set) in sets)
{
if (set.Count == 0)
continue;
var list = set.ToList();
list.Sort();
output[key] = list;
}
return (output.Count != 0 && output.Values.All(v => v.Count > 0)) ? output : null;
}
public unsafe Dictionary<string, List<ushort>>? GetBoneIndicesFromPap(string hash, bool persistToConfig = true)
{
if (string.IsNullOrWhiteSpace(hash))
return null;
if (_configService.Current.BonesDictionary.TryGetValue(hash, out var cached) && cached is not null)
return cached;
var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash);
if (cacheEntity == null || string.IsNullOrEmpty(cacheEntity.ResolvedFilepath) || !File.Exists(cacheEntity.ResolvedFilepath))
return null;
using var fs = File.Open(cacheEntity.ResolvedFilepath, FileMode.Open, FileAccess.Read, FileShare.Read);
using var reader = new BinaryReader(fs);
// PAP header (mostly from vfxeditor)
try
{
_ = reader.ReadInt32(); // ignore
_ = reader.ReadInt32(); // ignore
var numAnimations = reader.ReadInt16(); // num animations
var modelId = reader.ReadInt16(); // modelid
if (numAnimations < 0 || numAnimations > 1000)
{
_logger.LogWarning("PAP file {hash} has invalid animation count {count}, skipping", hash, numAnimations);
return null;
}
var type = reader.ReadByte(); // type
if (type != 0)
return null; // not human
_ = reader.ReadByte(); // variant
_ = reader.ReadInt32(); // ignore
var havokPosition = reader.ReadInt32();
var footerPosition = reader.ReadInt32();
if (havokPosition <= 0 || footerPosition <= havokPosition ||
footerPosition > fs.Length || havokPosition >= fs.Length)
{
_logger.LogWarning("PAP file {hash} has invalid offsets (havok={havok}, footer={footer}, length={length})",
hash, havokPosition, footerPosition, fs.Length);
return null;
}
var havokDataSizeLong = (long)footerPosition - havokPosition;
if (havokDataSizeLong <= 8 || havokDataSizeLong > int.MaxValue)
{
_logger.LogWarning("PAP file {hash} has invalid Havok data size {size}", hash, havokDataSizeLong);
return null;
}
var havokDataSize = (int)havokDataSizeLong;
reader.BaseStream.Position = havokPosition;
var havokData = new byte[havokDataSize];
var bytesRead = reader.Read(havokData, 0, havokDataSize);
if (bytesRead != havokDataSize)
{
_logger.LogWarning("PAP file {hash}: Expected to read {expected} bytes but got {actual}",
hash, havokDataSize, bytesRead);
return null;
}
if (havokData.Length < 8)
return null;
var tempSets = new Dictionary<string, HashSet<ushort>>(StringComparer.OrdinalIgnoreCase);
var tempFileName = $"lightless_pap_{Guid.NewGuid():N}_{hash.Substring(0, Math.Min(8, hash.Length))}.hkx";
var tempHavokDataPath = Path.Combine(Path.GetTempPath(), tempFileName);
IntPtr tempHavokDataPathAnsi = IntPtr.Zero;
try
{
var tempDir = Path.GetDirectoryName(tempHavokDataPath);
if (!Directory.Exists(tempDir))
{
_logger.LogWarning("Temp directory {dir} doesn't exist", tempDir);
return null;
}
File.WriteAllBytes(tempHavokDataPath, havokData);
if (!File.Exists(tempHavokDataPath))
{
_logger.LogWarning("Temporary havok file was not created at {path}", tempHavokDataPath);
return null;
}
var writtenFileInfo = new FileInfo(tempHavokDataPath);
if (writtenFileInfo.Length != havokData.Length)
{
_logger.LogWarning("Written temp file size mismatch: expected {expected}, got {actual}",
havokData.Length, writtenFileInfo.Length);
File.Delete(tempHavokDataPath);
return null;
}
tempHavokDataPathAnsi = Marshal.StringToHGlobalAnsi(tempHavokDataPath);
var loadoptions = stackalloc hkSerializeUtil.LoadOptions[1];
loadoptions->TypeInfoRegistry = hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry();
loadoptions->ClassNameRegistry = hkBuiltinTypeRegistry.Instance()->GetClassNameRegistry();
loadoptions->Flags = new hkFlags<hkSerializeUtil.LoadOptionBits, int>
{
Storage = (int)hkSerializeUtil.LoadOptionBits.Default
};
hkResource* resource = null;
try
{
resource = hkSerializeUtil.LoadFromFile((byte*)tempHavokDataPathAnsi, null, loadoptions);
}
catch (SEHException ex)
{
_logger.LogError(ex, "SEH exception loading Havok file from {path} (hash={hash}). Native error code: 0x{code:X}",
tempHavokDataPath, hash, ex.ErrorCode);
return null;
}
if (resource == null)
{
_logger.LogDebug("Havok resource was null after loading from {path} (hash={hash})", tempHavokDataPath, hash);
return null;
}
if ((nint)resource == nint.Zero || !IsValidPointer((IntPtr)resource))
{
_logger.LogDebug("Havok resource pointer is invalid (hash={hash})", hash);
return null;
}
var rootLevelName = @"hkRootLevelContainer"u8;
fixed (byte* n1 = rootLevelName)
{
var container = (hkRootLevelContainer*)resource->GetContentsPointer(n1, hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry());
if (container == null)
{
_logger.LogDebug("hkRootLevelContainer is null (hash={hash})", hash);
return null;
}
if ((nint)container == nint.Zero || !IsValidPointer((IntPtr)container))
{
_logger.LogDebug("hkRootLevelContainer pointer is invalid (hash={hash})", hash);
return null;
}
var animationName = @"hkaAnimationContainer"u8;
fixed (byte* n2 = animationName)
{
var animContainer = (hkaAnimationContainer*)container->findObjectByName(n2, null);
if (animContainer == null)
{
_logger.LogDebug("hkaAnimationContainer is null (hash={hash})", hash);
return null;
}
if ((nint)animContainer == nint.Zero || !IsValidPointer((IntPtr)animContainer))
{
_logger.LogDebug("hkaAnimationContainer pointer is invalid (hash={hash})", hash);
return null;
}
if (animContainer->Bindings.Length < 0 || animContainer->Bindings.Length > 10000)
{
_logger.LogDebug("Invalid bindings count {count} (hash={hash})", animContainer->Bindings.Length, hash);
return null;
}
for (int i = 0; i < animContainer->Bindings.Length; i++)
{
var binding = animContainer->Bindings[i].ptr;
if (binding == null)
continue;
if ((nint)binding == nint.Zero || !IsValidPointer((IntPtr)binding))
{
_logger.LogDebug("Skipping invalid binding at index {index} (hash={hash})", i, hash);
continue;
}
var rawSkel = binding->OriginalSkeletonName.String;
var skeletonKey = CanonicalizeSkeletonKey(rawSkel);
if (string.IsNullOrEmpty(skeletonKey))
continue;
var boneTransform = binding->TransformTrackToBoneIndices;
if (boneTransform.Length <= 0 || boneTransform.Length > 10000)
{
_logger.LogDebug("Invalid bone transform length {length} for skeleton {skel} (hash={hash})",
boneTransform.Length, skeletonKey, hash);
continue;
}
if (!tempSets.TryGetValue(skeletonKey, out var set))
{
set = [];
tempSets[skeletonKey] = set;
}
for (int boneIdx = 0; boneIdx < boneTransform.Length; boneIdx++)
{
var v = boneTransform[boneIdx];
if (v < 0 || v > ushort.MaxValue)
continue;
set.Add((ushort)v);
}
}
}
}
}
catch (SEHException ex)
{
_logger.LogError(ex, "SEH exception processing PAP file {hash} from {path}. Error code: 0x{code:X}",
hash, tempHavokDataPath, ex.ErrorCode);
return null;
}
catch (Exception ex)
{
_logger.LogError(ex, "Managed exception loading havok file {hash} from {path}", hash, tempHavokDataPath);
return null;
}
finally
{
if (tempHavokDataPathAnsi != IntPtr.Zero)
Marshal.FreeHGlobal(tempHavokDataPathAnsi);
int retryCount = 3;
while (retryCount > 0 && File.Exists(tempHavokDataPath))
{
try
{
File.Delete(tempHavokDataPath);
break;
}
catch (IOException ex)
{
retryCount--;
if (retryCount == 0)
{
_logger.LogDebug(ex, "Failed to delete temporary havok file after retries: {path}", tempHavokDataPath);
}
else
{
Thread.Sleep(50);
}
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Unexpected error deleting temporary havok file: {path}", tempHavokDataPath);
break;
}
}
}
if (tempSets.Count == 0)
{
_logger.LogDebug("No bone sets found in PAP file (hash={hash})", hash);
return null;
}
var output = new Dictionary<string, List<ushort>>(tempSets.Count, StringComparer.OrdinalIgnoreCase);
foreach (var (key, set) in tempSets)
{
if (set.Count == 0) continue;
var list = set.ToList();
list.Sort();
output[key] = list;
}
if (output.Count == 0)
return null;
_configService.Current.BonesDictionary[hash] = output;
if (persistToConfig)
_configService.Save();
return output;
}
catch (Exception ex)
{
_logger.LogError(ex, "Outer exception reading PAP file (hash={hash})", hash);
return null;
}
}
private static bool IsValidPointer(IntPtr ptr)
{
if (ptr == IntPtr.Zero)
return false;
try
{
_ = Marshal.ReadByte(ptr);
return true;
}
catch
{
return false;
}
}
public static string CanonicalizeSkeletonKey(string? raw)
{
if (string.IsNullOrWhiteSpace(raw))
return string.Empty;
var s = raw.Replace('\\', '/').Trim();
var underscore = s.LastIndexOf('_');
if (underscore > 0 && underscore + 1 < s.Length && char.IsDigit(s[underscore + 1]))
s = s[..underscore];
if (s.StartsWith("skeleton", StringComparison.OrdinalIgnoreCase))
return "skeleton";
var m = _bucketPathRegex.Match(s);
if (m.Success)
return m.Groups["bucket"].Value.ToLowerInvariant();
m = _bucketSklRegex.Match(s);
if (m.Success)
return m.Groups["bucket"].Value.ToLowerInvariant();
m = _bucketLooseRegex.Match(s);
if (m.Success)
return m.Groups["bucket"].Value.ToLowerInvariant();
return string.Empty;
}
public static bool ContainsIndexCompat(
HashSet<ushort> available,
ushort idx,
bool papLikelyOneBased,
bool allowOneBasedShift,
bool allowNeighborTolerance)
{
Span<ushort> candidates = stackalloc ushort[2];
int count = 0;
candidates[count++] = idx;
if (allowOneBasedShift && papLikelyOneBased && idx > 0)
candidates[count++] = (ushort)(idx - 1);
for (int i = 0; i < count; i++)
{
var c = candidates[i];
if (available.Contains(c))
return true;
if (allowNeighborTolerance)
{
if (c > 0 && available.Contains((ushort)(c - 1)))
return true;
if (c < ushort.MaxValue && available.Contains((ushort)(c + 1)))
return true;
}
}
return false;
}
public static bool IsPapCompatible(
IReadOnlyDictionary<string, HashSet<ushort>> localBoneSets,
IReadOnlyDictionary<string, List<ushort>> papBoneIndices,
AnimationValidationMode mode,
bool allowOneBasedShift,
bool allowNeighborTolerance,
out string reason)
{
reason = string.Empty;
if (mode == AnimationValidationMode.Unsafe)
return true;
var papBuckets = papBoneIndices.Keys
.Select(CanonicalizeSkeletonKey)
.Where(k => !string.IsNullOrEmpty(k))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
if (papBuckets.Count == 0)
{
reason = "No skeleton bucket bindings found in the PAP";
return false;
}
if (mode == AnimationValidationMode.Safe)
{
if (papBuckets.Any(b => localBoneSets.ContainsKey(b)))
return true;
reason = $"No matching skeleton bucket between PAP [{string.Join(", ", papBuckets)}] and local [{string.Join(", ", localBoneSets.Keys.Order())}].";
return false;
}
foreach (var bucket in papBuckets)
{
if (!localBoneSets.TryGetValue(bucket, out var available))
{
reason = $"Missing skeleton bucket '{bucket}' on local actor.";
return false;
}
var indices = papBoneIndices
.Where(kvp => string.Equals(CanonicalizeSkeletonKey(kvp.Key), bucket, StringComparison.OrdinalIgnoreCase))
.SelectMany(kvp => kvp.Value ?? Enumerable.Empty<ushort>())
.Distinct()
.ToList();
if (indices.Count == 0)
continue;
bool has0 = false, has1 = false;
ushort min = ushort.MaxValue;
foreach (var v in indices)
{
if (v == 0) has0 = true;
if (v == 1) has1 = true;
if (v < min) min = v;
}
bool papLikelyOneBased = allowOneBasedShift && (min == 1) && has1 && !has0;
foreach (var idx in indices)
{
if (!ContainsIndexCompat(available, idx, papLikelyOneBased, allowOneBasedShift, allowNeighborTolerance))
{
reason = $"No compatible local skeleton for PAP '{bucket}': missing bone index {idx}.";
return false;
}
}
}
return true;
}
public void DumpLocalSkeletonIndices(GameObjectHandler handler, string? filter = null)
{
var skels = GetSkeletonBoneIndices(handler);
if (skels == null)
{
_logger.LogTrace("DumpLocalSkeletonIndices: local skeleton indices are null or not found");
return;
}
var keys = skels.Keys
.Order(StringComparer.OrdinalIgnoreCase)
.ToArray();
_logger.LogTrace("Local skeleton indices found ({count}): {keys}",
keys.Length,
string.Join(", ", keys));
if (!string.IsNullOrWhiteSpace(filter))
{
var hits = keys.Where(k =>
k.Equals(filter, StringComparison.OrdinalIgnoreCase) ||
k.StartsWith(filter + "_", StringComparison.OrdinalIgnoreCase) ||
filter.StartsWith(k + "_", StringComparison.OrdinalIgnoreCase) ||
k.Contains(filter, StringComparison.OrdinalIgnoreCase))
.ToArray();
_logger.LogTrace("Matches found for '{filter}': {hits}",
filter,
hits.Length == 0 ? "<none>" : string.Join(", ", hits));
}
}
public async Task<long> GetTrianglesByHash(string hash)
{
if (_configService.Current.TriangleDictionary.TryGetValue(hash, out var cachedTris) && cachedTris > 0)
return cachedTris;
if (_failedCalculatedTris.Contains(hash, StringComparer.Ordinal))
return 0;
var path = _fileCacheManager.GetFileCacheByHash(hash);
if (path == null || !path.ResolvedFilepath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase))
return 0;
return CalculateTrianglesFromPath(hash, path.ResolvedFilepath, _configService.Current.TriangleDictionary, _failedCalculatedTris);
}
public async Task<long> GetEffectiveTrianglesByHash(string hash, string filePath)
{
if (_configService.Current.EffectiveTriangleDictionary.TryGetValue(hash, out var cachedTris) && cachedTris > 0)
return cachedTris;
if (_failedCalculatedEffectiveTris.Contains(hash, StringComparer.Ordinal))
return 0;
if (string.IsNullOrEmpty(filePath)
|| !filePath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase)
|| !File.Exists(filePath))
{
return 0;
}
return CalculateTrianglesFromPath(hash, filePath, _configService.Current.EffectiveTriangleDictionary, _failedCalculatedEffectiveTris);
}
private long CalculateTrianglesFromPath(
string hash,
string filePath,
ConcurrentDictionary<string, long> cache,
List<string> failedList)
{
try
{
_logger.LogDebug("Detected Model File {path}, calculating Tris", filePath);
var file = new MdlFile(filePath);
if (file.LodCount <= 0)
{
failedList.Add(hash);
cache[hash] = 0;
_configService.Save();
return 0;
}
long tris = 0;
foreach (var lod in file.Lods)
{
try
{
var meshIdx = lod.MeshIndex;
var meshCnt = lod.MeshCount;
tris = file.Meshes.Skip(meshIdx).Take(meshCnt).Sum(p => p.IndexCount) / 3;
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Could not load lod mesh {mesh} from path {path}", lod.MeshIndex, filePath);
continue;
}
if (tris > 0)
{
_logger.LogDebug("TriAnalysis: {filePath} => {tris} triangles", filePath, tris);
cache[hash] = tris;
_configService.Save();
break;
}
}
return tris;
}
catch (Exception e)
{
failedList.Add(hash);
cache[hash] = 0;
_configService.Save();
_logger.LogWarning(e, "Could not parse file {file}", filePath);
return 0;
}
}
// Regexes for canonicalizing skeleton keys
private static readonly Regex _bucketPathRegex =
BucketRegex();
private static readonly Regex _bucketSklRegex =
SklRegex();
private static readonly Regex _bucketLooseRegex =
LooseBucketRegex();
[GeneratedRegex(@"(?i)(?:^|/)(?<bucket>c\d{4})(?:/|$)", RegexOptions.Compiled, "en-NL")]
private static partial Regex BucketRegex();
[GeneratedRegex(@"(?i)\bskl_(?<bucket>c\d{4})[a-z]\d{4}\b", RegexOptions.Compiled, "en-NL")]
private static partial Regex SklRegex();
[GeneratedRegex(@"(?i)(?<![a-z0-9])(?<bucket>c\d{4})(?!\d)", RegexOptions.Compiled, "en-NL")]
private static partial Regex LooseBucketRegex();
}