Compare commits

...

19 Commits

Author SHA1 Message Date
defnotken
4664071eb3 Merge branch 'cake-attempts-2.0.3' of https://git.lightless-sync.org/Lightless-Sync/LightlessClient into cake-attempts-2.0.3 2026-01-03 17:28:25 -06:00
defnotken
11099c05ff Throwing analysis in background task so dalamud can breath 2026-01-03 17:28:05 -06:00
cake
6af61451dc Fixed naming of setting 2026-01-03 16:32:39 +01:00
cake
02d091eefa Added more loose matching options, fixed some race issues 2026-01-03 15:59:10 +01:00
cake
e41a7149c5 Refactored many parts, added settings for detection 2026-01-03 14:58:54 +01:00
cake
e16ddb0a1d change log to trace 2026-01-02 19:29:50 +01:00
cake
ba26edc33c Merge branch 'cake-attempts-2.0.3' of https://git.lightless-sync.org/Lightless-Sync/LightlessClient into cake-attempts-2.0.3 2026-01-02 18:31:14 +01:00
cake
14c4c1d669 Added caching in the playerdata factory, refactored 2026-01-02 18:30:37 +01:00
defnotken
e8c157d8ac Creating temp havok file to not crash the analyzer 2026-01-02 10:23:32 -06:00
defnotken
2af0b5774b Merge branch 'cake-attempts-2.0.3' of https://git.lightless-sync.org/Lightless-Sync/LightlessClient into cake-attempts-2.0.3 2026-01-02 09:29:22 -06:00
defnotken
bb365442cf Maybe? 2026-01-02 09:29:04 -06:00
cake
277d368f83 Adjustments in PAP handling. 2026-01-02 07:43:30 +01:00
defnotken
3487891185 Testing asyncing the transient task 2026-01-01 22:23:46 -06:00
defnotken
96f8d33cde Fixing dls ui and fixed cancel cache validation breaking menu. 2026-01-01 21:57:48 -06:00
cake
a033d4d4d8 Merge conflict 2026-01-02 04:01:44 +01:00
defnotken
7d2a914c84 Queue File compacting to let workers download as priority, Offload decompression task 2026-01-01 20:57:37 -06:00
cake
d6fe09ba8e Testing PAP handling changes. 2026-01-02 03:56:59 +01:00
7e61954541 Location Sharing 2.0 (#125)
Need: Lightless-Sync/LightlessServer#49
Authored-by: Tsubasahane <wozaiha@gmail.com>
Reviewed-on: #125
Reviewed-by: cake <cake@noreply.git.lightless-sync.org>
Co-authored-by: Tsubasa <tsubasa@noreply.git.lightless-sync.org>
Co-committed-by: Tsubasa <tsubasa@noreply.git.lightless-sync.org>
2025-12-31 17:31:31 +00:00
bbb3375661 2.0.3 staaato 2025-12-31 02:44:31 +00:00
26 changed files with 1790 additions and 482 deletions

View File

@@ -27,6 +27,7 @@ public sealed class FileCacheManager : IHostedService
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, FileCacheEntity> _fileCachesByPrefixedPath = new(StringComparer.OrdinalIgnoreCase);
private readonly SemaphoreSlim _getCachesByPathsSemaphore = new(1, 1);
private readonly SemaphoreSlim _evictSemaphore = new(1, 1);
private readonly Lock _fileWriteLock = new();
private readonly IpcManager _ipcManager;
private readonly ILogger<FileCacheManager> _logger;
@@ -226,13 +227,23 @@ public sealed class FileCacheManager : IHostedService
var compressed = LZ4Wrapper.WrapHC(raw, 0, raw.Length);
var tmpPath = compressedPath + ".tmp";
await File.WriteAllBytesAsync(tmpPath, compressed, token).ConfigureAwait(false);
File.Move(tmpPath, compressedPath, overwrite: true);
try
{
await File.WriteAllBytesAsync(tmpPath, compressed, token).ConfigureAwait(false);
File.Move(tmpPath, compressedPath, overwrite: true);
}
finally
{
try { if (File.Exists(tmpPath)) File.Delete(tmpPath); } catch { /* ignore */ }
}
var compressedSize = compressed.LongLength;
var compressedSize = new FileInfo(compressedPath).Length;
SetSizeInfo(hash, originalSize, compressedSize);
UpdateEntitiesSizes(hash, originalSize, compressedSize);
var maxBytes = GiBToBytes(_configService.Current.MaxLocalCacheInGiB);
await EnforceCacheLimitAsync(maxBytes, token).ConfigureAwait(false);
return compressed;
}
finally
@@ -877,6 +888,83 @@ public sealed class FileCacheManager : IHostedService
}, token).ConfigureAwait(false);
}
private async Task EnforceCacheLimitAsync(long maxBytes, CancellationToken token)
{
if (string.IsNullOrWhiteSpace(CacheFolder) || maxBytes <= 0) return;
await _evictSemaphore.WaitAsync(token).ConfigureAwait(false);
try
{
Directory.CreateDirectory(CacheFolder);
foreach (var tmp in Directory.EnumerateFiles(CacheFolder, "*" + _compressedCacheExtension + ".tmp"))
{
try { File.Delete(tmp); } catch { /* ignore */ }
}
var files = Directory.EnumerateFiles(CacheFolder, "*" + _compressedCacheExtension, SearchOption.TopDirectoryOnly)
.Select(p => new FileInfo(p))
.Where(fi => fi.Exists)
.OrderBy(fi => fi.LastWriteTimeUtc)
.ToList();
long total = files.Sum(f => f.Length);
if (total <= maxBytes) return;
foreach (var fi in files)
{
token.ThrowIfCancellationRequested();
if (total <= maxBytes) break;
var hash = Path.GetFileNameWithoutExtension(fi.Name);
try
{
var len = fi.Length;
fi.Delete();
total -= len;
_sizeCache.TryRemove(hash, out _);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to evict cache file {file}", fi.FullName);
}
}
}
finally
{
_evictSemaphore.Release();
}
}
private static long GiBToBytes(double gib)
{
if (double.IsNaN(gib) || double.IsInfinity(gib) || gib <= 0)
return 0;
var bytes = gib * 1024d * 1024d * 1024d;
if (bytes >= long.MaxValue) return long.MaxValue;
return (long)Math.Round(bytes, MidpointRounding.AwayFromZero);
}
private void CleanupOrphanCompressedCache()
{
if (string.IsNullOrWhiteSpace(CacheFolder) || !Directory.Exists(CacheFolder))
return;
foreach (var path in Directory.EnumerateFiles(CacheFolder, "*" + _compressedCacheExtension))
{
var hash = Path.GetFileNameWithoutExtension(path);
if (!_fileCaches.ContainsKey(hash))
{
try { File.Delete(path); }
catch (Exception ex) { _logger.LogWarning(ex, "Failed deleting orphan {file}", path); }
}
}
}
public async Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Starting FileCacheManager");
@@ -1060,6 +1148,8 @@ public sealed class FileCacheManager : IHostedService
{
await WriteOutFullCsvAsync(cancellationToken).ConfigureAwait(false);
}
CleanupOrphanCompressedCache();
}
_logger.LogInformation("Started FileCacheManager");

View File

@@ -297,7 +297,7 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
private void DalamudUtil_FrameworkUpdate()
{
RefreshPlayerRelatedAddressMap();
_ = Task.Run(() => RefreshPlayerRelatedAddressMap());
lock (_cacheAdditionLock)
{
@@ -306,20 +306,64 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
if (_lastClassJobId != _dalamudUtil.ClassJobId)
{
_lastClassJobId = _dalamudUtil.ClassJobId;
if (SemiTransientResources.TryGetValue(ObjectKind.Pet, out HashSet<string>? value))
{
value?.Clear();
}
PlayerConfig.JobSpecificCache.TryGetValue(_dalamudUtil.ClassJobId, out var jobSpecificData);
SemiTransientResources[ObjectKind.Player] = PlayerConfig.GlobalPersistentCache.Concat(jobSpecificData ?? []).ToHashSet(StringComparer.OrdinalIgnoreCase);
PlayerConfig.JobSpecificPetCache.TryGetValue(_dalamudUtil.ClassJobId, out var petSpecificData);
SemiTransientResources[ObjectKind.Pet] = new HashSet<string>(
petSpecificData ?? [],
StringComparer.OrdinalIgnoreCase);
UpdateClassJobCache();
}
CleanupAbsentObjects();
}
private void RefreshPlayerRelatedAddressMap()
{
var tempMap = new ConcurrentDictionary<nint, GameObjectHandler>();
var updatedFrameAddresses = new ConcurrentDictionary<nint, ObjectKind>();
lock (_playerRelatedLock)
{
foreach (var handler in _playerRelatedPointers)
{
var address = (nint)handler.Address;
if (address != nint.Zero)
{
tempMap[address] = handler;
updatedFrameAddresses[address] = handler.ObjectKind;
}
}
}
_playerRelatedByAddress.Clear();
foreach (var kvp in tempMap)
{
_playerRelatedByAddress[kvp.Key] = kvp.Value;
}
_cachedFrameAddresses.Clear();
foreach (var kvp in updatedFrameAddresses)
{
_cachedFrameAddresses[kvp.Key] = kvp.Value;
}
}
private void UpdateClassJobCache()
{
_lastClassJobId = _dalamudUtil.ClassJobId;
if (SemiTransientResources.TryGetValue(ObjectKind.Pet, out HashSet<string>? value))
{
value?.Clear();
}
PlayerConfig.JobSpecificCache.TryGetValue(_dalamudUtil.ClassJobId, out var jobSpecificData);
SemiTransientResources[ObjectKind.Player] = PlayerConfig.GlobalPersistentCache
.Concat(jobSpecificData ?? [])
.ToHashSet(StringComparer.OrdinalIgnoreCase);
PlayerConfig.JobSpecificPetCache.TryGetValue(_dalamudUtil.ClassJobId, out var petSpecificData);
SemiTransientResources[ObjectKind.Pet] = new HashSet<string>(
petSpecificData ?? [],
StringComparer.OrdinalIgnoreCase);
}
private void CleanupAbsentObjects()
{
foreach (var kind in Enum.GetValues(typeof(ObjectKind)).Cast<ObjectKind>())
{
if (!_cachedFrameAddresses.Any(k => k.Value == kind) && TransientResources.Remove(kind, out _))
@@ -349,26 +393,6 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
_semiTransientResources = null;
}
private void RefreshPlayerRelatedAddressMap()
{
_playerRelatedByAddress.Clear();
var updatedFrameAddresses = new ConcurrentDictionary<nint, ObjectKind>();
lock (_playerRelatedLock)
{
foreach (var handler in _playerRelatedPointers)
{
var address = (nint)handler.Address;
if (address != nint.Zero)
{
_playerRelatedByAddress[address] = handler;
updatedFrameAddresses[address] = handler.ObjectKind;
}
}
}
_cachedFrameAddresses = updatedFrameAddresses;
}
private void HandleActorTracked(ActorObjectService.ActorDescriptor descriptor)
{
if (descriptor.IsInGpose)

View File

@@ -4,6 +4,7 @@ using LightlessSync.LightlessConfiguration.Models;
using LightlessSync.UI;
using LightlessSync.UI.Models;
using Microsoft.Extensions.Logging;
using LightlessSync.PlayerData.Factories;
namespace LightlessSync.LightlessConfiguration.Configurations;
@@ -156,4 +157,8 @@ public class LightlessConfig : ILightlessConfiguration
public string? SelectedFinderSyncshell { get; set; } = null;
public string LastSeenVersion { get; set; } = string.Empty;
public HashSet<Guid> OrphanableTempCollections { get; set; } = [];
public AnimationValidationMode AnimationValidationMode { get; set; } = AnimationValidationMode.Safe;
public bool AnimationAllowOneBasedShift { get; set; } = true;
public bool AnimationAllowNeighborIndexTolerance { get; set; } = false;
}

View File

@@ -3,7 +3,7 @@
<PropertyGroup>
<Authors></Authors>
<Company></Company>
<Version>2.0.2</Version>
<Version>2.0.3</Version>
<Description></Description>
<Copyright></Copyright>
<PackageProjectUrl>https://github.com/Light-Public-Syncshells/LightlessClient</PackageProjectUrl>
@@ -37,6 +37,7 @@
</PackageReference>
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="10.0.1" />
<PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="10.0.1" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.1" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="10.0.1" />
<PackageReference Include="Glamourer.Api" Version="2.8.0" />
<PackageReference Include="NReco.Logging.File" Version="1.3.1" />

View File

@@ -0,0 +1,9 @@
namespace LightlessSync.PlayerData.Factories
{
public enum AnimationValidationMode
{
Unsafe = 0,
Safe = 1,
Safest = 2,
}
}

View File

@@ -2,12 +2,15 @@
using LightlessSync.API.Data.Enum;
using LightlessSync.FileCache;
using LightlessSync.Interop.Ipc;
using LightlessSync.LightlessConfiguration;
using LightlessSync.LightlessConfiguration.Models;
using LightlessSync.PlayerData.Data;
using LightlessSync.PlayerData.Handlers;
using LightlessSync.Services;
using LightlessSync.Services.Mediator;
using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
using System.Diagnostics;
namespace LightlessSync.PlayerData.Factories;
@@ -18,13 +21,34 @@ public class PlayerDataFactory
private readonly IpcManager _ipcManager;
private readonly ILogger<PlayerDataFactory> _logger;
private readonly PerformanceCollectorService _performanceCollector;
private readonly LightlessConfigService _configService;
private readonly XivDataAnalyzer _modelAnalyzer;
private readonly LightlessMediator _lightlessMediator;
private readonly TransientResourceManager _transientResourceManager;
private static readonly SemaphoreSlim _papParseLimiter = new(1, 1);
public PlayerDataFactory(ILogger<PlayerDataFactory> logger, DalamudUtilService dalamudUtil, IpcManager ipcManager,
TransientResourceManager transientResourceManager, FileCacheManager fileReplacementFactory,
PerformanceCollectorService performanceCollector, XivDataAnalyzer modelAnalyzer, LightlessMediator lightlessMediator)
// Transient resolved entries threshold
private const int _maxTransientResolvedEntries = 1000;
// Character build caches
private readonly ConcurrentDictionary<nint, Task<CharacterDataFragment>> _characterBuildInflight = new();
private readonly ConcurrentDictionary<nint, CacheEntry> _characterBuildCache = new();
// Time out thresholds
private static readonly TimeSpan _characterCacheTtl = TimeSpan.FromMilliseconds(750);
private static readonly TimeSpan _softReturnIfBusyAfter = TimeSpan.FromMilliseconds(250);
private static readonly TimeSpan _hardBuildTimeout = TimeSpan.FromSeconds(30);
public PlayerDataFactory(
ILogger<PlayerDataFactory> logger,
DalamudUtilService dalamudUtil,
IpcManager ipcManager,
TransientResourceManager transientResourceManager,
FileCacheManager fileReplacementFactory,
PerformanceCollectorService performanceCollector,
XivDataAnalyzer modelAnalyzer,
LightlessMediator lightlessMediator,
LightlessConfigService configService)
{
_logger = logger;
_dalamudUtil = dalamudUtil;
@@ -34,15 +58,15 @@ public class PlayerDataFactory
_performanceCollector = performanceCollector;
_modelAnalyzer = modelAnalyzer;
_lightlessMediator = lightlessMediator;
_configService = configService;
_logger.LogTrace("Creating {this}", nameof(PlayerDataFactory));
}
private sealed record CacheEntry(CharacterDataFragment Fragment, DateTime CreatedUtc);
public async Task<CharacterDataFragment?> BuildCharacterData(GameObjectHandler playerRelatedObject, CancellationToken token)
{
if (!_ipcManager.Initialized)
{
throw new InvalidOperationException("Penumbra or Glamourer is not connected");
}
if (playerRelatedObject == null) return null;
@@ -67,16 +91,17 @@ public class PlayerDataFactory
if (pointerIsZero)
{
_logger.LogTrace("Pointer was zero for {objectKind}", playerRelatedObject.ObjectKind);
_logger.LogTrace("Pointer was zero for {objectKind}; couldn't build character", playerRelatedObject.ObjectKind);
return null;
}
try
{
return await _performanceCollector.LogPerformance(this, $"CreateCharacterData>{playerRelatedObject.ObjectKind}", async () =>
{
return await CreateCharacterData(playerRelatedObject, token).ConfigureAwait(false);
}).ConfigureAwait(true);
return await _performanceCollector.LogPerformance(
this,
$"CreateCharacterData>{playerRelatedObject.ObjectKind}",
async () => await CreateCharacterData(playerRelatedObject, token).ConfigureAwait(false)
).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
@@ -92,17 +117,14 @@ public class PlayerDataFactory
}
private async Task<bool> CheckForNullDrawObject(IntPtr playerPointer)
{
return await _dalamudUtil.RunOnFrameworkThread(() => CheckForNullDrawObjectUnsafe(playerPointer)).ConfigureAwait(false);
}
=> await _dalamudUtil.RunOnFrameworkThread(() => CheckForNullDrawObjectUnsafe(playerPointer)).ConfigureAwait(false);
private unsafe bool CheckForNullDrawObjectUnsafe(IntPtr playerPointer)
private unsafe static bool CheckForNullDrawObjectUnsafe(IntPtr playerPointer)
{
if (playerPointer == IntPtr.Zero)
return true;
var character = (Character*)playerPointer;
if (character == null)
return true;
@@ -113,93 +135,167 @@ public class PlayerDataFactory
return gameObject->DrawObject == null;
}
private async Task<CharacterDataFragment> CreateCharacterData(GameObjectHandler playerRelatedObject, CancellationToken ct)
private static bool IsCacheFresh(CacheEntry entry)
=> (DateTime.UtcNow - entry.CreatedUtc) <= _characterCacheTtl;
private Task<CharacterDataFragment> CreateCharacterData(GameObjectHandler playerRelatedObject, CancellationToken ct)
=> CreateCharacterDataCoalesced(playerRelatedObject, ct);
private async Task<CharacterDataFragment> CreateCharacterDataCoalesced(GameObjectHandler obj, CancellationToken ct)
{
var key = obj.Address;
if (_characterBuildCache.TryGetValue(key, out var cached) && IsCacheFresh(cached) && !_characterBuildInflight.ContainsKey(key))
return cached.Fragment;
var buildTask = _characterBuildInflight.GetOrAdd(key, _ => BuildAndCacheAsync(obj, key));
if (_characterBuildCache.TryGetValue(key, out cached))
{
var completed = await Task.WhenAny(buildTask, Task.Delay(_softReturnIfBusyAfter, ct)).ConfigureAwait(false);
if (completed != buildTask && (DateTime.UtcNow - cached.CreatedUtc) <= TimeSpan.FromSeconds(5))
{
return cached.Fragment;
}
}
return await WithCancellation(buildTask, ct).ConfigureAwait(false);
}
private async Task<CharacterDataFragment> BuildAndCacheAsync(GameObjectHandler obj, nint key)
{
try
{
using var cts = new CancellationTokenSource(_hardBuildTimeout);
var fragment = await CreateCharacterDataInternal(obj, cts.Token).ConfigureAwait(false);
_characterBuildCache[key] = new CacheEntry(fragment, DateTime.UtcNow);
PruneCharacterCacheIfNeeded();
return fragment;
}
finally
{
_characterBuildInflight.TryRemove(key, out _);
}
}
private void PruneCharacterCacheIfNeeded()
{
if (_characterBuildCache.Count < 2048) return;
var cutoff = DateTime.UtcNow - TimeSpan.FromSeconds(10);
foreach (var kv in _characterBuildCache)
{
if (kv.Value.CreatedUtc < cutoff)
_characterBuildCache.TryRemove(kv.Key, out _);
}
}
private static async Task<T> WithCancellation<T>(Task<T> task, CancellationToken ct)
=> await task.WaitAsync(ct).ConfigureAwait(false);
private async Task<CharacterDataFragment> CreateCharacterDataInternal(GameObjectHandler playerRelatedObject, CancellationToken ct)
{
var objectKind = playerRelatedObject.ObjectKind;
CharacterDataFragment fragment = objectKind == ObjectKind.Player ? new CharacterDataFragmentPlayer() : new();
_logger.LogDebug("Building character data for {obj}", playerRelatedObject);
var logDebug = _logger.IsEnabled(LogLevel.Debug);
var sw = Stopwatch.StartNew();
// wait until chara is not drawing and present so nothing spontaneously explodes
await _dalamudUtil.WaitWhileCharacterIsDrawing(_logger, playerRelatedObject, Guid.NewGuid(), 30000, ct: ct).ConfigureAwait(false);
int totalWaitTime = 10000;
while (!await _dalamudUtil.IsObjectPresentAsync(await _dalamudUtil.CreateGameObjectAsync(playerRelatedObject.Address).ConfigureAwait(false)).ConfigureAwait(false) && totalWaitTime > 0)
_logger.LogDebug("Building character data for {obj}", playerRelatedObject);
await EnsureObjectPresentAsync(playerRelatedObject, ct).ConfigureAwait(false);
ct.ThrowIfCancellationRequested();
var waitRecordingTask = _transientResourceManager.WaitForRecording(ct);
await _dalamudUtil.WaitWhileCharacterIsDrawing(_logger, playerRelatedObject, Guid.NewGuid(), 30000, ct: ct)
.ConfigureAwait(false);
ct.ThrowIfCancellationRequested();
if (await CheckForNullDrawObject(playerRelatedObject.Address).ConfigureAwait(false))
throw new InvalidOperationException("DrawObject became null during build (actor despawned)");
Task<string> getGlamourerData = _ipcManager.Glamourer.GetCharacterCustomizationAsync(playerRelatedObject.Address);
Task<string?> getCustomizeData = _ipcManager.CustomizePlus.GetScaleAsync(playerRelatedObject.Address);
Task<string?>? getMoodlesData = null;
Task<string>? getHeelsOffset = null;
Task<string>? getHonorificTitle = null;
if (objectKind == ObjectKind.Player)
{
_logger.LogTrace("Character is null but it shouldn't be, waiting");
await Task.Delay(50, ct).ConfigureAwait(false);
totalWaitTime -= 50;
getHeelsOffset = _ipcManager.Heels.GetOffsetAsync();
getHonorificTitle = _ipcManager.Honorific.GetTitle();
getMoodlesData = _ipcManager.Moodles.GetStatusAsync(playerRelatedObject.Address);
}
var resolvedPaths = await _ipcManager.Penumbra.GetCharacterData(_logger, playerRelatedObject).ConfigureAwait(false) ?? throw new InvalidOperationException("Penumbra returned null data; couldn't proceed with character");
ct.ThrowIfCancellationRequested();
DateTime start = DateTime.UtcNow;
var staticBuildTask = Task.Run(() => BuildStaticReplacements(resolvedPaths), ct);
// penumbra call, it's currently broken
Dictionary<string, HashSet<string>>? resolvedPaths;
resolvedPaths = (await _ipcManager.Penumbra.GetCharacterData(_logger, playerRelatedObject).ConfigureAwait(false));
if (resolvedPaths == null) throw new InvalidOperationException("Penumbra returned null data");
ct.ThrowIfCancellationRequested();
fragment.FileReplacements =
new HashSet<FileReplacement>(resolvedPaths.Select(c => new FileReplacement([.. c.Value], c.Key)), FileReplacementComparer.Instance)
.Where(p => p.HasFileReplacement).ToHashSet();
fragment.FileReplacements.RemoveWhere(c => c.GamePaths.Any(g => !CacheMonitor.AllowedFileExtensions.Any(e => g.EndsWith(e, StringComparison.OrdinalIgnoreCase))));
ct.ThrowIfCancellationRequested();
fragment.FileReplacements = await staticBuildTask.ConfigureAwait(false);
if (logDebug)
{
_logger.LogDebug("== Static Replacements ==");
foreach (var replacement in fragment.FileReplacements.Where(i => i.HasFileReplacement).OrderBy(i => i.GamePaths.First(), StringComparer.OrdinalIgnoreCase))
foreach (var replacement in fragment.FileReplacements
.Where(i => i.HasFileReplacement)
.OrderBy(i => i.GamePaths.First(), StringComparer.OrdinalIgnoreCase))
{
_logger.LogDebug("=> {repl}", replacement);
ct.ThrowIfCancellationRequested();
}
}
else
var staticReplacements = new HashSet<FileReplacement>(fragment.FileReplacements, FileReplacementComparer.Instance);
var transientTask = ResolveTransientReplacementsAsync(
playerRelatedObject,
objectKind,
staticReplacements,
waitRecordingTask,
ct);
fragment.GlamourerString = await getGlamourerData.ConfigureAwait(false);
_logger.LogDebug("Glamourer is now: {data}", fragment.GlamourerString);
var customizeScale = await getCustomizeData.ConfigureAwait(false);
fragment.CustomizePlusScale = customizeScale ?? string.Empty;
_logger.LogDebug("Customize is now: {data}", fragment.CustomizePlusScale);
if (objectKind == ObjectKind.Player)
{
foreach (var replacement in fragment.FileReplacements.Where(i => i.HasFileReplacement))
{
ct.ThrowIfCancellationRequested();
}
}
CharacterDataFragmentPlayer? playerFragment = fragment as CharacterDataFragmentPlayer ?? throw new InvalidOperationException("Failed to cast CharacterDataFragment to Player variant");
await _transientResourceManager.WaitForRecording(ct).ConfigureAwait(false);
playerFragment.ManipulationString = _ipcManager.Penumbra.GetMetaManipulations();
playerFragment.HonorificData = await getHonorificTitle!.ConfigureAwait(false);
_logger.LogDebug("Honorific is now: {data}", playerFragment!.HonorificData);
// if it's pet then it's summoner, if it's summoner we actually want to keep all filereplacements alive at all times
// or we get into redraw city for every change and nothing works properly
if (objectKind == ObjectKind.Pet)
{
foreach (var item in fragment.FileReplacements.Where(i => i.HasFileReplacement).SelectMany(p => p.GamePaths))
{
if (_transientResourceManager.AddTransientResource(objectKind, item))
{
_logger.LogDebug("Marking static {item} for Pet as transient", item);
}
}
playerFragment.PetNamesData = _ipcManager.PetNames.GetLocalNames();
_logger.LogDebug("Pet Nicknames is now: {petnames}", playerFragment!.PetNamesData);
_logger.LogTrace("Clearing {count} Static Replacements for Pet", fragment.FileReplacements.Count);
fragment.FileReplacements.Clear();
playerFragment.HeelsData = await getHeelsOffset!.ConfigureAwait(false);
_logger.LogDebug("Heels is now: {heels}", playerFragment!.HeelsData);
playerFragment.MoodlesData = (await getMoodlesData!.ConfigureAwait(false)) ?? string.Empty;
_logger.LogDebug("Moodles is now: {moodles}", playerFragment!.MoodlesData);
}
ct.ThrowIfCancellationRequested();
_logger.LogDebug("Handling transient update for {obj}", playerRelatedObject);
// remove all potentially gathered paths from the transient resource manager that are resolved through static resolving
_transientResourceManager.ClearTransientPaths(objectKind, fragment.FileReplacements.SelectMany(c => c.GamePaths).ToList());
// get all remaining paths and resolve them
var transientPaths = ManageSemiTransientData(objectKind);
var resolvedTransientPaths = await GetFileReplacementsFromPaths(playerRelatedObject, transientPaths, new HashSet<string>(StringComparer.Ordinal)).ConfigureAwait(false);
var (resolvedTransientPaths, clearedForPet) = await transientTask.ConfigureAwait(false);
if (clearedForPet != null)
fragment.FileReplacements.Clear();
if (logDebug)
{
_logger.LogDebug("== Transient Replacements ==");
foreach (var replacement in resolvedTransientPaths.Select(c => new FileReplacement([.. c.Value], c.Key)).OrderBy(f => f.ResolvedPath, StringComparer.Ordinal))
foreach (var replacement in resolvedTransientPaths
.Select(c => new FileReplacement([.. c.Value], c.Key))
.OrderBy(f => f.ResolvedPath, StringComparer.Ordinal))
{
_logger.LogDebug("=> {repl}", replacement);
fragment.FileReplacements.Add(replacement);
@@ -208,85 +304,64 @@ public class PlayerDataFactory
else
{
foreach (var replacement in resolvedTransientPaths.Select(c => new FileReplacement([.. c.Value], c.Key)))
{
fragment.FileReplacements.Add(replacement);
}
}
// clean up all semi transient resources that don't have any file replacement (aka null resolve)
_transientResourceManager.CleanUpSemiTransientResources(objectKind, [.. fragment.FileReplacements]);
ct.ThrowIfCancellationRequested();
// make sure we only return data that actually has file replacements
fragment.FileReplacements = new HashSet<FileReplacement>(fragment.FileReplacements.Where(v => v.HasFileReplacement).OrderBy(v => v.ResolvedPath, StringComparer.Ordinal), FileReplacementComparer.Instance);
// gather up data from ipc
Task<string> getHeelsOffset = _ipcManager.Heels.GetOffsetAsync();
Task<string> getGlamourerData = _ipcManager.Glamourer.GetCharacterCustomizationAsync(playerRelatedObject.Address);
Task<string?> getCustomizeData = _ipcManager.CustomizePlus.GetScaleAsync(playerRelatedObject.Address);
Task<string> getHonorificTitle = _ipcManager.Honorific.GetTitle();
fragment.GlamourerString = await getGlamourerData.ConfigureAwait(false);
_logger.LogDebug("Glamourer is now: {data}", fragment.GlamourerString);
var customizeScale = await getCustomizeData.ConfigureAwait(false);
fragment.CustomizePlusScale = customizeScale ?? string.Empty;
_logger.LogDebug("Customize is now: {data}", fragment.CustomizePlusScale);
if (objectKind == ObjectKind.Player)
{
var playerFragment = (fragment as CharacterDataFragmentPlayer)!;
playerFragment.ManipulationString = _ipcManager.Penumbra.GetMetaManipulations();
playerFragment!.HonorificData = await getHonorificTitle.ConfigureAwait(false);
_logger.LogDebug("Honorific is now: {data}", playerFragment!.HonorificData);
playerFragment!.HeelsData = await getHeelsOffset.ConfigureAwait(false);
_logger.LogDebug("Heels is now: {heels}", playerFragment!.HeelsData);
playerFragment!.MoodlesData = await _ipcManager.Moodles.GetStatusAsync(playerRelatedObject.Address).ConfigureAwait(false) ?? string.Empty;
_logger.LogDebug("Moodles is now: {moodles}", playerFragment!.MoodlesData);
playerFragment!.PetNamesData = _ipcManager.PetNames.GetLocalNames();
_logger.LogDebug("Pet Nicknames is now: {petnames}", playerFragment!.PetNamesData);
}
fragment.FileReplacements = new HashSet<FileReplacement>(
fragment.FileReplacements
.Where(v => v.HasFileReplacement)
.OrderBy(v => v.ResolvedPath, StringComparer.Ordinal),
FileReplacementComparer.Instance);
ct.ThrowIfCancellationRequested();
var toCompute = fragment.FileReplacements.Where(f => !f.IsFileSwap).ToArray();
_logger.LogDebug("Getting Hashes for {amount} Files", toCompute.Length);
var computedPaths = _fileCacheManager.GetFileCachesByPaths(toCompute.Select(c => c.ResolvedPath).ToArray());
foreach (var file in toCompute)
await Task.Run(() =>
{
ct.ThrowIfCancellationRequested();
file.Hash = computedPaths[file.ResolvedPath]?.Hash ?? string.Empty;
}
var computedPaths = _fileCacheManager.GetFileCachesByPaths([.. toCompute.Select(c => c.ResolvedPath)]);
foreach (var file in toCompute)
{
ct.ThrowIfCancellationRequested();
file.Hash = computedPaths[file.ResolvedPath]?.Hash ?? string.Empty;
}
}, ct).ConfigureAwait(false);
var removed = fragment.FileReplacements.RemoveWhere(f => !f.IsFileSwap && string.IsNullOrEmpty(f.Hash));
if (removed > 0)
{
_logger.LogDebug("Removed {amount} of invalid files", removed);
}
ct.ThrowIfCancellationRequested();
Dictionary<string, List<ushort>>? boneIndices = null;
var hasPapFiles = false;
if (objectKind == ObjectKind.Player)
{
hasPapFiles = fragment.FileReplacements.Any(f =>
!f.IsFileSwap && f.GamePaths.First().EndsWith("pap", StringComparison.OrdinalIgnoreCase));
if (hasPapFiles)
{
boneIndices = await _dalamudUtil.RunOnFrameworkThread(() => _modelAnalyzer.GetSkeletonBoneIndices(playerRelatedObject)).ConfigureAwait(false);
}
}
if (objectKind == ObjectKind.Player)
{
hasPapFiles = fragment.FileReplacements.Any(f =>
!f.IsFileSwap && f.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase)));
if (hasPapFiles)
{
boneIndices = await _dalamudUtil
.RunOnFrameworkThread(() => _modelAnalyzer.GetSkeletonBoneIndices(playerRelatedObject))
.ConfigureAwait(false);
}
try
{
#if DEBUG
if (hasPapFiles && boneIndices != null)
_modelAnalyzer.DumpLocalSkeletonIndices(playerRelatedObject);
#endif
if (hasPapFiles)
{
await VerifyPlayerAnimationBones(boneIndices, (fragment as CharacterDataFragmentPlayer)!, ct).ConfigureAwait(false);
await VerifyPlayerAnimationBones(boneIndices, (CharacterDataFragmentPlayer)fragment, ct)
.ConfigureAwait(false);
}
}
catch (OperationCanceledException e)
@@ -300,105 +375,270 @@ public class PlayerDataFactory
}
}
_logger.LogInformation("Building character data for {obj} took {time}ms", objectKind, TimeSpan.FromTicks(DateTime.UtcNow.Ticks - start.Ticks).TotalMilliseconds);
_logger.LogInformation("Building character data for {obj} took {time}ms",
objectKind, sw.Elapsed.TotalMilliseconds);
return fragment;
}
private async Task VerifyPlayerAnimationBones(Dictionary<string, List<ushort>>? boneIndices, CharacterDataFragmentPlayer fragment, CancellationToken ct)
private async Task EnsureObjectPresentAsync(GameObjectHandler handler, CancellationToken ct)
{
if (boneIndices == null) return;
if (_logger.IsEnabled(LogLevel.Debug))
{
foreach (var kvp in boneIndices)
{
_logger.LogDebug("Found {skellyname} ({idx} bone indices) on player: {bones}", kvp.Key, kvp.Value.Any() ? kvp.Value.Max() : 0, string.Join(',', kvp.Value));
}
}
var maxPlayerBoneIndex = boneIndices.SelectMany(kvp => kvp.Value).DefaultIfEmpty().Max();
if (maxPlayerBoneIndex <= 0) return;
int noValidationFailed = 0;
foreach (var file in fragment.FileReplacements.Where(f => !f.IsFileSwap && f.GamePaths.First().EndsWith("pap", StringComparison.OrdinalIgnoreCase)).ToList())
var remaining = 10000;
while (remaining > 0)
{
ct.ThrowIfCancellationRequested();
var skeletonIndices = await _dalamudUtil.RunOnFrameworkThread(() => _modelAnalyzer.GetBoneIndicesFromPap(file.Hash)).ConfigureAwait(false);
bool validationFailed = false;
if (skeletonIndices != null)
var obj = await _dalamudUtil.CreateGameObjectAsync(handler.Address).ConfigureAwait(false);
if (await _dalamudUtil.IsObjectPresentAsync(obj).ConfigureAwait(false))
return;
_logger.LogTrace("Character is null but it shouldn't be, waiting");
await Task.Delay(50, ct).ConfigureAwait(false);
remaining -= 50;
}
}
private static HashSet<FileReplacement> BuildStaticReplacements(Dictionary<string, HashSet<string>> resolvedPaths)
{
var set = new HashSet<FileReplacement>(FileReplacementComparer.Instance);
foreach (var kvp in resolvedPaths)
{
var fr = new FileReplacement([.. kvp.Value], kvp.Key);
if (!fr.HasFileReplacement) continue;
var allAllowed = fr.GamePaths.All(g =>
CacheMonitor.AllowedFileExtensions.Any(e => g.EndsWith(e, StringComparison.OrdinalIgnoreCase)));
if (!allAllowed) continue;
set.Add(fr);
}
return set;
}
private async Task<(IReadOnlyDictionary<string, string[]> ResolvedPaths, HashSet<FileReplacement>? ClearedReplacements)>
ResolveTransientReplacementsAsync(
GameObjectHandler obj,
ObjectKind objectKind,
HashSet<FileReplacement> staticReplacements,
Task waitRecordingTask,
CancellationToken ct)
{
await waitRecordingTask.ConfigureAwait(false);
HashSet<FileReplacement>? clearedReplacements = null;
if (objectKind == ObjectKind.Pet)
{
foreach (var item in staticReplacements.Where(i => i.HasFileReplacement).SelectMany(p => p.GamePaths))
{
// 105 is the maximum vanilla skellington spoopy bone index
if (skeletonIndices.All(k => k.Value.Max() <= 105))
{
_logger.LogTrace("All indices of {path} are <= 105, ignoring", file.ResolvedPath);
continue;
}
if (_transientResourceManager.AddTransientResource(objectKind, item))
_logger.LogDebug("Marking static {item} for Pet as transient", item);
}
_logger.LogDebug("Verifying bone indices for {path}, found {x} skeletons", file.ResolvedPath, skeletonIndices.Count);
_logger.LogTrace("Clearing {count} Static Replacements for Pet", staticReplacements.Count);
clearedReplacements = staticReplacements;
}
foreach (var boneCount in skeletonIndices)
{
var maxAnimationIndex = boneCount.Value.DefaultIfEmpty().Max();
if (maxAnimationIndex > maxPlayerBoneIndex)
ct.ThrowIfCancellationRequested();
_transientResourceManager.ClearTransientPaths(objectKind, [.. staticReplacements.SelectMany(c => c.GamePaths)]);
var transientPaths = ManageSemiTransientData(objectKind);
if (transientPaths.Count == 0)
return (new Dictionary<string, string[]>(StringComparer.Ordinal), clearedReplacements);
var resolved = await GetFileReplacementsFromPaths(obj, transientPaths, new HashSet<string>(StringComparer.Ordinal))
.ConfigureAwait(false);
if (_maxTransientResolvedEntries > 0 && resolved.Count > _maxTransientResolvedEntries)
{
_logger.LogWarning("Transient entries ({resolved}) are above the threshold {max}; Please consider disable some mods (VFX have heavy load) to reduce transient load",
resolved.Count,
_maxTransientResolvedEntries);
}
return (resolved, clearedReplacements);
}
private async Task VerifyPlayerAnimationBones(
Dictionary<string, List<ushort>>? playerBoneIndices,
CharacterDataFragmentPlayer fragment,
CancellationToken ct)
{
var mode = _configService.Current.AnimationValidationMode;
var allowBasedShift = _configService.Current.AnimationAllowOneBasedShift;
var allownNightIndex = _configService.Current.AnimationAllowNeighborIndexTolerance;
if (mode == AnimationValidationMode.Unsafe)
return;
if (playerBoneIndices == null || playerBoneIndices.Count == 0)
return;
var localBoneSets = new Dictionary<string, HashSet<ushort>>(StringComparer.OrdinalIgnoreCase);
foreach (var (rawLocalKey, indices) in playerBoneIndices)
{
if (indices is not { Count: > 0 })
continue;
var key = XivDataAnalyzer.CanonicalizeSkeletonKey(rawLocalKey);
if (string.IsNullOrEmpty(key))
continue;
if (!localBoneSets.TryGetValue(key, out var set))
localBoneSets[key] = set = [];
foreach (var idx in indices)
set.Add(idx);
}
if (localBoneSets.Count == 0)
return;
if (_logger.IsEnabled(LogLevel.Debug))
{
_logger.LogDebug("SEND local buckets: {b}",
string.Join(", ", localBoneSets.Keys.Order(StringComparer.Ordinal)));
foreach (var kvp in localBoneSets.OrderBy(k => k.Key, StringComparer.OrdinalIgnoreCase))
{
var min = kvp.Value.Count > 0 ? kvp.Value.Min() : 0;
var max = kvp.Value.Count > 0 ? kvp.Value.Max() : 0;
_logger.LogDebug("Local bucket {bucket}: count={count} min={min} max={max}",
kvp.Key, kvp.Value.Count, min, max);
}
}
var papGroups = fragment.FileReplacements
.Where(f => !f.IsFileSwap
&& !string.IsNullOrEmpty(f.Hash)
&& f.GamePaths is { Count: > 0 }
&& f.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase)))
.GroupBy(f => f.Hash!, StringComparer.OrdinalIgnoreCase)
.ToList();
int noValidationFailed = 0;
foreach (var g in papGroups)
{
ct.ThrowIfCancellationRequested();
var hash = g.Key;
Dictionary<string, List<ushort>>? papIndices = null;
await _papParseLimiter.WaitAsync(ct).ConfigureAwait(false);
try
{
papIndices = await Task.Run(() => _modelAnalyzer.GetBoneIndicesFromPap(hash), ct)
.ConfigureAwait(false);
}
finally
{
_papParseLimiter.Release();
}
if (papIndices == null || papIndices.Count == 0)
continue;
if (papIndices.All(k => k.Value.DefaultIfEmpty().Max() <= 105))
continue;
if (_logger.IsEnabled(LogLevel.Debug))
{
var papBuckets = papIndices
.Select(kvp => new
{
_logger.LogWarning("Found more bone indices on the animation {path} skeleton {skl} (max indice {idx}) than on any player related skeleton (max indice {idx2})",
file.ResolvedPath, boneCount.Key, maxAnimationIndex, maxPlayerBoneIndex);
validationFailed = true;
break;
}
}
Raw = kvp.Key,
Key = XivDataAnalyzer.CanonicalizeSkeletonKey(kvp.Key),
Indices = kvp.Value
})
.Where(x => x.Indices is { Count: > 0 })
.GroupBy(x => string.IsNullOrEmpty(x.Key) ? x.Raw : x.Key!, StringComparer.OrdinalIgnoreCase)
.Select(grp =>
{
var all = grp.SelectMany(v => v.Indices).ToList();
var min = all.Count > 0 ? all.Min() : 0;
var max = all.Count > 0 ? all.Max() : 0;
var raws = string.Join(',', grp.Select(v => v.Raw).Distinct(StringComparer.OrdinalIgnoreCase));
return $"{grp.Key}(min={min},max={max},raw=[{raws}])";
})
.ToList();
_logger.LogDebug("SEND pap buckets for hash={hash}: {b}",
hash,
string.Join(" | ", papBuckets));
}
if (validationFailed)
{
noValidationFailed++;
_logger.LogDebug("Removing {file} from sent file replacements and transient data", file.ResolvedPath);
fragment.FileReplacements.Remove(file);
foreach (var gamePath in file.GamePaths)
{
_transientResourceManager.RemoveTransientResource(ObjectKind.Player, gamePath);
}
}
if (XivDataAnalyzer.IsPapCompatible(localBoneSets, papIndices, mode, allowBasedShift, allownNightIndex, out var reason))
continue;
noValidationFailed++;
_logger.LogWarning(
"Animation PAP hash {hash} is not compatible with local skeletons; dropping all mappings for this hash. Reason: {reason}",
hash,
reason);
var removedGamePaths = fragment.FileReplacements
.Where(fr => !fr.IsFileSwap
&& string.Equals(fr.Hash, hash, StringComparison.OrdinalIgnoreCase)
&& fr.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase)))
.SelectMany(fr => fr.GamePaths.Where(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase)))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
fragment.FileReplacements.RemoveWhere(fr =>
!fr.IsFileSwap
&& string.Equals(fr.Hash, hash, StringComparison.OrdinalIgnoreCase)
&& fr.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase)));
foreach (var gp in removedGamePaths)
_transientResourceManager.RemoveTransientResource(ObjectKind.Player, gp);
}
if (noValidationFailed > 0)
{
_lightlessMediator.Publish(new NotificationMessage("Invalid Skeleton Setup",
$"Your client is attempting to send {noValidationFailed} animation files with invalid bone data. Those animation files have been removed from your sent data. " +
$"Verify that you are using the correct skeleton for those animation files (Check /xllog for more information).",
NotificationType.Warning, TimeSpan.FromSeconds(10)));
_lightlessMediator.Publish(new NotificationMessage(
"Invalid Skeleton Setup",
$"Your client is attempting to send {noValidationFailed} animation files that don't match your current skeleton validation mode ({mode}). " +
"Please adjust your skeleton/mods or change the validation mode if this is unexpected. " +
"Those animation files have been removed from your sent (player) data. (Check /xllog for details).",
NotificationType.Warning,
TimeSpan.FromSeconds(10)));
}
}
private async Task<IReadOnlyDictionary<string, string[]>> GetFileReplacementsFromPaths(GameObjectHandler handler, HashSet<string> forwardResolve, HashSet<string> reverseResolve)
private async Task<IReadOnlyDictionary<string, string[]>> GetFileReplacementsFromPaths(
GameObjectHandler handler,
HashSet<string> forwardResolve,
HashSet<string> reverseResolve)
{
var forwardPaths = forwardResolve.ToArray();
var reversePaths = reverseResolve.ToArray();
Dictionary<string, List<string>> resolvedPaths = new(StringComparer.Ordinal);
if (handler.ObjectKind != ObjectKind.Player)
{
var (objectIndex, forwardResolved, reverseResolved) = await _dalamudUtil.RunOnFrameworkThread(() =>
{
var idx = handler.GetGameObject()?.ObjectIndex;
if (!idx.HasValue)
{
return ((int?)null, Array.Empty<string>(), Array.Empty<string[]>());
}
var resolvedForward = new string[forwardPaths.Length];
for (int i = 0; i < forwardPaths.Length; i++)
{
resolvedForward[i] = _ipcManager.Penumbra.ResolveGameObjectPath(forwardPaths[i], idx.Value);
}
var resolvedReverse = new string[reversePaths.Length][];
for (int i = 0; i < reversePaths.Length; i++)
{
resolvedReverse[i] = _ipcManager.Penumbra.ReverseResolveGameObjectPath(reversePaths[i], idx.Value);
}
return (idx, resolvedForward, resolvedReverse);
}).ConfigureAwait(false);
@@ -409,31 +649,21 @@ public class PlayerDataFactory
{
var filePath = forwardResolved[i]?.ToLowerInvariant();
if (string.IsNullOrEmpty(filePath))
{
continue;
}
if (resolvedPaths.TryGetValue(filePath, out var list))
{
list.Add(forwardPaths[i].ToLowerInvariant());
}
else
{
resolvedPaths[filePath] = [forwardPaths[i].ToLowerInvariant()];
}
}
for (int i = 0; i < reversePaths.Length; i++)
{
var filePath = reversePaths[i].ToLowerInvariant();
if (resolvedPaths.TryGetValue(filePath, out var list))
{
list.AddRange(reverseResolved[i].Select(c => c.ToLowerInvariant()));
}
else
{
resolvedPaths[filePath] = new List<string>(reverseResolved[i].Select(c => c.ToLowerInvariant()).ToList());
}
resolvedPaths[filePath] = [.. reverseResolved[i].Select(c => c.ToLowerInvariant()).ToList()];
}
return resolvedPaths.ToDictionary(k => k.Key, k => k.Value.ToArray(), StringComparer.OrdinalIgnoreCase).AsReadOnly();
@@ -441,30 +671,23 @@ public class PlayerDataFactory
}
var (forward, reverse) = await _ipcManager.Penumbra.ResolvePathsAsync(forwardPaths, reversePaths).ConfigureAwait(false);
for (int i = 0; i < forwardPaths.Length; i++)
{
var filePath = forward[i].ToLowerInvariant();
if (resolvedPaths.TryGetValue(filePath, out var list))
{
list.Add(forwardPaths[i].ToLowerInvariant());
}
else
{
resolvedPaths[filePath] = [forwardPaths[i].ToLowerInvariant()];
}
}
for (int i = 0; i < reversePaths.Length; i++)
{
var filePath = reversePaths[i].ToLowerInvariant();
if (resolvedPaths.TryGetValue(filePath, out var list))
{
list.AddRange(reverse[i].Select(c => c.ToLowerInvariant()));
}
else
{
resolvedPaths[filePath] = new List<string>(reverse[i].Select(c => c.ToLowerInvariant()).ToList());
}
resolvedPaths[filePath] = [.. reverse[i].Select(c => c.ToLowerInvariant()).ToList()];
}
return resolvedPaths.ToDictionary(k => k.Key, k => k.Value.ToArray(), StringComparer.OrdinalIgnoreCase).AsReadOnly();
@@ -475,11 +698,29 @@ public class PlayerDataFactory
_transientResourceManager.PersistTransientResources(objectKind);
HashSet<string> pathsToResolve = new(StringComparer.Ordinal);
foreach (var path in _transientResourceManager.GetSemiTransientResources(objectKind).Where(path => !string.IsNullOrEmpty(path)))
int scanned = 0, skippedEmpty = 0, skippedVfx = 0;
foreach (var path in _transientResourceManager.GetSemiTransientResources(objectKind))
{
scanned++;
if (string.IsNullOrEmpty(path))
{
skippedEmpty++;
continue;
}
pathsToResolve.Add(path);
}
if (_logger.IsEnabled(LogLevel.Debug))
{
_logger.LogDebug(
"ManageSemiTransientData({kind}): scanned={scanned}, added={added}, skippedEmpty={skippedEmpty}, skippedVfx={skippedVfx}",
objectKind, scanned, pathsToResolve.Count, skippedEmpty, skippedVfx);
}
return pathsToResolve;
}
}
}

View File

@@ -22,6 +22,7 @@ using Microsoft.Extensions.Logging;
using DalamudObjectKind = Dalamud.Game.ClientState.Objects.Enums.ObjectKind;
using ObjectKind = LightlessSync.API.Data.Enum.ObjectKind;
using FileReplacementDataComparer = LightlessSync.PlayerData.Data.FileReplacementDataComparer;
using LightlessSync.LightlessConfiguration;
namespace LightlessSync.PlayerData.Pairs;
@@ -46,7 +47,9 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
private readonly TextureDownscaleService _textureDownscaleService;
private readonly PairStateCache _pairStateCache;
private readonly PairPerformanceMetricsCache _performanceMetricsCache;
private readonly XivDataAnalyzer _modelAnalyzer;
private readonly PenumbraTempCollectionJanitor _tempCollectionJanitor;
private readonly LightlessConfigService _configService;
private readonly PairManager _pairManager;
private CancellationTokenSource? _applicationCancellationTokenSource;
private Guid _applicationId;
@@ -90,6 +93,10 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
".avfx",
".scd"
};
private readonly ConcurrentDictionary<string, byte> _blockedPapHashes = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, byte> _dumpedRemoteSkeletonForHash = new(StringComparer.OrdinalIgnoreCase);
private DateTime? _invisibleSinceUtc;
private DateTime? _visibilityEvictionDueAtUtc;
private DateTime _nextActorLookupUtc = DateTime.MinValue;
@@ -184,7 +191,9 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
TextureDownscaleService textureDownscaleService,
PairStateCache pairStateCache,
PairPerformanceMetricsCache performanceMetricsCache,
PenumbraTempCollectionJanitor tempCollectionJanitor) : base(logger, mediator)
PenumbraTempCollectionJanitor tempCollectionJanitor,
XivDataAnalyzer modelAnalyzer,
LightlessConfigService configService) : base(logger, mediator)
{
_pairManager = pairManager;
Ident = ident;
@@ -203,6 +212,8 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
_pairStateCache = pairStateCache;
_performanceMetricsCache = performanceMetricsCache;
_tempCollectionJanitor = tempCollectionJanitor;
_modelAnalyzer = modelAnalyzer;
_configService = configService;
}
public void Initialize()
@@ -1669,11 +1680,36 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
return;
}
SplitPapMappings(moddedPaths, out var withoutPap, out var papOnly);
await _ipcManager.Penumbra.AssignTemporaryCollectionAsync(Logger, penumbraCollection, objIndex.Value).ConfigureAwait(false);
await _ipcManager.Penumbra.SetTemporaryModsAsync(Logger, _applicationId, penumbraCollection,
moddedPaths.ToDictionary(k => k.Key.GamePath, k => k.Value, StringComparer.Ordinal)).ConfigureAwait(false);
_lastAppliedModdedPaths = new Dictionary<(string GamePath, string? Hash), string>(moddedPaths, moddedPaths.Comparer);
await _ipcManager.Penumbra.SetTemporaryModsAsync(
Logger, _applicationId, penumbraCollection,
withoutPap.ToDictionary(k => k.Key.GamePath, k => k.Value, StringComparer.Ordinal))
.ConfigureAwait(false);
await _ipcManager.Penumbra.RedrawAsync(Logger, handlerForApply, _applicationId, token).ConfigureAwait(false);
if (handlerForApply.Address != nint.Zero)
await _actorObjectService.WaitForFullyLoadedAsync(handlerForApply.Address, token).ConfigureAwait(false);
var removedPap = await StripIncompatiblePapAsync(handlerForApply, charaData, papOnly, token).ConfigureAwait(false);
if (removedPap > 0)
{
Logger.LogTrace("[{applicationId}] Removed {removedPap} incompatible PAP mappings found for {handler}", _applicationId, removedPap, GetLogIdentifier());
}
var merged = new Dictionary<(string GamePath, string? Hash), string>(withoutPap, withoutPap.Comparer);
foreach (var kv in papOnly)
merged[kv.Key] = kv.Value;
await _ipcManager.Penumbra.SetTemporaryModsAsync(
Logger, _applicationId, penumbraCollection,
merged.ToDictionary(k => k.Key.GamePath, k => k.Value, StringComparer.Ordinal))
.ConfigureAwait(false);
_lastAppliedModdedPaths = new Dictionary<(string GamePath, string? Hash), string>(merged, merged.Comparer);
LastAppliedDataBytes = -1;
foreach (var path in moddedPaths.Values.Distinct(StringComparer.OrdinalIgnoreCase).Select(v => new FileInfo(v)).Where(p => p.Exists))
{
@@ -1706,45 +1742,45 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
_needsCollectionRebuild = false;
if (LastAppliedApproximateVRAMBytes < 0 || LastAppliedApproximateEffectiveVRAMBytes < 0)
{
_playerPerformanceService.ComputeAndAutoPauseOnVRAMUsageThresholds(this, charaData, new List<DownloadFileTransfer>());
}
if (LastAppliedDataTris < 0)
{
await _playerPerformanceService.CheckTriangleUsageThresholds(this, charaData).ConfigureAwait(false);
}
_playerPerformanceService.ComputeAndAutoPauseOnVRAMUsageThresholds(this, charaData, new List<DownloadFileTransfer>());
}
if (LastAppliedDataTris < 0)
{
await _playerPerformanceService.CheckTriangleUsageThresholds(this, charaData).ConfigureAwait(false);
}
StorePerformanceMetrics(charaData);
_lastSuccessfulApplyAt = DateTime.UtcNow;
ClearFailureState();
Logger.LogDebug("[{applicationId}] Application finished", _applicationId);
}
catch (OperationCanceledException)
{
Logger.LogDebug("[{applicationId}] Application cancelled for {handler}", _applicationId, GetLogIdentifier());
_cachedData = charaData;
_pairStateCache.Store(Ident, charaData);
_forceFullReapply = true;
RecordFailure("Application cancelled", "Cancellation");
}
catch (Exception ex)
{
if (ex is AggregateException aggr && aggr.InnerExceptions.Any(e => e is ArgumentNullException))
StorePerformanceMetrics(charaData);
_lastSuccessfulApplyAt = DateTime.UtcNow;
ClearFailureState();
Logger.LogDebug("[{applicationId}] Application finished", _applicationId);
}
catch (OperationCanceledException)
{
IsVisible = false;
_forceApplyMods = true;
Logger.LogDebug("[{applicationId}] Application cancelled for {handler}", _applicationId, GetLogIdentifier());
_cachedData = charaData;
_pairStateCache.Store(Ident, charaData);
_forceFullReapply = true;
Logger.LogDebug("[{applicationId}] Cancelled, player turned null during application", _applicationId);
RecordFailure("Application cancelled", "Cancellation");
}
else
catch (Exception ex)
{
Logger.LogWarning(ex, "[{applicationId}] Cancelled", _applicationId);
_forceFullReapply = true;
if (ex is AggregateException aggr && aggr.InnerExceptions.Any(e => e is ArgumentNullException))
{
IsVisible = false;
_forceApplyMods = true;
_cachedData = charaData;
_pairStateCache.Store(Ident, charaData);
_forceFullReapply = true;
Logger.LogDebug("[{applicationId}] Cancelled, player turned null during application", _applicationId);
}
else
{
Logger.LogWarning(ex, "[{applicationId}] Cancelled", _applicationId);
_forceFullReapply = true;
}
RecordFailure($"Application failed: {ex.Message}", "Exception");
}
RecordFailure($"Application failed: {ex.Message}", "Exception");
}
}
private void FrameworkUpdate()
{
@@ -1978,14 +2014,37 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
if (string.IsNullOrEmpty(new FileInfo(fileCache.ResolvedFilepath).Extension))
{
hasMigrationChanges = true;
fileCache = _fileDbManager.MigrateFileHashToExtension(fileCache, item.GamePaths[0].Split(".")[^1]);
var anyGamePath = item.GamePaths.FirstOrDefault();
if (!string.IsNullOrEmpty(anyGamePath))
{
var ext = Path.GetExtension(anyGamePath);
var extNoDot = ext.StartsWith('.') ? ext[1..] : ext;
if (!string.IsNullOrEmpty(extNoDot))
{
hasMigrationChanges = true;
fileCache = _fileDbManager.MigrateFileHashToExtension(fileCache, extNoDot);
}
}
}
foreach (var gamePath in item.GamePaths)
{
var mode = _configService.Current.AnimationValidationMode;
if (mode != AnimationValidationMode.Unsafe
&& gamePath.EndsWith(".pap", StringComparison.OrdinalIgnoreCase)
&& !string.IsNullOrEmpty(item.Hash)
&& _blockedPapHashes.ContainsKey(item.Hash))
{
continue;
}
var preferredPath = skipDownscaleForPair
? fileCache.ResolvedFilepath
: _textureDownscaleService.GetPreferredPath(item.Hash, fileCache.ResolvedFilepath);
outputDict[(gamePath, item.Hash)] = preferredPath;
}
}
@@ -2295,7 +2354,7 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
HandleVisibilityLoss(logChange: false);
}
private bool TryResolveDescriptorHash(ActorObjectService.ActorDescriptor descriptor, out string hashedCid)
private static bool TryResolveDescriptorHash(ActorObjectService.ActorDescriptor descriptor, out string hashedCid)
{
hashedCid = descriptor.HashedContentId ?? string.Empty;
if (!string.IsNullOrEmpty(hashedCid))
@@ -2308,6 +2367,106 @@ internal sealed class PairHandlerAdapter : DisposableMediatorSubscriberBase, IPa
return !string.IsNullOrEmpty(hashedCid);
}
private static void SplitPapMappings(
Dictionary<(string GamePath, string? Hash), string> moddedPaths,
out Dictionary<(string GamePath, string? Hash), string> withoutPap,
out Dictionary<(string GamePath, string? Hash), string> papOnly)
{
withoutPap = new(moddedPaths.Comparer);
papOnly = new(moddedPaths.Comparer);
foreach (var kv in moddedPaths)
{
var gamePath = kv.Key.GamePath;
if (gamePath.EndsWith(".pap", StringComparison.OrdinalIgnoreCase))
papOnly[kv.Key] = kv.Value;
else
withoutPap[kv.Key] = kv.Value;
}
}
private async Task<int> StripIncompatiblePapAsync(
GameObjectHandler handlerForApply,
CharacterData charaData,
Dictionary<(string GamePath, string? Hash), string> papOnly,
CancellationToken token)
{
var mode = _configService.Current.AnimationValidationMode;
var allowBasedShift = _configService.Current.AnimationAllowOneBasedShift;
var allownNightIndex = _configService.Current.AnimationAllowNeighborIndexTolerance;
if (mode == AnimationValidationMode.Unsafe || papOnly.Count == 0)
return 0;
var boneIndices = await _dalamudUtil.RunOnFrameworkThread(
() => _modelAnalyzer.GetSkeletonBoneIndices(handlerForApply))
.ConfigureAwait(false);
if (boneIndices == null || boneIndices.Count == 0)
{
var removedCount = papOnly.Count;
papOnly.Clear();
return removedCount;
}
var localBoneSets = new Dictionary<string, HashSet<ushort>>(StringComparer.OrdinalIgnoreCase);
foreach (var (rawKey, list) in boneIndices)
{
var key = XivDataAnalyzer.CanonicalizeSkeletonKey(rawKey);
if (string.IsNullOrEmpty(key)) continue;
if (!localBoneSets.TryGetValue(key, out var set))
localBoneSets[key] = set = [];
foreach (var v in list)
set.Add(v);
}
int removed = 0;
foreach (var hash in papOnly.Keys.Select(k => k.Hash).Where(h => !string.IsNullOrEmpty(h)).Distinct(StringComparer.OrdinalIgnoreCase).ToList())
{
token.ThrowIfCancellationRequested();
var papIndices = await _dalamudUtil.RunOnFrameworkThread(
() => _modelAnalyzer.GetBoneIndicesFromPap(hash!))
.ConfigureAwait(false);
if (papIndices == null || papIndices.Count == 0)
continue;
if (papIndices.All(k => k.Value.DefaultIfEmpty().Max() <= 105))
continue;
if (XivDataAnalyzer.IsPapCompatible(localBoneSets, papIndices, mode, allowBasedShift, allownNightIndex, out var reason))
continue;
var keysToRemove = papOnly.Keys.Where(k => string.Equals(k.Hash, hash, StringComparison.OrdinalIgnoreCase)).ToList();
foreach (var k in keysToRemove)
papOnly.Remove(k);
removed += keysToRemove.Count;
if (_blockedPapHashes.TryAdd(hash!, 0))
Logger.LogWarning("Blocked remote object PAP (hash {hash}) for {handler}: {reason}", hash, GetLogIdentifier(), reason);
if (charaData.FileReplacements.TryGetValue(ObjectKind.Player, out var list))
{
list.RemoveAll(r => string.Equals(r.Hash, hash, StringComparison.OrdinalIgnoreCase)
&& r.GamePaths.Any(p => p.EndsWith(".pap", StringComparison.OrdinalIgnoreCase)));
}
}
var nullHashKeys = papOnly.Keys.Where(k => string.IsNullOrEmpty(k.Hash)).ToList();
foreach (var k in nullHashKeys)
{
papOnly.Remove(k);
removed++;
}
return removed;
}
private async Task ApplyCustomizeAsync(nint address, string customizeData, ObjectKind kind)
{
_customizeIds[kind] = await _ipcManager.CustomizePlus.SetBodyScaleAsync(address, customizeData).ConfigureAwait(false);

View File

@@ -1,5 +1,6 @@
using LightlessSync.FileCache;
using LightlessSync.Interop.Ipc;
using LightlessSync.LightlessConfiguration;
using LightlessSync.PlayerData.Factories;
using LightlessSync.Services;
using LightlessSync.Services.ActorTracking;
@@ -32,6 +33,8 @@ internal sealed class PairHandlerAdapterFactory : IPairHandlerAdapterFactory
private readonly PairStateCache _pairStateCache;
private readonly PairPerformanceMetricsCache _pairPerformanceMetricsCache;
private readonly PenumbraTempCollectionJanitor _tempCollectionJanitor;
private readonly LightlessConfigService _configService;
private readonly XivDataAnalyzer _modelAnalyzer;
public PairHandlerAdapterFactory(
ILoggerFactory loggerFactory,
@@ -50,7 +53,9 @@ internal sealed class PairHandlerAdapterFactory : IPairHandlerAdapterFactory
TextureDownscaleService textureDownscaleService,
PairStateCache pairStateCache,
PairPerformanceMetricsCache pairPerformanceMetricsCache,
PenumbraTempCollectionJanitor tempCollectionJanitor)
PenumbraTempCollectionJanitor tempCollectionJanitor,
XivDataAnalyzer modelAnalyzer,
LightlessConfigService configService)
{
_loggerFactory = loggerFactory;
_mediator = mediator;
@@ -69,6 +74,8 @@ internal sealed class PairHandlerAdapterFactory : IPairHandlerAdapterFactory
_pairStateCache = pairStateCache;
_pairPerformanceMetricsCache = pairPerformanceMetricsCache;
_tempCollectionJanitor = tempCollectionJanitor;
_modelAnalyzer = modelAnalyzer;
_configService = configService;
}
public IPairHandlerAdapter Create(string ident)
@@ -95,6 +102,8 @@ internal sealed class PairHandlerAdapterFactory : IPairHandlerAdapterFactory
_textureDownscaleService,
_pairStateCache,
_pairPerformanceMetricsCache,
_tempCollectionJanitor);
_tempCollectionJanitor,
_modelAnalyzer,
_configService);
}
}

View File

@@ -140,6 +140,7 @@ public sealed class Plugin : IDalamudPlugin
services.AddSingleton<IdDisplayHandler>();
services.AddSingleton<PlayerPerformanceService>();
services.AddSingleton<PenumbraTempCollectionJanitor>();
services.AddSingleton<LocationShareService>();
services.AddSingleton<TextureMetadataHelper>(sp =>
new TextureMetadataHelper(sp.GetRequiredService<ILogger<TextureMetadataHelper>>(), gameData));

View File

@@ -28,7 +28,7 @@ public sealed class CharacterAnalyzer : MediatorSubscriberBase, IDisposable
{
_baseAnalysisCts = _baseAnalysisCts.CancelRecreate();
var token = _baseAnalysisCts.Token;
_ = BaseAnalysis(msg.CharacterData, token);
_ = Task.Run(async () => await BaseAnalysis(msg.CharacterData, token).ConfigureAwait(false), token);
});
_fileCacheManager = fileCacheManager;
_xivDataAnalyzer = modelAnalyzer;

View File

@@ -10,7 +10,6 @@ using LightlessSync.UI;
using LightlessSync.UI.Services;
using LightlessSync.Utils;
using LightlessSync.WebAPI;
using Lumina.Excel.Sheets;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
@@ -172,9 +171,8 @@ internal class ContextMenuService : IHostedService
_logger.LogTrace("Cannot send pair request to {TargetName}@{World} while in PvP or GPose.", target.TargetName, target.TargetHomeWorld.RowId);
return;
}
var world = GetWorld(target.TargetHomeWorld.RowId);
if (!IsWorldValid(world))
if (!IsWorldValid(target.TargetHomeWorld.RowId))
{
_logger.LogTrace("Target player {TargetName}@{World} is on an invalid world.", target.TargetName, target.TargetHomeWorld.RowId);
return;
@@ -226,9 +224,8 @@ internal class ContextMenuService : IHostedService
{
if (args.Target is not MenuTargetDefault target)
return;
var world = GetWorld(target.TargetHomeWorld.RowId);
if (!IsWorldValid(world))
if (!target.TargetHomeWorld.IsValid || !IsWorldValid(target.TargetHomeWorld.RowId))
return;
try
@@ -237,7 +234,7 @@ internal class ContextMenuService : IHostedService
if (targetData == null || targetData.Address == nint.Zero)
{
_logger.LogWarning("Target player {TargetName}@{World} not found in object table.", target.TargetName, world.Name);
_logger.LogWarning("Target player {TargetName}@{World} not found in object table.", target.TargetName, target.TargetHomeWorld.Value.Name);
return;
}
@@ -252,7 +249,7 @@ internal class ContextMenuService : IHostedService
}
// Notify in chat when NotificationService is disabled
NotifyInChat($"Pair request sent to {target.TargetName}@{world.Name}.", NotificationType.Info);
NotifyInChat($"Pair request sent to {target.TargetName}@{target.TargetHomeWorld.Value.Name}.", NotificationType.Info);
}
catch (Exception ex)
{
@@ -312,37 +309,8 @@ internal class ContextMenuService : IHostedService
p.HomeWorld.RowId == target.TargetHomeWorld.RowId);
}
private World GetWorld(uint worldId)
private bool IsWorldValid(uint worldId)
{
var sheet = _gameData.GetExcelSheet<World>()!;
var luminaWorlds = sheet.Where(x =>
{
var dc = x.DataCenter.ValueNullable;
var name = x.Name.ExtractText();
var internalName = x.InternalName.ExtractText();
if (dc == null || dc.Value.Region == 0 || string.IsNullOrWhiteSpace(dc.Value.Name.ExtractText()))
return false;
if (string.IsNullOrWhiteSpace(name) || string.IsNullOrWhiteSpace(internalName))
return false;
if (name.Contains('-', StringComparison.Ordinal) || name.Contains('_', StringComparison.Ordinal))
return false;
return x.DataCenter.Value.Region != 5 || x.RowId > 3001 && x.RowId != 1200 && IsChineseJapaneseKoreanString(name);
});
return luminaWorlds.FirstOrDefault(x => x.RowId == worldId);
}
private static bool IsChineseJapaneseKoreanString(string text) => text.All(IsChineseJapaneseKoreanCharacter);
private static bool IsChineseJapaneseKoreanCharacter(char c) => c >= 0x4E00 && c <= 0x9FFF;
public static bool IsWorldValid(World world)
{
var name = world.Name.ToString();
return !string.IsNullOrWhiteSpace(name) && char.IsUpper(name[0]);
return _dalamudUtil.WorldData.Value.ContainsKey((ushort)worldId);
}
}

View File

@@ -1,11 +1,13 @@
using Dalamud.Game.ClientState.Conditions;
using Dalamud.Game.ClientState.Objects.SubKinds;
using Dalamud.Game.ClientState.Objects.Types;
using Dalamud.Game.Text;
using Dalamud.Plugin.Services;
using Dalamud.Utility;
using FFXIVClientStructs.FFXIV.Client.Game;
using FFXIVClientStructs.FFXIV.Client.Game.Character;
using FFXIVClientStructs.FFXIV.Client.Game.Control;
using FFXIVClientStructs.FFXIV.Client.Game.UI;
using FFXIVClientStructs.FFXIV.Client.Graphics.Scene;
using FFXIVClientStructs.FFXIV.Client.UI.Agent;
using LightlessSync.API.Dto.CharaData;
@@ -20,12 +22,15 @@ using LightlessSync.Utils;
using Lumina.Excel.Sheets;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System.Diagnostics;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using BattleNpcSubKind = FFXIVClientStructs.FFXIV.Client.Game.Object.BattleNpcSubKind;
using DalamudObjectKind = Dalamud.Game.ClientState.Objects.Enums.ObjectKind;
using GameObject = FFXIVClientStructs.FFXIV.Client.Game.Object.GameObject;
using Map = Lumina.Excel.Sheets.Map;
using VisibilityFlags = FFXIVClientStructs.FFXIV.Client.Game.Object.VisibilityFlags;
namespace LightlessSync.Services;
@@ -57,6 +62,7 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber
private string _lastGlobalBlockReason = string.Empty;
private ushort _lastZone = 0;
private ushort _lastWorldId = 0;
private uint _lastMapId = 0;
private bool _sentBetweenAreas = false;
private Lazy<ulong> _cid;
@@ -86,7 +92,8 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber
WorldData = new(() =>
{
return gameData.GetExcelSheet<Lumina.Excel.Sheets.World>(clientLanguage)!
.Where(w => !w.Name.IsEmpty && w.DataCenter.RowId != 0 && (w.IsPublic || char.IsUpper(w.Name.ToString()[0])))
.Where(w => !w.Name.IsEmpty && w.DataCenter.RowId != 0 && (w.IsPublic || char.IsUpper(w.Name.ToString()[0])
|| w is { RowId: > 1000, Region: 101 or 201 }))
.ToDictionary(w => (ushort)w.RowId, w => w.Name.ToString());
});
JobData = new(() =>
@@ -659,7 +666,7 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber
var location = new LocationInfo();
location.ServerId = _playerState.CurrentWorld.RowId;
//location.InstanceId = UIState.Instance()->PublicInstance.InstanceId; //TODO:Need API update first
location.InstanceId = UIState.Instance()->PublicInstance.InstanceId;
location.TerritoryId = _clientState.TerritoryType;
location.MapId = _clientState.MapId;
if (houseMan != null)
@@ -685,7 +692,7 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber
var outside = houseMan->OutdoorTerritory;
var house = outside->HouseId;
location.WardId = house.WardIndex + 1u;
location.HouseId = (uint)houseMan->GetCurrentPlot() + 1;
//location.HouseId = (uint)houseMan->GetCurrentPlot() + 1;
location.DivisionId = houseMan->GetCurrentDivision();
}
//_logger.LogWarning(LocationToString(location));
@@ -713,10 +720,10 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber
str += $" - {MapData.Value[(ushort)location.MapId].MapName}";
}
// if (location.InstanceId is not 0)
// {
// str += ((SeIconChar)(57520 + location.InstanceId)).ToIconString();
// }
if (location.InstanceId is not 0)
{
str += ((SeIconChar)(57520 + location.InstanceId)).ToIconString();
}
if (location.WardId is not 0)
{
@@ -838,31 +845,41 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber
return Task.CompletedTask;
}
public async Task WaitWhileCharacterIsDrawing(ILogger logger, GameObjectHandler handler, Guid redrawId, int timeOut = 5000, CancellationToken? ct = null)
public async Task WaitWhileCharacterIsDrawing(
ILogger logger,
GameObjectHandler handler,
Guid redrawId,
int timeOut = 5000,
CancellationToken? ct = null)
{
if (!_clientState.IsLoggedIn) return;
if (ct == null)
ct = CancellationToken.None;
var token = ct ?? CancellationToken.None;
const int tick = 250;
const int initialSettle = 50;
var sw = Stopwatch.StartNew();
const int tick = 250;
int curWaitTime = 0;
try
{
logger.LogTrace("[{redrawId}] Starting wait for {handler} to draw", redrawId, handler);
await Task.Delay(tick, ct.Value).ConfigureAwait(true);
curWaitTime += tick;
while ((!ct.Value.IsCancellationRequested)
&& curWaitTime < timeOut
&& await handler.IsBeingDrawnRunOnFrameworkAsync().ConfigureAwait(false)) // 0b100000000000 is "still rendering" or something
await Task.Delay(initialSettle, token).ConfigureAwait(false);
while (!token.IsCancellationRequested
&& sw.ElapsedMilliseconds < timeOut
&& await handler.IsBeingDrawnRunOnFrameworkAsync().ConfigureAwait(false))
{
logger.LogTrace("[{redrawId}] Waiting for {handler} to finish drawing", redrawId, handler);
curWaitTime += tick;
await Task.Delay(tick, ct.Value).ConfigureAwait(true);
await Task.Delay(tick, token).ConfigureAwait(false);
}
logger.LogTrace("[{redrawId}] Finished drawing after {curWaitTime}ms", redrawId, curWaitTime);
logger.LogTrace("[{redrawId}] Finished drawing after {ms}ms", redrawId, sw.ElapsedMilliseconds);
}
catch (OperationCanceledException)
{
// ignore
}
catch (AccessViolationException ex)
{
@@ -1135,6 +1152,18 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber
Mediator.Publish(new ZoneSwitchEndMessage());
Mediator.Publish(new ResumeScanMessage(nameof(ConditionFlag.BetweenAreas)));
}
//Map
if (!_sentBetweenAreas)
{
var mapid = _clientState.MapId;
if (mapid != _lastMapId)
{
_lastMapId = mapid;
Mediator.Publish(new MapChangedMessage(mapid));
}
}
var localPlayer = _objectTable.LocalPlayer;
if (localPlayer != null)

View File

@@ -0,0 +1,137 @@
using LightlessSync.API.Data;
using LightlessSync.API.Dto.CharaData;
using LightlessSync.API.Dto.User;
using LightlessSync.Services.Mediator;
using LightlessSync.WebAPI;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Primitives;
namespace LightlessSync.Services
{
public class LocationShareService : DisposableMediatorSubscriberBase
{
private readonly DalamudUtilService _dalamudUtilService;
private readonly ApiController _apiController;
private IMemoryCache _locations = new MemoryCache(new MemoryCacheOptions());
private IMemoryCache _sharingStatus = new MemoryCache(new MemoryCacheOptions());
private CancellationTokenSource _resetToken = new CancellationTokenSource();
public LocationShareService(ILogger<LocationShareService> logger, LightlessMediator mediator, DalamudUtilService dalamudUtilService, ApiController apiController) : base(logger, mediator)
{
_dalamudUtilService = dalamudUtilService;
_apiController = apiController;
Mediator.Subscribe<DisconnectedMessage>(this, (msg) =>
{
_resetToken.Cancel();
_resetToken.Dispose();
_resetToken = new CancellationTokenSource();
});
Mediator.Subscribe<ConnectedMessage>(this, (msg) =>
{
_ = _apiController.UpdateLocation(new LocationDto(new UserData(_apiController.UID, apiController.DisplayName), _dalamudUtilService.GetMapData()));
_ = RequestAllLocation();
} );
Mediator.Subscribe<LocationSharingMessage>(this, UpdateLocationList);
Mediator.Subscribe<MapChangedMessage>(this,
msg => _ = _apiController.UpdateLocation(new LocationDto(new UserData(_apiController.UID, _apiController.DisplayName), _dalamudUtilService.GetMapData())));
}
private void UpdateLocationList(LocationSharingMessage msg)
{
if (_locations.TryGetValue(msg.User.UID, out _) && msg.LocationInfo.ServerId is 0)
{
_locations.Remove(msg.User.UID);
return;
}
if ( msg.LocationInfo.ServerId is not 0 && msg.ExpireAt > DateTime.UtcNow)
{
AddLocationInfo(msg.User.UID, msg.LocationInfo, msg.ExpireAt);
}
}
private void AddLocationInfo(string uid, LocationInfo location, DateTimeOffset expireAt)
{
var options = new MemoryCacheEntryOptions()
.SetAbsoluteExpiration(expireAt)
.AddExpirationToken(new CancellationChangeToken(_resetToken.Token));
_locations.Set(uid, location, options);
}
private async Task RequestAllLocation()
{
try
{
var (data, status) = await _apiController.RequestAllLocationInfo().ConfigureAwait(false);
foreach (var dto in data)
{
AddLocationInfo(dto.LocationDto.User.UID, dto.LocationDto.Location, dto.ExpireAt);
}
foreach (var dto in status)
{
AddStatus(dto.User.UID, dto.ExpireAt);
}
}
catch (Exception e)
{
Logger.LogError(e,"RequestAllLocation error : ");
throw;
}
}
private void AddStatus(string uid, DateTimeOffset expireAt)
{
var options = new MemoryCacheEntryOptions()
.SetAbsoluteExpiration(expireAt)
.AddExpirationToken(new CancellationChangeToken(_resetToken.Token));
_sharingStatus.Set(uid, expireAt, options);
}
public string GetUserLocation(string uid)
{
try
{
if (_locations.TryGetValue<LocationInfo>(uid, out var location))
{
return _dalamudUtilService.LocationToString(location);
}
return String.Empty;
}
catch (Exception e)
{
Logger.LogError(e,"GetUserLocation error : ");
throw;
}
}
public DateTimeOffset GetSharingStatus(string uid)
{
try
{
if (_sharingStatus.TryGetValue<DateTimeOffset>(uid, out var expireAt))
{
return expireAt;
}
return DateTimeOffset.MinValue;
}
catch (Exception e)
{
Logger.LogError(e,"GetSharingStatus error : ");
throw;
}
}
public void UpdateSharingStatus(List<string> users, DateTimeOffset expireAt)
{
foreach (var user in users)
{
AddStatus(user, expireAt);
}
}
}
}

View File

@@ -135,5 +135,7 @@ public record ChatChannelsUpdated : MessageBase;
public record ChatChannelMessageAdded(string ChannelKey, ChatMessageEntry Message) : MessageBase;
public record GroupCollectionChangedMessage : MessageBase;
public record OpenUserProfileMessage(UserData User) : MessageBase;
public record LocationSharingMessage(UserData User, LocationInfo LocationInfo, DateTimeOffset ExpireAt) : MessageBase;
public record MapChangedMessage(uint MapId) : MessageBase;
#pragma warning restore S2094
#pragma warning restore MA0048 // File name must match type name

View File

@@ -6,13 +6,15 @@ using FFXIVClientStructs.Havok.Common.Serialize.Util;
using LightlessSync.FileCache;
using LightlessSync.Interop.GameModel;
using LightlessSync.LightlessConfiguration;
using LightlessSync.PlayerData.Factories;
using LightlessSync.PlayerData.Handlers;
using Microsoft.Extensions.Logging;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
namespace LightlessSync.Services;
public sealed class XivDataAnalyzer
public sealed partial class XivDataAnalyzer
{
private readonly ILogger<XivDataAnalyzer> _logger;
private readonly FileCacheManager _fileCacheManager;
@@ -29,127 +31,441 @@ public sealed class XivDataAnalyzer
public unsafe Dictionary<string, List<ushort>>? GetSkeletonBoneIndices(GameObjectHandler handler)
{
if (handler.Address == nint.Zero) return null;
var chara = (CharacterBase*)(((Character*)handler.Address)->GameObject.DrawObject);
if (chara->GetModelType() != CharacterBase.ModelType.Human) return null;
var resHandles = chara->Skeleton->SkeletonResourceHandles;
Dictionary<string, List<ushort>> outputIndices = [];
if (handler is null || handler.Address == nint.Zero)
return null;
Dictionary<string, HashSet<ushort>> sets = new(StringComparer.OrdinalIgnoreCase);
try
{
for (int i = 0; i < chara->Skeleton->PartialSkeletonCount; i++)
var drawObject = ((Character*)handler.Address)->GameObject.DrawObject;
if (drawObject == null)
return null;
var chara = (CharacterBase*)drawObject;
if (chara->GetModelType() != CharacterBase.ModelType.Human)
return null;
var skeleton = chara->Skeleton;
if (skeleton == null)
return null;
var resHandles = skeleton->SkeletonResourceHandles;
var partialCount = skeleton->PartialSkeletonCount;
if (partialCount <= 0)
return null;
for (int i = 0; i < partialCount; i++)
{
var handle = *(resHandles + i);
_logger.LogTrace("Iterating over SkeletonResourceHandle #{i}:{x}", i, ((nint)handle).ToString("X"));
if ((nint)handle == nint.Zero) continue;
var curBones = handle->BoneCount;
// this is unrealistic, the filename shouldn't ever be that long
if (handle->FileName.Length > 1024) continue;
var skeletonName = handle->FileName.ToString();
if (string.IsNullOrEmpty(skeletonName)) continue;
outputIndices[skeletonName] = [];
for (ushort boneIdx = 0; boneIdx < curBones; boneIdx++)
if ((nint)handle == nint.Zero)
continue;
if (handle->FileName.Length > 1024)
continue;
var rawName = handle->FileName.ToString();
if (string.IsNullOrWhiteSpace(rawName))
continue;
var skeletonKey = CanonicalizeSkeletonKey(rawName);
if (string.IsNullOrEmpty(skeletonKey))
continue;
var boneCount = handle->BoneCount;
if (boneCount == 0)
continue;
var havokSkel = handle->HavokSkeleton;
if ((nint)havokSkel == nint.Zero)
continue;
if (!sets.TryGetValue(skeletonKey, out var set))
{
var boneName = handle->HavokSkeleton->Bones[boneIdx].Name.String;
if (boneName == null) continue;
outputIndices[skeletonName].Add((ushort)(boneIdx + 1));
set = [];
sets[skeletonKey] = set;
}
uint maxExclusive = boneCount;
uint ushortExclusive = (uint)ushort.MaxValue + 1u;
if (maxExclusive > ushortExclusive)
maxExclusive = ushortExclusive;
for (uint boneIdx = 0; boneIdx < maxExclusive; boneIdx++)
{
var name = havokSkel->Bones[boneIdx].Name.String;
if (name == null)
continue;
set.Add((ushort)boneIdx);
}
_logger.LogTrace("Local skeleton raw file='{raw}', key='{key}', boneCount={count}",
rawName, skeletonKey, boneCount);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Could not process skeleton data");
return null;
}
return (outputIndices.Count != 0 && outputIndices.Values.All(u => u.Count > 0)) ? outputIndices : null;
if (sets.Count == 0)
return null;
var output = new Dictionary<string, List<ushort>>(sets.Count, StringComparer.OrdinalIgnoreCase);
foreach (var (key, set) in sets)
{
if (set.Count == 0)
continue;
var list = set.ToList();
list.Sort();
output[key] = list;
}
return (output.Count != 0 && output.Values.All(v => v.Count > 0)) ? output : null;
}
public unsafe Dictionary<string, List<ushort>>? GetBoneIndicesFromPap(string hash)
public unsafe Dictionary<string, List<ushort>>? GetBoneIndicesFromPap(string hash, bool persistToConfig = true)
{
if (_configService.Current.BonesDictionary.TryGetValue(hash, out var bones)) return bones;
if (string.IsNullOrWhiteSpace(hash))
return null;
if (_configService.Current.BonesDictionary.TryGetValue(hash, out var cached) && cached is not null)
return cached;
var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash);
if (cacheEntity == null) return null;
if (cacheEntity == null || string.IsNullOrEmpty(cacheEntity.ResolvedFilepath) || !File.Exists(cacheEntity.ResolvedFilepath))
return null;
using BinaryReader reader = new(File.Open(cacheEntity.ResolvedFilepath, FileMode.Open, FileAccess.Read, FileShare.Read));
using var fs = File.Open(cacheEntity.ResolvedFilepath, FileMode.Open, FileAccess.Read, FileShare.Read);
using var reader = new BinaryReader(fs);
// most of this shit is from vfxeditor, surely nothing will change in the pap format :copium:
reader.ReadInt32(); // ignore
reader.ReadInt32(); // ignore
reader.ReadInt16(); // read 2 (num animations)
reader.ReadInt16(); // read 2 (modelid)
var type = reader.ReadByte();// read 1 (type)
if (type != 0) return null; // it's not human, just ignore it, whatever
// PAP header (mostly from vfxeditor)
_ = reader.ReadInt32(); // ignore
_ = reader.ReadInt32(); // ignore
_ = reader.ReadInt16(); // num animations
_ = reader.ReadInt16(); // modelid
var type = reader.ReadByte(); // type
if (type != 0)
return null; // not human
_ = reader.ReadByte(); // variant
_ = reader.ReadInt32(); // ignore
reader.ReadByte(); // read 1 (variant)
reader.ReadInt32(); // ignore
var havokPosition = reader.ReadInt32();
var footerPosition = reader.ReadInt32();
var havokDataSize = footerPosition - havokPosition;
// sanity checks
if (havokPosition <= 0 || footerPosition <= havokPosition || footerPosition > fs.Length)
return null;
var havokDataSizeLong = (long)footerPosition - havokPosition;
if (havokDataSizeLong <= 8 || havokDataSizeLong > int.MaxValue)
return null;
var havokDataSize = (int)havokDataSizeLong;
reader.BaseStream.Position = havokPosition;
var havokData = reader.ReadBytes(havokDataSize);
if (havokData.Length <= 8) return null; // no havok data
if (havokData.Length <= 8)
return null;
var output = new Dictionary<string, List<ushort>>(StringComparer.OrdinalIgnoreCase);
var tempHavokDataPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()) + ".hkx";
var tempHavokDataPathAnsi = Marshal.StringToHGlobalAnsi(tempHavokDataPath);
var tempSets = new Dictionary<string, HashSet<ushort>>(StringComparer.OrdinalIgnoreCase);
var tempHavokDataPath = Path.Combine(Path.GetTempPath(), $"lightless_{Guid.NewGuid():N}.hkx");
IntPtr tempHavokDataPathAnsi = IntPtr.Zero;
try
{
File.WriteAllBytes(tempHavokDataPath, havokData);
if (!File.Exists(tempHavokDataPath))
{
_logger.LogTrace("Temporary havok file did not exist when attempting to load: {path}", tempHavokDataPath);
return null;
}
tempHavokDataPathAnsi = Marshal.StringToHGlobalAnsi(tempHavokDataPath);
var loadoptions = stackalloc hkSerializeUtil.LoadOptions[1];
loadoptions->TypeInfoRegistry = hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry();
loadoptions->ClassNameRegistry = hkBuiltinTypeRegistry.Instance()->GetClassNameRegistry();
loadoptions->Flags = new hkFlags<hkSerializeUtil.LoadOptionBits, int>
{
Storage = (int)(hkSerializeUtil.LoadOptionBits.Default)
Storage = (int)hkSerializeUtil.LoadOptionBits.Default
};
var resource = hkSerializeUtil.LoadFromFile((byte*)tempHavokDataPathAnsi, null, loadoptions);
if (resource == null)
{
throw new InvalidOperationException("Resource was null after loading");
_logger.LogWarning("Havok resource was null after loading from {path}", tempHavokDataPath);
return null;
}
var rootLevelName = @"hkRootLevelContainer"u8;
fixed (byte* n1 = rootLevelName)
{
var container = (hkRootLevelContainer*)resource->GetContentsPointer(n1, hkBuiltinTypeRegistry.Instance()->GetTypeInfoRegistry());
if (container == null)
return null;
var animationName = @"hkaAnimationContainer"u8;
fixed (byte* n2 = animationName)
{
var animContainer = (hkaAnimationContainer*)container->findObjectByName(n2, null);
if (animContainer == null)
return null;
for (int i = 0; i < animContainer->Bindings.Length; i++)
{
var binding = animContainer->Bindings[i].ptr;
if (binding == null)
continue;
var rawSkel = binding->OriginalSkeletonName.String;
var skeletonKey = CanonicalizeSkeletonKey(rawSkel);
if (string.IsNullOrEmpty(skeletonKey))
continue;
var boneTransform = binding->TransformTrackToBoneIndices;
string name = binding->OriginalSkeletonName.String! + "_" + i;
output[name] = [];
if (boneTransform.Length <= 0)
continue;
if (!tempSets.TryGetValue(skeletonKey, out var set))
{
set = [];
tempSets[skeletonKey] = set;
}
for (int boneIdx = 0; boneIdx < boneTransform.Length; boneIdx++)
{
output[name].Add((ushort)boneTransform[boneIdx]);
var v = boneTransform[boneIdx];
if (v < 0) continue;
set.Add((ushort)v);
}
output[name].Sort();
}
}
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Could not load havok file in {path}", tempHavokDataPath);
return null;
}
finally
{
Marshal.FreeHGlobal(tempHavokDataPathAnsi);
File.Delete(tempHavokDataPath);
if (tempHavokDataPathAnsi != IntPtr.Zero)
Marshal.FreeHGlobal(tempHavokDataPathAnsi);
try
{
if (File.Exists(tempHavokDataPath))
File.Delete(tempHavokDataPath);
}
catch (Exception ex)
{
_logger.LogTrace(ex, "Could not delete temporary havok file: {path}", tempHavokDataPath);
}
}
if (tempSets.Count == 0)
return null;
var output = new Dictionary<string, List<ushort>>(tempSets.Count, StringComparer.OrdinalIgnoreCase);
foreach (var (key, set) in tempSets)
{
if (set.Count == 0) continue;
var list = set.ToList();
list.Sort();
output[key] = list;
}
if (output.Count == 0)
return null;
_configService.Current.BonesDictionary[hash] = output;
_configService.Save();
if (persistToConfig)
_configService.Save();
return output;
}
public static string CanonicalizeSkeletonKey(string? raw)
{
if (string.IsNullOrWhiteSpace(raw))
return string.Empty;
var s = raw.Replace('\\', '/').Trim();
var underscore = s.LastIndexOf('_');
if (underscore > 0 && underscore + 1 < s.Length && char.IsDigit(s[underscore + 1]))
s = s[..underscore];
if (s.StartsWith("skeleton", StringComparison.OrdinalIgnoreCase))
return "skeleton";
var m = _bucketPathRegex.Match(s);
if (m.Success)
return m.Groups["bucket"].Value.ToLowerInvariant();
m = _bucketSklRegex.Match(s);
if (m.Success)
return m.Groups["bucket"].Value.ToLowerInvariant();
m = _bucketLooseRegex.Match(s);
if (m.Success)
return m.Groups["bucket"].Value.ToLowerInvariant();
return string.Empty;
}
public static bool ContainsIndexCompat(
HashSet<ushort> available,
ushort idx,
bool papLikelyOneBased,
bool allowOneBasedShift,
bool allowNeighborTolerance)
{
Span<ushort> candidates = stackalloc ushort[2];
int count = 0;
candidates[count++] = idx;
if (allowOneBasedShift && papLikelyOneBased && idx > 0)
candidates[count++] = (ushort)(idx - 1);
for (int i = 0; i < count; i++)
{
var c = candidates[i];
if (available.Contains(c))
return true;
if (allowNeighborTolerance)
{
if (c > 0 && available.Contains((ushort)(c - 1)))
return true;
if (c < ushort.MaxValue && available.Contains((ushort)(c + 1)))
return true;
}
}
return false;
}
public static bool IsPapCompatible(
IReadOnlyDictionary<string, HashSet<ushort>> localBoneSets,
IReadOnlyDictionary<string, List<ushort>> papBoneIndices,
AnimationValidationMode mode,
bool allowOneBasedShift,
bool allowNeighborTolerance,
out string reason)
{
reason = string.Empty;
if (mode == AnimationValidationMode.Unsafe)
return true;
var papBuckets = papBoneIndices.Keys
.Select(CanonicalizeSkeletonKey)
.Where(k => !string.IsNullOrEmpty(k))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
if (papBuckets.Count == 0)
{
reason = "No skeleton bucket bindings found in the PAP";
return false;
}
if (mode == AnimationValidationMode.Safe)
{
if (papBuckets.Any(b => localBoneSets.ContainsKey(b)))
return true;
reason = $"No matching skeleton bucket between PAP [{string.Join(", ", papBuckets)}] and local [{string.Join(", ", localBoneSets.Keys.Order())}].";
return false;
}
foreach (var bucket in papBuckets)
{
if (!localBoneSets.TryGetValue(bucket, out var available))
{
reason = $"Missing skeleton bucket '{bucket}' on local actor.";
return false;
}
var indices = papBoneIndices
.Where(kvp => string.Equals(CanonicalizeSkeletonKey(kvp.Key), bucket, StringComparison.OrdinalIgnoreCase))
.SelectMany(kvp => kvp.Value ?? Enumerable.Empty<ushort>())
.Distinct()
.ToList();
if (indices.Count == 0)
continue;
bool has0 = false, has1 = false;
ushort min = ushort.MaxValue;
foreach (var v in indices)
{
if (v == 0) has0 = true;
if (v == 1) has1 = true;
if (v < min) min = v;
}
bool papLikelyOneBased = allowOneBasedShift && (min == 1) && has1 && !has0;
foreach (var idx in indices)
{
if (!ContainsIndexCompat(available, idx, papLikelyOneBased, allowOneBasedShift, allowNeighborTolerance))
{
reason = $"No compatible local skeleton for PAP '{bucket}': missing bone index {idx}.";
return false;
}
}
}
return true;
}
public void DumpLocalSkeletonIndices(GameObjectHandler handler, string? filter = null)
{
var skels = GetSkeletonBoneIndices(handler);
if (skels == null)
{
_logger.LogTrace("DumpLocalSkeletonIndices: local skeleton indices are null or not found");
return;
}
var keys = skels.Keys
.Order(StringComparer.OrdinalIgnoreCase)
.ToArray();
_logger.LogTrace("Local skeleton indices found ({count}): {keys}",
keys.Length,
string.Join(", ", keys));
if (!string.IsNullOrWhiteSpace(filter))
{
var hits = keys.Where(k =>
k.Equals(filter, StringComparison.OrdinalIgnoreCase) ||
k.StartsWith(filter + "_", StringComparison.OrdinalIgnoreCase) ||
filter.StartsWith(k + "_", StringComparison.OrdinalIgnoreCase) ||
k.Contains(filter, StringComparison.OrdinalIgnoreCase))
.ToArray();
_logger.LogTrace("Matches found for '{filter}': {hits}",
filter,
hits.Length == 0 ? "<none>" : string.Join(", ", hits));
}
}
public async Task<long> GetTrianglesByHash(string hash)
{
if (_configService.Current.TriangleDictionary.TryGetValue(hash, out var cachedTris) && cachedTris > 0)
@@ -212,4 +528,23 @@ public sealed class XivDataAnalyzer
return 0;
}
}
// Regexes for canonicalizing skeleton keys
private static readonly Regex _bucketPathRegex =
BucketRegex();
private static readonly Regex _bucketSklRegex =
SklRegex();
private static readonly Regex _bucketLooseRegex =
LooseBucketRegex();
[GeneratedRegex(@"(?i)(?:^|/)(?<bucket>c\d{4})(?:/|$)", RegexOptions.Compiled, "en-NL")]
private static partial Regex BucketRegex();
[GeneratedRegex(@"(?i)\bskl_(?<bucket>c\d{4})[a-z]\d{4}\b", RegexOptions.Compiled, "en-NL")]
private static partial Regex SklRegex();
[GeneratedRegex(@"(?i)(?<![a-z0-9])(?<bucket>c\d{4})(?!\d)", RegexOptions.Compiled, "en-NL")]
private static partial Regex LooseBucketRegex();
}

View File

@@ -37,6 +37,7 @@ public class DrawUserPair
private readonly UiSharedService _uiSharedService;
private readonly PlayerPerformanceConfigService _performanceConfigService;
private readonly LightlessConfigService _configService;
private readonly LocationShareService _locationShareService;
private readonly CharaDataManager _charaDataManager;
private readonly PairLedger _pairLedger;
private float _menuWidth = -1;
@@ -57,6 +58,7 @@ public class DrawUserPair
UiSharedService uiSharedService,
PlayerPerformanceConfigService performanceConfigService,
LightlessConfigService configService,
LocationShareService locationShareService,
CharaDataManager charaDataManager,
PairLedger pairLedger)
{
@@ -74,6 +76,7 @@ public class DrawUserPair
_uiSharedService = uiSharedService;
_performanceConfigService = performanceConfigService;
_configService = configService;
_locationShareService = locationShareService;
_charaDataManager = charaDataManager;
_pairLedger = pairLedger;
}
@@ -216,6 +219,48 @@ public class DrawUserPair
_ = _apiController.UserSetPairPermissions(new UserPermissionsDto(_pair.UserData, permissions));
}
UiSharedService.AttachToolTip("Changes VFX sync permissions with this user." + (individual ? individualText : string.Empty));
ImGui.SetCursorPosX(10f);
_uiSharedService.IconText(FontAwesomeIcon.Globe);
ImGui.SameLine();
if (ImGui.BeginMenu("Toggle Location sharing"))
{
if (ImGui.MenuItem("Share for 30 Mins"))
{
_ = ToggleLocationSharing([_pair.UserData.UID], DateTimeOffset.UtcNow.AddMinutes(30));
}
if (ImGui.MenuItem("Share for 1 Hour"))
{
_ = ToggleLocationSharing([_pair.UserData.UID], DateTimeOffset.UtcNow.AddHours(1));
}
if (ImGui.MenuItem("Share for 3 Hours"))
{
_ = ToggleLocationSharing([_pair.UserData.UID], DateTimeOffset.UtcNow.AddHours(3));
}
if (ImGui.MenuItem("Share until manually stop"))
{
_ = ToggleLocationSharing([_pair.UserData.UID], DateTimeOffset.MaxValue);
}
ImGui.Separator();
if (ImGui.MenuItem("Stop Sharing"))
{
_ = ToggleLocationSharing([_pair.UserData.UID], DateTimeOffset.MinValue);
}
ImGui.EndMenu();
}
}
private async Task ToggleLocationSharing(List<string> users, DateTimeOffset expireAt)
{
var updated = await _apiController.ToggleLocationSharing(new LocationSharingToggleDto(users, expireAt)).ConfigureAwait(false);
if (updated)
{
_locationShareService.UpdateSharingStatus(users, expireAt);
}
}
private void DrawIndividualMenu()
@@ -574,6 +619,71 @@ public class DrawUserPair
var individualVFXDisabled = (_pair.UserPair?.OwnPermissions.IsDisableVFX() ?? false) || (_pair.UserPair?.OtherPermissions.IsDisableVFX() ?? false);
var individualIsSticky = _pair.UserPair!.OwnPermissions.IsSticky();
var individualIcon = individualIsSticky ? FontAwesomeIcon.ArrowCircleUp : FontAwesomeIcon.InfoCircle;
var shareLocationIcon = FontAwesomeIcon.Globe;
var location = _locationShareService.GetUserLocation(_pair.UserPair!.User.UID);
var shareLocation = !string.IsNullOrEmpty(location);
var expireAt = _locationShareService.GetSharingStatus(_pair.UserPair!.User.UID);
var shareLocationToOther = expireAt > DateTimeOffset.UtcNow;
var shareColor = shareLocation switch
{
true when shareLocationToOther => UIColors.Get("LightlessGreen"),
true when !shareLocationToOther => UIColors.Get("LightlessBlue"),
_ => UIColors.Get("LightlessYellow"),
};
if (shareLocation || shareLocationToOther)
{
currentRightSide -= (_uiSharedService.GetIconSize(shareLocationIcon).X + spacingX);
ImGui.SameLine(currentRightSide);
using (ImRaii.PushColor(ImGuiCol.Text, shareColor, shareLocation || shareLocationToOther))
_uiSharedService.IconText(shareLocationIcon);
if (ImGui.IsItemHovered())
{
ImGui.BeginTooltip();
if (_pair.IsOnline)
{
if (shareLocation)
{
if (!string.IsNullOrEmpty(location))
{
_uiSharedService.IconText(FontAwesomeIcon.LocationArrow);
ImGui.SameLine();
ImGui.TextUnformatted(location);
}
else
{
ImGui.TextUnformatted("Location info not updated, reconnect or wait for update.");
}
}
else
{
ImGui.TextUnformatted("NOT Sharing location with you. o(TヘTo)");
}
}
else
{
ImGui.TextUnformatted("User not online. (´・ω・`)?");
}
ImGui.Separator();
if (shareLocationToOther)
{
ImGui.TextUnformatted("Sharing your location. ヾ(•ω•`)o");
if (expireAt != DateTimeOffset.MaxValue)
{
ImGui.TextUnformatted("Expires at " + expireAt.ToLocalTime().ToString("g"));
}
}
else
{
ImGui.TextUnformatted("NOT sharing your location.  ̄へ ̄");
}
ImGui.EndTooltip();
}
}
if (individualAnimDisabled || individualSoundsDisabled || individualVFXDisabled || individualIsSticky)
{

View File

@@ -2183,7 +2183,7 @@ public class DataAnalysisUi : WindowMediatorSubscriberBase
bool toggleClicked = false;
if (showToggle)
{
var icon = isCollapsed ? FontAwesomeIcon.ChevronRight : FontAwesomeIcon.ChevronLeft;
var icon = !isCollapsed ? FontAwesomeIcon.ChevronRight : FontAwesomeIcon.ChevronLeft;
Vector2 iconSize;
using (_uiSharedService.IconFont.Push())
{

View File

@@ -25,6 +25,8 @@ public class DownloadUi : WindowMediatorSubscriberBase
private readonly ConcurrentDictionary<GameObjectHandler, bool> _uploadingPlayers = new();
private readonly Dictionary<GameObjectHandler, Vector2> _smoothed = [];
private readonly Dictionary<GameObjectHandler, DownloadSpeedTracker> _downloadSpeeds = [];
private readonly Dictionary<GameObjectHandler, (int TotalFiles, long TotalBytes)> _downloadInitialTotals = [];
private byte _transferBoxTransparency = 100;
private bool _notificationDismissed = true;
@@ -66,6 +68,10 @@ public class DownloadUi : WindowMediatorSubscriberBase
Mediator.Subscribe<DownloadStartedMessage>(this, (msg) =>
{
_currentDownloads[msg.DownloadId] = msg.DownloadStatus;
// Capture initial totals when download starts
var totalFiles = msg.DownloadStatus.Values.Sum(s => s.TotalFiles);
var totalBytes = msg.DownloadStatus.Values.Sum(s => s.TotalBytes);
_downloadInitialTotals[msg.DownloadId] = (totalFiles, totalBytes);
_notificationDismissed = false;
});
Mediator.Subscribe<DownloadFinishedMessage>(this, (msg) =>
@@ -167,7 +173,7 @@ public class DownloadUi : WindowMediatorSubscriberBase
List<KeyValuePair<GameObjectHandler, Dictionary<string, FileDownloadStatus>>> transfers;
try
{
transfers = _currentDownloads.ToList();
transfers = [.. _currentDownloads];
}
catch (ArgumentException)
{
@@ -435,9 +441,13 @@ public class DownloadUi : WindowMediatorSubscriberBase
var handler = transfer.Key;
var statuses = transfer.Value.Values;
var playerTotalFiles = statuses.Sum(s => s.TotalFiles);
var playerTransferredFiles = statuses.Sum(s => s.TransferredFiles);
var playerTotalBytes = statuses.Sum(s => s.TotalBytes);
var (playerTotalFiles, playerTotalBytes) = _downloadInitialTotals.TryGetValue(handler, out var totals)
? totals
: (statuses.Sum(s => s.TotalFiles), statuses.Sum(s => s.TotalBytes));
var playerTransferredFiles = statuses.Count(s =>
s.DownloadStatus == DownloadStatus.Decompressing ||
s.TransferredBytes >= s.TotalBytes);
var playerTransferredBytes = statuses.Sum(s => s.TransferredBytes);
totalFiles += playerTotalFiles;

View File

@@ -29,6 +29,7 @@ public class DrawEntityFactory
private readonly LightlessConfigService _configService;
private readonly UiSharedService _uiSharedService;
private readonly PlayerPerformanceConfigService _playerPerformanceConfigService;
private readonly LocationShareService _locationShareService;
private readonly CharaDataManager _charaDataManager;
private readonly SelectTagForPairUi _selectTagForPairUi;
private readonly RenamePairTagUi _renamePairTagUi;
@@ -53,6 +54,7 @@ public class DrawEntityFactory
LightlessConfigService configService,
UiSharedService uiSharedService,
PlayerPerformanceConfigService playerPerformanceConfigService,
LocationShareService locationShareService,
CharaDataManager charaDataManager,
SelectTagForSyncshellUi selectTagForSyncshellUi,
RenameSyncshellTagUi renameSyncshellTagUi,
@@ -72,6 +74,7 @@ public class DrawEntityFactory
_configService = configService;
_uiSharedService = uiSharedService;
_playerPerformanceConfigService = playerPerformanceConfigService;
_locationShareService = locationShareService;
_charaDataManager = charaDataManager;
_selectTagForSyncshellUi = selectTagForSyncshellUi;
_renameSyncshellTagUi = renameSyncshellTagUi;
@@ -162,6 +165,7 @@ public class DrawEntityFactory
_uiSharedService,
_playerPerformanceConfigService,
_configService,
_locationShareService,
_charaDataManager,
_pairLedger);
}

View File

@@ -14,6 +14,7 @@ using LightlessSync.Interop.Ipc;
using LightlessSync.LightlessConfiguration;
using LightlessSync.LightlessConfiguration.Configurations;
using LightlessSync.LightlessConfiguration.Models;
using LightlessSync.PlayerData.Factories;
using LightlessSync.PlayerData.Handlers;
using LightlessSync.PlayerData.Pairs;
using LightlessSync.Services;
@@ -40,6 +41,7 @@ using System.Globalization;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Numerics;
using System.Runtime.InteropServices;
using System.Text;
using System.Text.Json;
@@ -105,8 +107,8 @@ public class SettingsUi : WindowMediatorSubscriberBase
};
private readonly UiSharedService.TabOption<TransferSettingsTab>[] _transferTabOptions = new UiSharedService.TabOption<TransferSettingsTab>[2];
private readonly List<UiSharedService.TabOption<ServerSettingsTab>> _serverTabOptions = new(4);
private readonly string[] _generalTreeNavOrder = new[]
{
private readonly string[] _generalTreeNavOrder =
[
"Import & Export",
"Popup & Auto Fill",
"Behavior",
@@ -116,7 +118,8 @@ public class SettingsUi : WindowMediatorSubscriberBase
"Colors",
"Server Info Bar",
"Nameplate",
};
"Animation & Bones"
];
private static readonly HashSet<string> _generalNavSeparatorAfter = new(StringComparer.Ordinal)
{
"Popup & Auto Fill",
@@ -1141,7 +1144,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
private async Task<List<string>?> RunSpeedTest(List<string> servers, CancellationToken token)
{
List<string> speedTestResults = new();
List<string> speedTestResults = [];
foreach (var server in servers)
{
HttpResponseMessage? result = null;
@@ -1925,14 +1928,25 @@ public class SettingsUi : WindowMediatorSubscriberBase
{
using (ImRaii.PushIndent(20f))
{
if (_validationTask.IsCompleted)
if (_validationTask.IsCompletedSuccessfully)
{
UiSharedService.TextWrapped(
$"The storage validation has completed and removed {_validationTask.Result.Count} invalid files from storage.");
}
else if (_validationTask.IsCanceled)
{
UiSharedService.ColorTextWrapped(
"Storage validation was cancelled.",
UIColors.Get("LightlessYellow"));
}
else if (_validationTask.IsFaulted)
{
UiSharedService.ColorTextWrapped(
"Storage validation failed with an error.",
UIColors.Get("DimRed"));
}
else
{
UiSharedService.TextWrapped(
$"Storage validation is running: {_currentProgress.Item1}/{_currentProgress.Item2}");
if (_currentProgress.Item3 != null)
@@ -3074,10 +3088,102 @@ public class SettingsUi : WindowMediatorSubscriberBase
}
ImGui.Separator();
ImGui.Dummy(new Vector2(10));
_uiShared.BigText("Animation");
using (var animationTree = BeginGeneralTree("Animation & Bones", UIColors.Get("LightlessPurple")))
{
if (animationTree.Visible)
{
ImGui.TextUnformatted("Animation Options");
var modes = new[]
{
AnimationValidationMode.Unsafe,
AnimationValidationMode.Safe,
AnimationValidationMode.Safest,
};
var labels = new[]
{
"Unsafe",
"Safe (Race)",
"Safest (Race + Bones)",
};
var tooltips = new[]
{
"No validation. Fastest, but may allow incompatible animations (riskier).",
"Validates skeleton race + modded skeleton check (recommended).",
"Requires matching skeleton race + bone compatibility (strictest).",
};
var currentMode = _configService.Current.AnimationValidationMode;
int selectedIndex = Array.IndexOf(modes, currentMode);
if (selectedIndex < 0) selectedIndex = 1;
ImGui.SetNextItemWidth(140 * ImGuiHelpers.GlobalScale);
bool open = ImGui.BeginCombo("Animation validation", labels[selectedIndex]);
if (ImGui.IsItemHovered())
ImGui.SetTooltip(tooltips[selectedIndex]);
if (open)
{
for (int i = 0; i < modes.Length; i++)
{
bool isSelected = (i == selectedIndex);
if (ImGui.Selectable(labels[i], isSelected))
{
selectedIndex = i;
_configService.Current.AnimationValidationMode = modes[i];
_configService.Save();
}
if (ImGui.IsItemHovered())
ImGui.SetTooltip(tooltips[i]);
if (isSelected)
ImGui.SetItemDefaultFocus();
}
ImGui.EndCombo();
}
UiSharedService.ColoredSeparator(UIColors.Get("LightlessPurple"), 1.5f);
var cfg = _configService.Current;
bool oneBased = cfg.AnimationAllowOneBasedShift;
if (ImGui.Checkbox("Treat 1-based PAP indices as compatible", ref oneBased))
{
cfg.AnimationAllowOneBasedShift = oneBased;
_configService.Save();
}
if (ImGui.IsItemHovered())
ImGui.SetTooltip("Fixes off-by-one PAPs (one bone differance in bones and PAP). Can also increase crashing, toggle off if alot of crashing is happening");
bool neighbor = cfg.AnimationAllowNeighborIndexTolerance;
if (ImGui.Checkbox("Allow 1+- bone index tolerance", ref neighbor))
{
cfg.AnimationAllowNeighborIndexTolerance = neighbor;
_configService.Save();
}
if (ImGui.IsItemHovered())
ImGui.SetTooltip("Looser matching on bone matching. Can reduce false blocks happening but also reduces safety and more prone to crashing.");
ImGui.TreePop();
animationTree.MarkContentEnd();
}
}
ImGui.EndChild();
ImGui.EndGroup();
ImGui.Separator();
generalSelune.DrawHighlightOnly(ImGui.GetIO().DeltaTime);
}
}
@@ -3167,6 +3273,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
return 1f - (elapsed / GeneralTreeHighlightDuration);
}
[StructLayout(LayoutKind.Auto)]
private struct GeneralTreeScope : IDisposable
{
private readonly bool _visible;
@@ -3474,7 +3581,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
_uiShared.DrawHelpText("Controls whether Lightless reduces index textures that exceed the size limit.");
var dimensionOptions = new[] { 512, 1024, 2048, 4096 };
var optionLabels = dimensionOptions.Select(static value => value.ToString()).ToArray();
var optionLabels = dimensionOptions.Select(selector: static value => value.ToString()).ToArray();
var currentDimension = textureConfig.TextureDownscaleMaxDimension;
var selectedIndex = Array.IndexOf(dimensionOptions, currentDimension);
if (selectedIndex < 0)

View File

@@ -30,6 +30,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
private readonly ConcurrentDictionary<ThrottledStream, byte> _activeDownloadStreams;
private readonly SemaphoreSlim _decompressGate =
new(Math.Max(1, Environment.ProcessorCount / 2), Math.Max(1, Environment.ProcessorCount / 2));
private readonly ConcurrentQueue<string> _deferredCompressionQueue = new();
private volatile bool _disableDirectDownloads;
private int _consecutiveDirectDownloadFailures;
@@ -404,76 +406,32 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
private async Task WaitForDownloadReady(List<DownloadFileTransfer> downloadFileTransfer, Guid requestId, CancellationToken downloadCt)
{
bool alreadyCancelled = false;
try
while (true)
{
CancellationTokenSource localTimeoutCts = new();
localTimeoutCts.CancelAfter(TimeSpan.FromSeconds(5));
CancellationTokenSource composite = CancellationTokenSource.CreateLinkedTokenSource(downloadCt, localTimeoutCts.Token);
downloadCt.ThrowIfCancellationRequested();
while (!_orchestrator.IsDownloadReady(requestId))
if (_orchestrator.IsDownloadReady(requestId))
break;
using var resp = await _orchestrator.SendRequestAsync(
HttpMethod.Get,
LightlessFiles.RequestCheckQueueFullPath(downloadFileTransfer[0].DownloadUri, requestId),
downloadFileTransfer.Select(t => t.Hash).ToList(),
downloadCt).ConfigureAwait(false);
resp.EnsureSuccessStatusCode();
var body = (await resp.Content.ReadAsStringAsync(downloadCt).ConfigureAwait(false)).Trim();
if (string.Equals(body, "true", StringComparison.OrdinalIgnoreCase) ||
body.Contains("\"ready\":true", StringComparison.OrdinalIgnoreCase))
{
try
{
await Task.Delay(250, composite.Token).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
if (downloadCt.IsCancellationRequested) throw;
var req = await _orchestrator.SendRequestAsync(
HttpMethod.Get,
LightlessFiles.RequestCheckQueueFullPath(downloadFileTransfer[0].DownloadUri, requestId),
downloadFileTransfer.Select(c => c.Hash).ToList(),
downloadCt).ConfigureAwait(false);
req.EnsureSuccessStatusCode();
localTimeoutCts.Dispose();
composite.Dispose();
localTimeoutCts = new();
localTimeoutCts.CancelAfter(TimeSpan.FromSeconds(5));
composite = CancellationTokenSource.CreateLinkedTokenSource(downloadCt, localTimeoutCts.Token);
}
break;
}
localTimeoutCts.Dispose();
composite.Dispose();
Logger.LogDebug("Download {requestId} ready", requestId);
await Task.Delay(250, downloadCt).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
try
{
await _orchestrator.SendRequestAsync(HttpMethod.Get, LightlessFiles.RequestCancelFullPath(downloadFileTransfer[0].DownloadUri, requestId))
.ConfigureAwait(false);
alreadyCancelled = true;
}
catch
{
// ignore
}
throw;
}
finally
{
if (downloadCt.IsCancellationRequested && !alreadyCancelled)
{
try
{
await _orchestrator.SendRequestAsync(HttpMethod.Get, LightlessFiles.RequestCancelFullPath(downloadFileTransfer[0].DownloadUri, requestId))
.ConfigureAwait(false);
}
catch
{
// ignore
}
}
_orchestrator.ClearDownloadRequest(requestId);
}
_orchestrator.ClearDownloadRequest(requestId);
}
private async Task DownloadQueuedBlockFileAsync(
@@ -532,11 +490,9 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
try
{
// sanity check length
if (fileLengthBytes < 0 || fileLengthBytes > int.MaxValue)
throw new InvalidDataException($"Invalid block entry length: {fileLengthBytes}");
// safe cast after check
var len = checked((int)fileLengthBytes);
if (!replacementLookup.TryGetValue(fileHash, out var repl))
@@ -546,38 +502,39 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
continue;
}
// decompress
var filePath = _fileDbManager.GetCacheFilePath(fileHash, repl.Extension);
Logger.LogTrace("{dlName}: Decompressing {file}:{len} => {dest}", downloadLabel, fileHash, fileLengthBytes, filePath);
// read compressed data
var compressed = new byte[len];
await ReadExactlyAsync(fileBlockStream, compressed.AsMemory(0, len), ct).ConfigureAwait(false);
if (len == 0)
{
await _fileCompactor.WriteAllBytesAsync(filePath, Array.Empty<byte>(), ct).ConfigureAwait(false);
await File.WriteAllBytesAsync(filePath, Array.Empty<byte>(), ct).ConfigureAwait(false);
PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale);
continue;
}
MungeBuffer(compressed);
// limit concurrent decompressions
await _decompressGate.WaitAsync(ct).ConfigureAwait(false);
try
{
var sw = System.Diagnostics.Stopwatch.StartNew();
// offload CPU-intensive decompression to threadpool to free up worker
await Task.Run(async () =>
{
var sw = System.Diagnostics.Stopwatch.StartNew();
// decompress
var decompressed = LZ4Wrapper.Unwrap(compressed);
// decompress
var decompressed = LZ4Wrapper.Unwrap(compressed);
Logger.LogTrace("{dlName}: Unwrap {fileHash} took {ms}ms (compressed {c} bytes, decompressed {d} bytes)",
downloadLabel, fileHash, sw.ElapsedMilliseconds, compressed.Length, decompressed?.Length ?? -1);
Logger.LogTrace("{dlName}: Unwrap {fileHash} took {ms}ms (compressed {c} bytes, decompressed {d} bytes)",
downloadLabel, fileHash, sw.ElapsedMilliseconds, compressed.Length, decompressed?.Length ?? -1);
// write to file
await _fileCompactor.WriteAllBytesAsync(filePath, decompressed, ct).ConfigureAwait(false);
PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale);
// write to file without compacting during download
await File.WriteAllBytesAsync(filePath, decompressed, ct).ConfigureAwait(false);
PersistFileToStorage(fileHash, filePath, repl.GamePath, skipDownscale);
}, ct).ConfigureAwait(false);
}
finally
{
@@ -752,8 +709,16 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
if (gameObjectHandler is not null)
Mediator.Publish(new DownloadStartedMessage(gameObjectHandler, _downloadStatus));
// work based on cpu count and slots
var coreCount = Environment.ProcessorCount;
var baseWorkers = Math.Min(slots, coreCount);
// only add buffer if decompression has capacity AND we have cores to spare
var availableDecompressSlots = _decompressGate.CurrentCount;
var extraWorkers = (availableDecompressSlots > 0 && coreCount >= 6) ? 2 : 0;
// allow some extra workers so downloads can continue while earlier items decompress.
var workerDop = Math.Clamp(slots * 2, 2, 16);
var workerDop = Math.Clamp(baseWorkers + extraWorkers, 2, coreCount);
// batch downloads
Task batchTask = batchChunks.Length == 0
@@ -769,6 +734,9 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
await Task.WhenAll(batchTask, directTask).ConfigureAwait(false);
// process deferred compressions after all downloads complete
await ProcessDeferredCompressionsAsync(ct).ConfigureAwait(false);
Logger.LogDebug("Download end: {id}", objectName);
ClearDownload();
}
@@ -793,7 +761,6 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
try
{
// download (with slot)
var progress = CreateInlineProgress(bytes => AddTransferredBytes(statusKey, bytes));
// Download slot held on get
@@ -873,7 +840,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
byte[] compressedBytes = await File.ReadAllBytesAsync(tempFilename, ct).ConfigureAwait(false);
var decompressedBytes = LZ4Wrapper.Unwrap(compressedBytes);
await _fileCompactor.WriteAllBytesAsync(finalFilename, decompressedBytes, ct).ConfigureAwait(false);
await File.WriteAllBytesAsync(finalFilename, decompressedBytes, ct).ConfigureAwait(false);
PersistFileToStorage(directDownload.Hash, finalFilename, repl.GamePath, skipDownscale);
MarkTransferredFiles(directDownload.DirectDownloadUrl!, 1);
@@ -974,14 +941,12 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
if (!_orchestrator.IsInitialized)
throw new InvalidOperationException("FileTransferManager is not initialized");
// batch request
var response = await _orchestrator.SendRequestAsync(
HttpMethod.Get,
LightlessFiles.ServerFilesGetSizesFullPath(_orchestrator.FilesCdnUri!),
hashes,
ct).ConfigureAwait(false);
// ensure success
return await response.Content.ReadFromJsonAsync<List<DownloadFileDto>>(cancellationToken: ct).ConfigureAwait(false) ?? [];
}
@@ -1001,6 +966,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
fi.LastAccessTime = DateTime.Today;
fi.LastWriteTime = RandomDayInThePast().Invoke();
// queue file for deferred compression instead of compressing immediately
if (_configService.Current.UseCompactor)
_deferredCompressionQueue.Enqueue(filePath);
try
{
var entry = _fileDbManager.CreateCacheEntry(filePath);
@@ -1026,6 +995,52 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
private static IProgress<long> CreateInlineProgress(Action<long> callback) => new InlineProgress(callback);
private async Task ProcessDeferredCompressionsAsync(CancellationToken ct)
{
if (_deferredCompressionQueue.IsEmpty)
return;
var filesToCompress = new List<string>();
while (_deferredCompressionQueue.TryDequeue(out var filePath))
{
if (File.Exists(filePath))
filesToCompress.Add(filePath);
}
if (filesToCompress.Count == 0)
return;
Logger.LogDebug("Starting deferred compression of {count} files", filesToCompress.Count);
var compressionWorkers = Math.Clamp(Environment.ProcessorCount / 4, 2, 4);
await Parallel.ForEachAsync(filesToCompress,
new ParallelOptions
{
MaxDegreeOfParallelism = compressionWorkers,
CancellationToken = ct
},
async (filePath, token) =>
{
try
{
await Task.Yield();
if (_configService.Current.UseCompactor && File.Exists(filePath))
{
var bytes = await File.ReadAllBytesAsync(filePath, token).ConfigureAwait(false);
await _fileCompactor.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false);
Logger.LogTrace("Compressed file: {filePath}", filePath);
}
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Failed to compress file: {filePath}", filePath);
}
}).ConfigureAwait(false);
Logger.LogDebug("Completed deferred compression of {count} files", filesToCompress.Count);
}
private sealed class InlineProgress : IProgress<long>
{
private readonly Action<long> _callback;

View File

@@ -200,5 +200,21 @@ public partial class ApiController
await UserPushData(new(visibleCharacters, character, censusDto)).ConfigureAwait(false);
}
public async Task UpdateLocation(LocationDto locationDto, bool offline = false)
{
if (!IsConnected) return;
await _lightlessHub!.SendAsync(nameof(UpdateLocation), locationDto, offline).ConfigureAwait(false);
}
public async Task<(List<LocationWithTimeDto>, List<SharingStatusDto>)> RequestAllLocationInfo()
{
if (!IsConnected) return ([],[]);
return await _lightlessHub!.InvokeAsync<(List<LocationWithTimeDto>, List<SharingStatusDto>)>(nameof(RequestAllLocationInfo)).ConfigureAwait(false);
}
public async Task<bool> ToggleLocationSharing(LocationSharingToggleDto dto)
{
if (!IsConnected) return false;
return await _lightlessHub!.InvokeAsync<bool>(nameof(ToggleLocationSharing), dto).ConfigureAwait(false);
}
}
#pragma warning restore MA0040

View File

@@ -259,6 +259,13 @@ public partial class ApiController
ExecuteSafely(() => Mediator.Publish(new GPoseLobbyReceiveWorldData(userData, worldData)));
return Task.CompletedTask;
}
public Task Client_SendLocationToClient(LocationDto locationDto, DateTimeOffset expireAt)
{
Logger.LogDebug($"{nameof(Client_SendLocationToClient)}: {locationDto.User} {expireAt}");
ExecuteSafely(() => Mediator.Publish(new LocationSharingMessage(locationDto.User, locationDto.Location, expireAt)));
return Task.CompletedTask;
}
public void OnDownloadReady(Action<Guid> act)
{
@@ -441,6 +448,12 @@ public partial class ApiController
_lightlessHub!.On(nameof(Client_GposeLobbyPushWorldData), act);
}
public void OnReceiveLocation(Action<LocationDto, DateTimeOffset> act)
{
if (_initialized) return;
_lightlessHub!.On(nameof(Client_SendLocationToClient), act);
}
private void ExecuteSafely(Action act)
{
try

View File

@@ -606,6 +606,7 @@ public sealed partial class ApiController : DisposableMediatorSubscriberBase, IL
OnGposeLobbyPushCharacterData((dto) => _ = Client_GposeLobbyPushCharacterData(dto));
OnGposeLobbyPushPoseData((dto, data) => _ = Client_GposeLobbyPushPoseData(dto, data));
OnGposeLobbyPushWorldData((dto, data) => _ = Client_GposeLobbyPushWorldData(dto, data));
OnReceiveLocation((dto, time) => _ = Client_SendLocationToClient(dto, time));
_healthCheckTokenSource?.Cancel();
_healthCheckTokenSource?.Dispose();
@@ -774,5 +775,6 @@ public sealed partial class ApiController : DisposableMediatorSubscriberBase, IL
ServerState = state;
}
}
#pragma warning restore MA0040

View File

@@ -76,6 +76,19 @@
"Microsoft.AspNetCore.SignalR.Common": "10.0.1"
}
},
"Microsoft.Extensions.Caching.Memory": {
"type": "Direct",
"requested": "[10.0.1, )",
"resolved": "10.0.1",
"contentHash": "NxqSP0Ky4dZ5ybszdZCqs1X2C70s+dXflqhYBUh/vhcQVTIooNCXIYnLVbafoAFGZMs51d9+rHxveXs0ZC3SQQ==",
"dependencies": {
"Microsoft.Extensions.Caching.Abstractions": "10.0.1",
"Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.1",
"Microsoft.Extensions.Logging.Abstractions": "10.0.1",
"Microsoft.Extensions.Options": "10.0.1",
"Microsoft.Extensions.Primitives": "10.0.1"
}
},
"Microsoft.Extensions.Hosting": {
"type": "Direct",
"requested": "[10.0.1, )",
@@ -233,6 +246,14 @@
"Microsoft.AspNetCore.SignalR.Common": "10.0.1"
}
},
"Microsoft.Extensions.Caching.Abstractions": {
"type": "Transitive",
"resolved": "10.0.1",
"contentHash": "Vb1vVAQDxHpXVdL9fpOX2BzeV7bbhzG4pAcIKRauRl0/VfkE8mq0f+fYC+gWICh3dlzTZInJ/cTeBS2MgU/XvQ==",
"dependencies": {
"Microsoft.Extensions.Primitives": "10.0.1"
}
},
"Microsoft.Extensions.Configuration": {
"type": "Transitive",
"resolved": "10.0.1",
@@ -618,7 +639,7 @@
"FlatSharp.Compiler": "[7.9.0, )",
"FlatSharp.Runtime": "[7.9.0, )",
"OtterGui": "[1.0.0, )",
"Penumbra.Api": "[5.13.0, )",
"Penumbra.Api": "[5.13.1, )",
"Penumbra.String": "[1.0.7, )"
}
},