highly experimental runtime model decimation + file cache adjustment to clean up processed file copies

LightlessSync/Services/ModelDecimation/ModelDecimationService.cs

@@ -0,0 +1,275 @@
+using LightlessSync.LightlessConfiguration;
+using Microsoft.Extensions.Logging;
+using System.Collections.Concurrent;
+
+namespace LightlessSync.Services.ModelDecimation;
+
+public sealed class ModelDecimationService
+{
+    private const int MaxConcurrentJobs = 1;
+    private const double MinTargetRatio = 0.01;
+    private const double MaxTargetRatio = 0.99;
+
+    private readonly ILogger<ModelDecimationService> _logger;
+    private readonly LightlessConfigService _configService;
+    private readonly PlayerPerformanceConfigService _performanceConfigService;
+    private readonly XivDataStorageService _xivDataStorageService;
+    private readonly SemaphoreSlim _decimationSemaphore = new(MaxConcurrentJobs);
+
+    private readonly ConcurrentDictionary<string, Task> _activeJobs = new(StringComparer.OrdinalIgnoreCase);
+    private readonly ConcurrentDictionary<string, string> _decimatedPaths = new(StringComparer.OrdinalIgnoreCase);
+    private readonly ConcurrentDictionary<string, byte> _failedHashes = new(StringComparer.OrdinalIgnoreCase);
+
+    public ModelDecimationService(
+        ILogger<ModelDecimationService> logger,
+        LightlessConfigService configService,
+        PlayerPerformanceConfigService performanceConfigService,
+        XivDataStorageService xivDataStorageService)
+    {
+        _logger = logger;
+        _configService = configService;
+        _performanceConfigService = performanceConfigService;
+        _xivDataStorageService = xivDataStorageService;
+    }
+
+    public void ScheduleDecimation(string hash, string filePath, string? gamePath = null)
+    {
+        if (!ShouldScheduleDecimation(hash, filePath, gamePath))
+        {
+            return;
+        }
+
+        if (_decimatedPaths.ContainsKey(hash) || _failedHashes.ContainsKey(hash) || _activeJobs.ContainsKey(hash))
+        {
+            return;
+        }
+
+        _logger.LogInformation("Queued model decimation for {Hash}", hash);
+
+        _activeJobs[hash] = Task.Run(async () =>
+        {
+            await _decimationSemaphore.WaitAsync().ConfigureAwait(false);
+            try
+            {
+                await DecimateInternalAsync(hash, filePath).ConfigureAwait(false);
+            }
+            catch (Exception ex)
+            {
+                _failedHashes[hash] = 1;
+                _logger.LogWarning(ex, "Model decimation failed for {Hash}", hash);
+            }
+            finally
+            {
+                _decimationSemaphore.Release();
+                _activeJobs.TryRemove(hash, out _);
+            }
+        }, CancellationToken.None);
+    }
+
+    public bool ShouldScheduleDecimation(string hash, string filePath, string? gamePath = null)
+        => IsDecimationEnabled()
+            && filePath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase)
+            && IsDecimationAllowed(gamePath)
+            && !ShouldSkipByTriangleCache(hash);
+
+    public string GetPreferredPath(string hash, string originalPath)
+    {
+        if (!IsDecimationEnabled())
+        {
+            return originalPath;
+        }
+
+        if (_decimatedPaths.TryGetValue(hash, out var existing) && File.Exists(existing))
+        {
+            return existing;
+        }
+
+        var resolved = GetExistingDecimatedPath(hash);
+        if (!string.IsNullOrEmpty(resolved))
+        {
+            _decimatedPaths[hash] = resolved;
+            return resolved;
+        }
+
+        return originalPath;
+    }
+
+    public Task WaitForPendingJobsAsync(IEnumerable<string>? hashes, CancellationToken token)
+    {
+        if (hashes is null)
+        {
+            return Task.CompletedTask;
+        }
+
+        var pending = new List<Task>();
+        var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+        foreach (var hash in hashes)
+        {
+            if (string.IsNullOrEmpty(hash) || !seen.Add(hash))
+            {
+                continue;
+            }
+
+            if (_activeJobs.TryGetValue(hash, out var job))
+            {
+                pending.Add(job);
+            }
+        }
+
+        if (pending.Count == 0)
+        {
+            return Task.CompletedTask;
+        }
+
+        return Task.WhenAll(pending).WaitAsync(token);
+    }
+
+    private Task DecimateInternalAsync(string hash, string sourcePath)
+    {
+        if (!File.Exists(sourcePath))
+        {
+            _failedHashes[hash] = 1;
+            _logger.LogWarning("Cannot decimate model {Hash}; source path missing: {Path}", hash, sourcePath);
+            return Task.CompletedTask;
+        }
+
+        if (!TryGetDecimationSettings(out var triangleThreshold, out var targetRatio))
+        {
+            _logger.LogInformation("Model decimation disabled or invalid settings for {Hash}", hash);
+            return Task.CompletedTask;
+        }
+
+        _logger.LogInformation("Starting model decimation for {Hash} (threshold {Threshold}, ratio {Ratio:0.##})", hash, triangleThreshold, targetRatio);
+
+        var destination = Path.Combine(GetDecimatedDirectory(), $"{hash}.mdl");
+        if (File.Exists(destination))
+        {
+            _decimatedPaths[hash] = destination;
+            return Task.CompletedTask;
+        }
+
+        if (!MdlDecimator.TryDecimate(sourcePath, destination, triangleThreshold, targetRatio, _logger))
+        {
+            _failedHashes[hash] = 1;
+            _logger.LogInformation("Model decimation skipped for {Hash}", hash);
+            return Task.CompletedTask;
+        }
+
+        _decimatedPaths[hash] = destination;
+        _logger.LogInformation("Decimated model {Hash} -> {Path}", hash, destination);
+        return Task.CompletedTask;
+    }
+
+    private bool IsDecimationEnabled()
+        => _performanceConfigService.Current.EnableModelDecimation;
+
+    private bool ShouldSkipByTriangleCache(string hash)
+    {
+        if (string.IsNullOrEmpty(hash))
+        {
+            return false;
+        }
+
+        if (!_xivDataStorageService.Current.TriangleDictionary.TryGetValue(hash, out var cachedTris) || cachedTris <= 0)
+        {
+            return false;
+        }
+
+        var threshold = Math.Max(0, _performanceConfigService.Current.ModelDecimationTriangleThreshold);
+        return threshold > 0 && cachedTris < threshold;
+    }
+
+    private bool IsDecimationAllowed(string? gamePath)
+    {
+        if (string.IsNullOrWhiteSpace(gamePath))
+        {
+            return true;
+        }
+
+        var normalized = NormalizeGamePath(gamePath);
+        if (normalized.Contains("/hair/", StringComparison.Ordinal))
+        {
+            return false;
+        }
+
+        if (normalized.Contains("/chara/equipment/", StringComparison.Ordinal))
+        {
+            return _performanceConfigService.Current.ModelDecimationAllowClothing;
+        }
+
+        if (normalized.Contains("/chara/accessory/", StringComparison.Ordinal))
+        {
+            return _performanceConfigService.Current.ModelDecimationAllowAccessories;
+        }
+
+        if (normalized.Contains("/chara/human/", StringComparison.Ordinal))
+        {
+            if (normalized.Contains("/body/", StringComparison.Ordinal))
+            {
+                return _performanceConfigService.Current.ModelDecimationAllowBody;
+            }
+
+            if (normalized.Contains("/face/", StringComparison.Ordinal) || normalized.Contains("/head/", StringComparison.Ordinal))
+            {
+                return _performanceConfigService.Current.ModelDecimationAllowFaceHead;
+            }
+
+            if (normalized.Contains("/tail/", StringComparison.Ordinal))
+            {
+                return _performanceConfigService.Current.ModelDecimationAllowTail;
+            }
+        }
+
+        return true;
+    }
+
+    private static string NormalizeGamePath(string path)
+        => path.Replace('\\', '/').ToLowerInvariant();
+
+    private bool TryGetDecimationSettings(out int triangleThreshold, out double targetRatio)
+    {
+        triangleThreshold = 15_000;
+        targetRatio = 0.8;
+
+        var config = _performanceConfigService.Current;
+        if (!config.EnableModelDecimation)
+        {
+            return false;
+        }
+
+        triangleThreshold = Math.Max(0, config.ModelDecimationTriangleThreshold);
+        targetRatio = config.ModelDecimationTargetRatio;
+        if (double.IsNaN(targetRatio) || double.IsInfinity(targetRatio))
+        {
+            return false;
+        }
+
+        targetRatio = Math.Clamp(targetRatio, MinTargetRatio, MaxTargetRatio);
+        return true;
+    }
+
+    private string? GetExistingDecimatedPath(string hash)
+    {
+        var candidate = Path.Combine(GetDecimatedDirectory(), $"{hash}.mdl");
+        return File.Exists(candidate) ? candidate : null;
+    }
+
+    private string GetDecimatedDirectory()
+    {
+        var directory = Path.Combine(_configService.Current.CacheFolder, "decimated");
+        if (!Directory.Exists(directory))
+        {
+            try
+            {
+                Directory.CreateDirectory(directory);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogTrace(ex, "Failed to create decimated directory {Directory}", directory);
+            }
+        }
+
+        return directory;
+    }
+}

LightlessSync/Services/PlayerPerformanceService.cs

@@ -4,6 +4,7 @@ using LightlessSync.LightlessConfiguration;
 using LightlessSync.PlayerData.Pairs;
 using LightlessSync.Services.Events;
 using LightlessSync.Services.Mediator;
+using LightlessSync.Services.ModelDecimation;
 using LightlessSync.Services.TextureCompression;
 using LightlessSync.UI;
 using LightlessSync.WebAPI.Files.Models;
@@ -18,12 +19,14 @@ public class PlayerPerformanceService
     private readonly ILogger<PlayerPerformanceService> _logger;
     private readonly LightlessMediator _mediator;
     private readonly PlayerPerformanceConfigService _playerPerformanceConfigService;
+    private readonly ModelDecimationService _modelDecimationService;
     private readonly TextureDownscaleService _textureDownscaleService;
     private readonly Dictionary<string, bool> _warnedForPlayers = new(StringComparer.Ordinal);
 
     public PlayerPerformanceService(ILogger<PlayerPerformanceService> logger, LightlessMediator mediator,
         PlayerPerformanceConfigService playerPerformanceConfigService, FileCacheManager fileCacheManager,
-        XivDataAnalyzer xivDataAnalyzer, TextureDownscaleService textureDownscaleService)
+        XivDataAnalyzer xivDataAnalyzer, TextureDownscaleService textureDownscaleService,
+        ModelDecimationService modelDecimationService)
     {
         _logger = logger;
         _mediator = mediator;
@@ -31,6 +34,7 @@ public class PlayerPerformanceService
         _fileCacheManager = fileCacheManager;
         _xivDataAnalyzer = xivDataAnalyzer;
         _textureDownscaleService = textureDownscaleService;
+        _modelDecimationService = modelDecimationService;
     }
 
     public async Task<bool> CheckBothThresholds(IPairPerformanceSubject pairHandler, CharacterData charaData)
@@ -111,10 +115,12 @@ public class PlayerPerformanceService
         var config = _playerPerformanceConfigService.Current;
 
         long triUsage = 0;
+        long effectiveTriUsage = 0;
 
         if (!charaData.FileReplacements.TryGetValue(API.Data.Enum.ObjectKind.Player, out List<FileReplacementData>? playerReplacements))
         {
             pairHandler.LastAppliedDataTris = 0;
+            pairHandler.LastAppliedApproximateEffectiveTris = 0;
             return true;
         }
 
@@ -125,12 +131,31 @@ public class PlayerPerformanceService
 
         foreach (var hash in moddedModelHashes)
         {
-            triUsage += await _xivDataAnalyzer.GetTrianglesByHash(hash).ConfigureAwait(false);
+            var tris = await _xivDataAnalyzer.GetTrianglesByHash(hash).ConfigureAwait(false);
+            triUsage += tris;
+
+            long effectiveTris = tris;
+            var fileEntry = _fileCacheManager.GetFileCacheByHash(hash);
+            if (fileEntry != null)
+            {
+                var preferredPath = _modelDecimationService.GetPreferredPath(hash, fileEntry.ResolvedFilepath);
+                if (!string.Equals(preferredPath, fileEntry.ResolvedFilepath, StringComparison.OrdinalIgnoreCase))
+                {
+                    var decimatedTris = await _xivDataAnalyzer.GetEffectiveTrianglesByHash(hash, preferredPath).ConfigureAwait(false);
+                    if (decimatedTris > 0)
+                    {
+                        effectiveTris = decimatedTris;
+                    }
+                }
+            }
+
+            effectiveTriUsage += effectiveTris;
         }
 
         pairHandler.LastAppliedDataTris = triUsage;
+        pairHandler.LastAppliedApproximateEffectiveTris = effectiveTriUsage;
 
-        _logger.LogDebug("Calculated VRAM usage for {p}", pairHandler);
+        _logger.LogDebug("Calculated triangle usage for {p}", pairHandler);
 
         // no warning of any kind on ignored pairs
         if (config.UIDsToIgnore
@@ -274,4 +299,4 @@ public class PlayerPerformanceService
 
     private static bool CheckForThreshold(bool thresholdEnabled, long threshold, long value, bool checkForPrefPerm, bool isPrefPerm) =>
         thresholdEnabled && threshold > 0 && threshold < value && ((checkForPrefPerm && isPrefPerm) || !isPrefPerm);
-}
+}

LightlessSync/Services/TextureCompression/TextureDownscaleService.cs

@@ -678,7 +678,7 @@ public sealed class TextureDownscaleService
 
             if (!string.Equals(cacheEntry.PrefixedFilePath, prefixed, StringComparison.OrdinalIgnoreCase))
             {
-                _fileCacheManager.RemoveHashedFile(cacheEntry.Hash, cacheEntry.PrefixedFilePath);
+                _fileCacheManager.RemoveHashedFile(cacheEntry.Hash, cacheEntry.PrefixedFilePath, removeDerivedFiles: false);
             }
 
             _fileCacheManager.UpdateHashedFile(replacement, computeProperties: false);

LightlessSync/Services/XivDataAnalyzer.cs

@@ -8,6 +8,7 @@ using LightlessSync.Interop.GameModel;
 using LightlessSync.LightlessConfiguration;
 using LightlessSync.PlayerData.Handlers;
 using Microsoft.Extensions.Logging;
+using System.Collections.Concurrent;
 using System.Runtime.InteropServices;
 
 namespace LightlessSync.Services;
@@ -18,6 +19,7 @@ public sealed class XivDataAnalyzer
     private readonly FileCacheManager _fileCacheManager;
     private readonly XivDataStorageService _configService;
     private readonly List<string> _failedCalculatedTris = [];
+    private readonly List<string> _failedCalculatedEffectiveTris = [];
 
     public XivDataAnalyzer(ILogger<XivDataAnalyzer> logger, FileCacheManager fileCacheManager,
         XivDataStorageService configService)
@@ -162,16 +164,41 @@ public sealed class XivDataAnalyzer
         if (path == null || !path.ResolvedFilepath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase))
             return 0;
 
-        var filePath = path.ResolvedFilepath;
+        return CalculateTrianglesFromPath(hash, path.ResolvedFilepath, _configService.Current.TriangleDictionary, _failedCalculatedTris);
+    }
 
+    public async Task<long> GetEffectiveTrianglesByHash(string hash, string filePath)
+    {
+        if (_configService.Current.EffectiveTriangleDictionary.TryGetValue(hash, out var cachedTris) && cachedTris > 0)
+            return cachedTris;
+
+        if (_failedCalculatedEffectiveTris.Contains(hash, StringComparer.Ordinal))
+            return 0;
+
+        if (string.IsNullOrEmpty(filePath)
+            || !filePath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase)
+            || !File.Exists(filePath))
+        {
+            return 0;
+        }
+
+        return CalculateTrianglesFromPath(hash, filePath, _configService.Current.EffectiveTriangleDictionary, _failedCalculatedEffectiveTris);
+    }
+
+    private long CalculateTrianglesFromPath(
+        string hash,
+        string filePath,
+        ConcurrentDictionary<string, long> cache,
+        List<string> failedList)
+    {
         try
         {
             _logger.LogDebug("Detected Model File {path}, calculating Tris", filePath);
             var file = new MdlFile(filePath);
             if (file.LodCount <= 0)
             {
-                _failedCalculatedTris.Add(hash);
-                _configService.Current.TriangleDictionary[hash] = 0;
+                failedList.Add(hash);
+                cache[hash] = 0;
                 _configService.Save();
                 return 0;
             }
@@ -195,7 +222,7 @@ public sealed class XivDataAnalyzer
                 if (tris > 0)
                 {
                     _logger.LogDebug("TriAnalysis: {filePath} => {tris} triangles", filePath, tris);
-                    _configService.Current.TriangleDictionary[hash] = tris;
+                    cache[hash] = tris;
                     _configService.Save();
                     break;
                 }
@@ -205,8 +232,8 @@ public sealed class XivDataAnalyzer
         }
         catch (Exception e)
         {
-            _failedCalculatedTris.Add(hash);
-            _configService.Current.TriangleDictionary[hash] = 0;
+            failedList.Add(hash);
+            cache[hash] = 0;
             _configService.Save();
             _logger.LogWarning(e, "Could not parse file {file}", filePath);
             return 0;