From 92cb86171005b5d7d08eda2bfcdcd18f7e3c429c Mon Sep 17 00:00:00 2001 From: azyges Date: Fri, 2 Jan 2026 11:04:15 +0900 Subject: [PATCH] readjust cache clean up and add keep originals setting for models --- LightlessSync/FileCache/CacheMonitor.cs | 179 ++++++++++-------- .../Configurations/PlayerPerformanceConfig.cs | 1 + .../ModelDecimation/ModelDecimationService.cs | 110 ++++++++++- LightlessSync/UI/SettingsUi.cs | 10 + 4 files changed, 221 insertions(+), 79 deletions(-) diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 5143f26..165a58c 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -103,6 +103,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase } record WatcherChange(WatcherChangeTypes ChangeType, string? OldPath = null); + private readonly record struct CacheEvictionCandidate(string FullPath, long Size, DateTime LastAccessTime); private readonly Dictionary _watcherChanges = new(StringComparer.OrdinalIgnoreCase); private readonly Dictionary _lightlessChanges = new(StringComparer.OrdinalIgnoreCase); @@ -441,76 +442,40 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase Logger.LogWarning(ex, "Could not determine drive size for storage folder {folder}", _configService.Current.CacheFolder); } - var files = Directory.EnumerateFiles(_configService.Current.CacheFolder) - .Select(f => new FileInfo(f)) - .OrderBy(f => f.LastAccessTime) - .ToList(); - + var cacheFolder = _configService.Current.CacheFolder; + var candidates = new List(); long totalSize = 0; + totalSize += AddFolderCandidates(cacheFolder, candidates, token, isWine); + totalSize += AddFolderCandidates(Path.Combine(cacheFolder, "downscaled"), candidates, token, isWine); + totalSize += AddFolderCandidates(Path.Combine(cacheFolder, "decimated"), candidates, token, isWine); - foreach (var f in files) - { - token.ThrowIfCancellationRequested(); - - try - { - long size = 0; - - if (!isWine) - { - try - { - size = _fileCompactor.GetFileSizeOnDisk(f); - } - catch (Exception ex) - { - Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName); - size = f.Length; - } - } - else - { - size = f.Length; - } - - totalSize += size; - } - catch (Exception ex) - { - Logger.LogTrace(ex, "Error getting size for {file}", f.FullName); - } - } - - long totalSizeExtras = 0; FileCacheSize = totalSize; - totalSizeExtras += GetFolderSize(Path.Combine(_configService.Current.CacheFolder, "downscaled"), token, isWine); - totalSizeExtras += GetFolderSize(Path.Combine(_configService.Current.CacheFolder, "decimated"), token, isWine); - - FileCacheSize = totalSize + totalSizeExtras; - var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d); if (FileCacheSize < maxCacheInBytes) return; var maxCacheBuffer = maxCacheInBytes * 0.05d; - while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer && files.Count > 0) + candidates.Sort(static (a, b) => a.LastAccessTime.CompareTo(b.LastAccessTime)); + + var evictionTarget = maxCacheInBytes - (long)maxCacheBuffer; + var index = 0; + while (FileCacheSize > evictionTarget && index < candidates.Count) { - var oldestFile = files[0]; + var oldestFile = candidates[index]; try { - long fileSize = oldestFile.Length; - File.Delete(oldestFile.FullName); - FileCacheSize -= fileSize; + EvictCacheCandidate(oldestFile, cacheFolder); + FileCacheSize -= oldestFile.Size; } catch (Exception ex) { - Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullName); + Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullPath); } - files.RemoveAt(0); + index++; } } @@ -519,54 +484,114 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase HaltScanLocks.Clear(); } - private long GetFolderSize(string directory, CancellationToken token, bool isWine) + private long AddFolderCandidates(string directory, List candidates, CancellationToken token, bool isWine) { if (!Directory.Exists(directory)) { return 0; } - var files = Directory.EnumerateFiles(directory) - .Select(f => new FileInfo(f)) - .OrderBy(f => f.LastAccessTime) - .ToList(); - long totalSize = 0; - foreach (var file in files) + foreach (var path in Directory.EnumerateFiles(directory)) { token.ThrowIfCancellationRequested(); try { - long size; - if (!isWine) - { - try - { - size = _fileCompactor.GetFileSizeOnDisk(file); - } - catch (Exception ex) - { - Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", file.FullName); - size = file.Length; - } - } - else - { - size = file.Length; - } - + var file = new FileInfo(path); + var size = GetFileSizeOnDisk(file, isWine); totalSize += size; + candidates.Add(new CacheEvictionCandidate(file.FullName, size, file.LastAccessTime)); } catch (Exception ex) { - Logger.LogTrace(ex, "Error getting size for {file}", file.FullName); + Logger.LogTrace(ex, "Error getting size for {file}", path); } } return totalSize; } + private long GetFileSizeOnDisk(FileInfo file, bool isWine) + { + if (isWine) + { + return file.Length; + } + + try + { + return _fileCompactor.GetFileSizeOnDisk(file); + } + catch (Exception ex) + { + Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", file.FullName); + return file.Length; + } + } + + private void EvictCacheCandidate(CacheEvictionCandidate candidate, string cacheFolder) + { + if (TryGetCacheHashAndPrefixedPath(candidate.FullPath, cacheFolder, out var hash, out var prefixedPath)) + { + _fileDbManager.RemoveHashedFile(hash, prefixedPath); + } + + try + { + if (File.Exists(candidate.FullPath)) + { + File.Delete(candidate.FullPath); + } + } + catch (Exception ex) + { + Logger.LogTrace(ex, "Failed to delete old file {file}", candidate.FullPath); + } + } + + private static bool TryGetCacheHashAndPrefixedPath(string filePath, string cacheFolder, out string hash, out string prefixedPath) + { + hash = string.Empty; + prefixedPath = string.Empty; + + if (string.IsNullOrEmpty(cacheFolder)) + { + return false; + } + + var fileName = Path.GetFileNameWithoutExtension(filePath); + if (string.IsNullOrEmpty(fileName) || !IsSha1Hash(fileName)) + { + return false; + } + + var relative = Path.GetRelativePath(cacheFolder, filePath) + .Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + var sanitizedRelative = relative.TrimStart(Path.DirectorySeparatorChar); + prefixedPath = Path.Combine(FileCacheManager.CachePrefix, sanitizedRelative); + hash = fileName; + return true; + } + + private static bool IsSha1Hash(string value) + { + if (value.Length != 40) + { + return false; + } + + foreach (var ch in value) + { + if (!Uri.IsHexDigit(ch)) + { + return false; + } + } + + return true; + } + public void ResumeScan(string source) { if (!HaltScanLocks.ContainsKey(source)) HaltScanLocks[source] = 0; diff --git a/LightlessSync/LightlessConfiguration/Configurations/PlayerPerformanceConfig.cs b/LightlessSync/LightlessConfiguration/Configurations/PlayerPerformanceConfig.cs index ce380f4..b905c05 100644 --- a/LightlessSync/LightlessConfiguration/Configurations/PlayerPerformanceConfig.cs +++ b/LightlessSync/LightlessConfiguration/Configurations/PlayerPerformanceConfig.cs @@ -25,6 +25,7 @@ public class PlayerPerformanceConfig : ILightlessConfiguration public bool EnableModelDecimation { get; set; } = false; public int ModelDecimationTriangleThreshold { get; set; } = 50_000; public double ModelDecimationTargetRatio { get; set; } = 0.8; + public bool KeepOriginalModelFiles { get; set; } = true; public bool ModelDecimationAllowBody { get; set; } = false; public bool ModelDecimationAllowFaceHead { get; set; } = false; public bool ModelDecimationAllowTail { get; set; } = false; diff --git a/LightlessSync/Services/ModelDecimation/ModelDecimationService.cs b/LightlessSync/Services/ModelDecimation/ModelDecimationService.cs index 8dc2571..f666805 100644 --- a/LightlessSync/Services/ModelDecimation/ModelDecimationService.cs +++ b/LightlessSync/Services/ModelDecimation/ModelDecimationService.cs @@ -1,6 +1,8 @@ +using LightlessSync.FileCache; using LightlessSync.LightlessConfiguration; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; +using System.Globalization; namespace LightlessSync.Services.ModelDecimation; @@ -12,6 +14,7 @@ public sealed class ModelDecimationService private readonly ILogger _logger; private readonly LightlessConfigService _configService; + private readonly FileCacheManager _fileCacheManager; private readonly PlayerPerformanceConfigService _performanceConfigService; private readonly XivDataStorageService _xivDataStorageService; private readonly SemaphoreSlim _decimationSemaphore = new(MaxConcurrentJobs); @@ -23,11 +26,13 @@ public sealed class ModelDecimationService public ModelDecimationService( ILogger logger, LightlessConfigService configService, + FileCacheManager fileCacheManager, PlayerPerformanceConfigService performanceConfigService, XivDataStorageService xivDataStorageService) { _logger = logger; _configService = configService; + _fileCacheManager = fileCacheManager; _performanceConfigService = performanceConfigService; _xivDataStorageService = xivDataStorageService; } @@ -145,7 +150,7 @@ public sealed class ModelDecimationService var destination = Path.Combine(GetDecimatedDirectory(), $"{hash}.mdl"); if (File.Exists(destination)) { - _decimatedPaths[hash] = destination; + RegisterDecimatedModel(hash, sourcePath, destination); return Task.CompletedTask; } @@ -156,11 +161,92 @@ public sealed class ModelDecimationService return Task.CompletedTask; } - _decimatedPaths[hash] = destination; + RegisterDecimatedModel(hash, sourcePath, destination); _logger.LogInformation("Decimated model {Hash} -> {Path}", hash, destination); return Task.CompletedTask; } + private void RegisterDecimatedModel(string hash, string sourcePath, string destination) + { + _decimatedPaths[hash] = destination; + + var performanceConfig = _performanceConfigService.Current; + if (performanceConfig.KeepOriginalModelFiles) + { + return; + } + + if (string.Equals(sourcePath, destination, StringComparison.OrdinalIgnoreCase)) + { + return; + } + + if (!TryReplaceCacheEntryWithDecimated(hash, sourcePath, destination)) + { + return; + } + + TryDelete(sourcePath); + } + + private bool TryReplaceCacheEntryWithDecimated(string hash, string sourcePath, string destination) + { + try + { + var cacheEntry = _fileCacheManager.GetFileCacheByHash(hash); + if (cacheEntry is null || !cacheEntry.IsCacheEntry) + { + return File.Exists(sourcePath) ? false : true; + } + + var cacheFolder = _configService.Current.CacheFolder; + if (string.IsNullOrEmpty(cacheFolder)) + { + return false; + } + + if (!destination.StartsWith(cacheFolder, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var info = new FileInfo(destination); + if (!info.Exists) + { + return false; + } + + var relative = Path.GetRelativePath(cacheFolder, destination) + .Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + var sanitizedRelative = relative.TrimStart(Path.DirectorySeparatorChar); + var prefixed = Path.Combine(FileCacheManager.CachePrefix, sanitizedRelative); + + var replacement = new FileCacheEntity( + hash, + prefixed, + info.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), + info.Length, + cacheEntry.CompressedSize); + replacement.SetResolvedFilePath(destination); + + if (!string.Equals(cacheEntry.PrefixedFilePath, prefixed, StringComparison.OrdinalIgnoreCase)) + { + _fileCacheManager.RemoveHashedFile(cacheEntry.Hash, cacheEntry.PrefixedFilePath, removeDerivedFiles: false); + } + + _fileCacheManager.UpdateHashedFile(replacement, computeProperties: false); + _fileCacheManager.WriteOutFullCsv(); + + _logger.LogTrace("Replaced cache entry for model {Hash} to decimated path {Path}", hash, destination); + return true; + } + catch (Exception ex) + { + _logger.LogTrace(ex, "Failed to replace cache entry for model {Hash}", hash); + return false; + } + } + private bool IsDecimationEnabled() => _performanceConfigService.Current.EnableModelDecimation; @@ -272,4 +358,24 @@ public sealed class ModelDecimationService return directory; } + + private static void TryDelete(string? path) + { + if (string.IsNullOrEmpty(path)) + { + return; + } + + try + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + catch + { + // ignored + } + } } diff --git a/LightlessSync/UI/SettingsUi.cs b/LightlessSync/UI/SettingsUi.cs index 9084436..1e5e055 100644 --- a/LightlessSync/UI/SettingsUi.cs +++ b/LightlessSync/UI/SettingsUi.cs @@ -3639,6 +3639,16 @@ public class SettingsUi : WindowMediatorSubscriberBase } _uiShared.DrawHelpText("When enabled, Lightless generates a decimated copy of given model after download."); + var keepOriginalModels = performanceConfig.KeepOriginalModelFiles; + if (ImGui.Checkbox("Keep original model files", ref keepOriginalModels)) + { + performanceConfig.KeepOriginalModelFiles = keepOriginalModels; + _playerPerformanceConfigService.Save(); + } + _uiShared.DrawHelpText("When disabled, Lightless removes the original model after a decimated copy is created."); + ImGui.SameLine(); + _uiShared.DrawNoteLine("! ", UIColors.Get("LightlessYellow"), new SeStringUtils.RichTextEntry("If disabled, saved + effective triangle usage information will not work.", UIColors.Get("LightlessYellow"))); + var triangleThreshold = performanceConfig.ModelDecimationTriangleThreshold; ImGui.SetNextItemWidth(300 * ImGuiHelpers.GlobalScale); if (ImGui.SliderInt("Decimate models above", ref triangleThreshold, 10_000, 100_000))