Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 52 Wiki Activity

readjust cache clean up and add keep originals setting for models

Browse Source
This commit is contained in:
Essie
2026-01-02 11:04:15 +09:00
parent aeed8503c2
commit 92cb861710
4 changed files with 221 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

179
LightlessSync/FileCache/CacheMonitor.cs
View File

@@ -103,6 +103,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
}
record WatcherChange(WatcherChangeTypes ChangeType, string? OldPath = null);
private readonly record struct CacheEvictionCandidate(string FullPath, long Size, DateTime LastAccessTime);
private readonly Dictionary<string, WatcherChange> _watcherChanges = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, WatcherChange> _lightlessChanges = new(StringComparer.OrdinalIgnoreCase);
@@ -441,76 +442,40 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
Logger.LogWarning(ex, "Could not determine drive size for storage folder {folder}", _configService.Current.CacheFolder);
}
var files = Directory.EnumerateFiles(_configService.Current.CacheFolder)
.Select(f => new FileInfo(f))
.OrderBy(f => f.LastAccessTime)
.ToList();
var cacheFolder = _configService.Current.CacheFolder;
var candidates = new List<CacheEvictionCandidate>();
long totalSize = 0;
totalSize += AddFolderCandidates(cacheFolder, candidates, token, isWine);
totalSize += AddFolderCandidates(Path.Combine(cacheFolder, "downscaled"), candidates, token, isWine);
totalSize += AddFolderCandidates(Path.Combine(cacheFolder, "decimated"), candidates, token, isWine);
foreach (var f in files)
{
token.ThrowIfCancellationRequested();
try
{
long size = 0;
if (!isWine)
{
try
{
size = _fileCompactor.GetFileSizeOnDisk(f);
}
catch (Exception ex)
{
Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName);
size = f.Length;
}
}
else
{
size = f.Length;
}
totalSize += size;
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Error getting size for {file}", f.FullName);
}
}
long totalSizeExtras = 0;
FileCacheSize = totalSize;
totalSizeExtras += GetFolderSize(Path.Combine(_configService.Current.CacheFolder, "downscaled"), token, isWine);
totalSizeExtras += GetFolderSize(Path.Combine(_configService.Current.CacheFolder, "decimated"), token, isWine);
FileCacheSize = totalSize + totalSizeExtras;
var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d);
if (FileCacheSize < maxCacheInBytes)
return;
var maxCacheBuffer = maxCacheInBytes * 0.05d;
while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer && files.Count > 0)
candidates.Sort(static (a, b) => a.LastAccessTime.CompareTo(b.LastAccessTime));
var evictionTarget = maxCacheInBytes - (long)maxCacheBuffer;
var index = 0;
while (FileCacheSize > evictionTarget && index < candidates.Count)
{
var oldestFile = files[0];
var oldestFile = candidates[index];
try
{
long fileSize = oldestFile.Length;
File.Delete(oldestFile.FullName);
FileCacheSize -= fileSize;
EvictCacheCandidate(oldestFile, cacheFolder);
FileCacheSize -= oldestFile.Size;
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullName);
Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullPath);
}
files.RemoveAt(0);
index++;
}
}
@@ -519,54 +484,114 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
HaltScanLocks.Clear();
}
private long GetFolderSize(string directory, CancellationToken token, bool isWine)
private long AddFolderCandidates(string directory, List<CacheEvictionCandidate> candidates, CancellationToken token, bool isWine)
{
if (!Directory.Exists(directory))
{
return 0;
}
var files = Directory.EnumerateFiles(directory)
.Select(f => new FileInfo(f))
.OrderBy(f => f.LastAccessTime)
.ToList();
long totalSize = 0;
foreach (var file in files)
foreach (var path in Directory.EnumerateFiles(directory))
{
token.ThrowIfCancellationRequested();
try
{
long size;
if (!isWine)
{
try
{
size = _fileCompactor.GetFileSizeOnDisk(file);
}
catch (Exception ex)
{
Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", file.FullName);
size = file.Length;
}
}
else
{
size = file.Length;
}
var file = new FileInfo(path);
var size = GetFileSizeOnDisk(file, isWine);
totalSize += size;
candidates.Add(new CacheEvictionCandidate(file.FullName, size, file.LastAccessTime));
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Error getting size for {file}", file.FullName);
Logger.LogTrace(ex, "Error getting size for {file}", path);
}
}
return totalSize;
}
private long GetFileSizeOnDisk(FileInfo file, bool isWine)
{
if (isWine)
{
return file.Length;
}
try
{
return _fileCompactor.GetFileSizeOnDisk(file);
}
catch (Exception ex)
{
Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", file.FullName);
return file.Length;
}
}
private void EvictCacheCandidate(CacheEvictionCandidate candidate, string cacheFolder)
{
if (TryGetCacheHashAndPrefixedPath(candidate.FullPath, cacheFolder, out var hash, out var prefixedPath))
{
_fileDbManager.RemoveHashedFile(hash, prefixedPath);
}
try
{
if (File.Exists(candidate.FullPath))
{
File.Delete(candidate.FullPath);
}
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Failed to delete old file {file}", candidate.FullPath);
}
}
private static bool TryGetCacheHashAndPrefixedPath(string filePath, string cacheFolder, out string hash, out string prefixedPath)
{
hash = string.Empty;
prefixedPath = string.Empty;
if (string.IsNullOrEmpty(cacheFolder))
{
return false;
}
var fileName = Path.GetFileNameWithoutExtension(filePath);
if (string.IsNullOrEmpty(fileName) || !IsSha1Hash(fileName))
{
return false;
}
var relative = Path.GetRelativePath(cacheFolder, filePath)
.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
var sanitizedRelative = relative.TrimStart(Path.DirectorySeparatorChar);
prefixedPath = Path.Combine(FileCacheManager.CachePrefix, sanitizedRelative);
hash = fileName;
return true;
}
private static bool IsSha1Hash(string value)
{
if (value.Length != 40)
{
return false;
}
foreach (var ch in value)
{
if (!Uri.IsHexDigit(ch))
{
return false;
}
}
return true;
}
public void ResumeScan(string source)
{
if (!HaltScanLocks.ContainsKey(source)) HaltScanLocks[source] = 0;

1
LightlessSync/LightlessConfiguration/Configurations/PlayerPerformanceConfig.cs
View File

@@ -25,6 +25,7 @@ public class PlayerPerformanceConfig : ILightlessConfiguration
public bool EnableModelDecimation { get; set; } = false;
public int ModelDecimationTriangleThreshold { get; set; } = 50_000;
public double ModelDecimationTargetRatio { get; set; } = 0.8;
public bool KeepOriginalModelFiles { get; set; } = true;
public bool ModelDecimationAllowBody { get; set; } = false;
public bool ModelDecimationAllowFaceHead { get; set; } = false;
public bool ModelDecimationAllowTail { get; set; } = false;

110
LightlessSync/Services/ModelDecimation/ModelDecimationService.cs
View File

@@ -1,6 +1,8 @@
using LightlessSync.FileCache;
using LightlessSync.LightlessConfiguration;
using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
using System.Globalization;
namespace LightlessSync.Services.ModelDecimation;
@@ -12,6 +14,7 @@ public sealed class ModelDecimationService
private readonly ILogger<ModelDecimationService> _logger;
private readonly LightlessConfigService _configService;
private readonly FileCacheManager _fileCacheManager;
private readonly PlayerPerformanceConfigService _performanceConfigService;
private readonly XivDataStorageService _xivDataStorageService;
private readonly SemaphoreSlim _decimationSemaphore = new(MaxConcurrentJobs);
@@ -23,11 +26,13 @@ public sealed class ModelDecimationService
public ModelDecimationService(
ILogger<ModelDecimationService> logger,
LightlessConfigService configService,
FileCacheManager fileCacheManager,
PlayerPerformanceConfigService performanceConfigService,
XivDataStorageService xivDataStorageService)
{
_logger = logger;
_configService = configService;
_fileCacheManager = fileCacheManager;
_performanceConfigService = performanceConfigService;
_xivDataStorageService = xivDataStorageService;
}
@@ -145,7 +150,7 @@ public sealed class ModelDecimationService
var destination = Path.Combine(GetDecimatedDirectory(), $"{hash}.mdl");
if (File.Exists(destination))
{
_decimatedPaths[hash] = destination;
RegisterDecimatedModel(hash, sourcePath, destination);
return Task.CompletedTask;
}
@@ -156,11 +161,92 @@ public sealed class ModelDecimationService
return Task.CompletedTask;
}
_decimatedPaths[hash] = destination;
RegisterDecimatedModel(hash, sourcePath, destination);
_logger.LogInformation("Decimated model {Hash} -> {Path}", hash, destination);
return Task.CompletedTask;
}
private void RegisterDecimatedModel(string hash, string sourcePath, string destination)
{
_decimatedPaths[hash] = destination;
var performanceConfig = _performanceConfigService.Current;
if (performanceConfig.KeepOriginalModelFiles)
{
return;
}
if (string.Equals(sourcePath, destination, StringComparison.OrdinalIgnoreCase))
{
return;
}
if (!TryReplaceCacheEntryWithDecimated(hash, sourcePath, destination))
{
return;
}
TryDelete(sourcePath);
}
private bool TryReplaceCacheEntryWithDecimated(string hash, string sourcePath, string destination)
{
try
{
var cacheEntry = _fileCacheManager.GetFileCacheByHash(hash);
if (cacheEntry is null || !cacheEntry.IsCacheEntry)
{
return File.Exists(sourcePath) ? false : true;
}
var cacheFolder = _configService.Current.CacheFolder;
if (string.IsNullOrEmpty(cacheFolder))
{
return false;
}
if (!destination.StartsWith(cacheFolder, StringComparison.OrdinalIgnoreCase))
{
return false;
}
var info = new FileInfo(destination);
if (!info.Exists)
{
return false;
}
var relative = Path.GetRelativePath(cacheFolder, destination)
.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
var sanitizedRelative = relative.TrimStart(Path.DirectorySeparatorChar);
var prefixed = Path.Combine(FileCacheManager.CachePrefix, sanitizedRelative);
var replacement = new FileCacheEntity(
hash,
prefixed,
info.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture),
info.Length,
cacheEntry.CompressedSize);
replacement.SetResolvedFilePath(destination);
if (!string.Equals(cacheEntry.PrefixedFilePath, prefixed, StringComparison.OrdinalIgnoreCase))
{
_fileCacheManager.RemoveHashedFile(cacheEntry.Hash, cacheEntry.PrefixedFilePath, removeDerivedFiles: false);
}
_fileCacheManager.UpdateHashedFile(replacement, computeProperties: false);
_fileCacheManager.WriteOutFullCsv();
_logger.LogTrace("Replaced cache entry for model {Hash} to decimated path {Path}", hash, destination);
return true;
}
catch (Exception ex)
{
_logger.LogTrace(ex, "Failed to replace cache entry for model {Hash}", hash);
return false;
}
}
private bool IsDecimationEnabled()
=> _performanceConfigService.Current.EnableModelDecimation;
@@ -272,4 +358,24 @@ public sealed class ModelDecimationService
return directory;
}
private static void TryDelete(string? path)
{
if (string.IsNullOrEmpty(path))
{
return;
}
try
{
if (File.Exists(path))
{
File.Delete(path);
}
}
catch
{
// ignored
}
}
}

10
LightlessSync/UI/SettingsUi.cs
View File

@@ -3639,6 +3639,16 @@ public class SettingsUi : WindowMediatorSubscriberBase
}
_uiShared.DrawHelpText("When enabled, Lightless generates a decimated copy of given model after download.");
var keepOriginalModels = performanceConfig.KeepOriginalModelFiles;
if (ImGui.Checkbox("Keep original model files", ref keepOriginalModels))
{
performanceConfig.KeepOriginalModelFiles = keepOriginalModels;
_playerPerformanceConfigService.Save();
}
_uiShared.DrawHelpText("When disabled, Lightless removes the original model after a decimated copy is created.");
ImGui.SameLine();
_uiShared.DrawNoteLine("! ", UIColors.Get("LightlessYellow"), new SeStringUtils.RichTextEntry("If disabled, saved + effective triangle usage information will not work.", UIColors.Get("LightlessYellow")));
var triangleThreshold = performanceConfig.ModelDecimationTriangleThreshold;
ImGui.SetNextItemWidth(300 * ImGuiHelpers.GlobalScale);
if (ImGui.SliderInt("Decimate models above", ref triangleThreshold, 10_000, 100_000))