Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 3 Actions Packages Projects Releases 57 Wiki Activity

highly experimental runtime model decimation + file cache adjustment to clean up processed file copies

Browse Source
This commit is contained in:
Essie
2026-01-02 09:54:34 +09:00
parent 44bb53023e
commit aeed8503c2
46 changed files with 9990 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
LightlessSync/FileCache/CacheMonitor.cs
View File

@@ -481,53 +481,13 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
}
}
long totalSizeExtras = 0;
FileCacheSize = totalSize;
if (Directory.Exists(_configService.Current.CacheFolder + "/downscaled"))
{
var filesDownscaled = Directory.EnumerateFiles(_configService.Current.CacheFolder + "/downscaled").Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList();
totalSizeExtras += GetFolderSize(Path.Combine(_configService.Current.CacheFolder, "downscaled"), token, isWine);
totalSizeExtras += GetFolderSize(Path.Combine(_configService.Current.CacheFolder, "decimated"), token, isWine);
long totalSizeDownscaled = 0;
foreach (var f in filesDownscaled)
{
token.ThrowIfCancellationRequested();
try
{
long size = 0;
if (!isWine)
{
try
{
size = _fileCompactor.GetFileSizeOnDisk(f);
}
catch (Exception ex)
{
Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName);
size = f.Length;
}
}
else
{
size = f.Length;
}
totalSizeDownscaled += size;
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Error getting size for {file}", f.FullName);
}
}
FileCacheSize = (totalSize + totalSizeDownscaled);
}
else
{
FileCacheSize = totalSize;
}
FileCacheSize = totalSize + totalSizeExtras;
var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d);
if (FileCacheSize < maxCacheInBytes)
@@ -559,6 +519,54 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
HaltScanLocks.Clear();
}
private long GetFolderSize(string directory, CancellationToken token, bool isWine)
{
if (!Directory.Exists(directory))
{
return 0;
}
var files = Directory.EnumerateFiles(directory)
.Select(f => new FileInfo(f))
.OrderBy(f => f.LastAccessTime)
.ToList();
long totalSize = 0;
foreach (var file in files)
{
token.ThrowIfCancellationRequested();
try
{
long size;
if (!isWine)
{
try
{
size = _fileCompactor.GetFileSizeOnDisk(file);
}
catch (Exception ex)
{
Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", file.FullName);
size = file.Length;
}
}
else
{
size = file.Length;
}
totalSize += size;
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Error getting size for {file}", file.FullName);
}
}
return totalSize;
}
public void ResumeScan(string source)
{
if (!HaltScanLocks.ContainsKey(source)) HaltScanLocks[source] = 0;

42
LightlessSync/FileCache/FileCacheManager.cs
View File

@@ -582,9 +582,10 @@ public sealed class FileCacheManager : IHostedService
}
}
public void RemoveHashedFile(string hash, string prefixedFilePath)
public void RemoveHashedFile(string hash, string prefixedFilePath, bool removeDerivedFiles = true)
{
var normalizedPath = NormalizePrefixedPathKey(prefixedFilePath);
var removedHash = false;
if (_fileCaches.TryGetValue(hash, out var caches))
{
@@ -597,11 +598,16 @@ public sealed class FileCacheManager : IHostedService
if (caches.IsEmpty)
{
_fileCaches.TryRemove(hash, out _);
removedHash = _fileCaches.TryRemove(hash, out _);
}
}
_fileCachesByPrefixedPath.TryRemove(normalizedPath, out _);
if (removeDerivedFiles && removedHash)
{
RemoveDerivedCacheFiles(hash);
}
}
public void UpdateHashedFile(FileCacheEntity fileCache, bool computeProperties = true)
@@ -617,7 +623,8 @@ public sealed class FileCacheManager : IHostedService
fileCache.Hash = Crypto.ComputeFileHash(fileCache.ResolvedFilepath, Crypto.HashAlgo.Sha1);
fileCache.LastModifiedDateTicks = fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
}
RemoveHashedFile(oldHash, prefixedPath);
var removeDerivedFiles = !string.Equals(oldHash, fileCache.Hash, StringComparison.OrdinalIgnoreCase);
RemoveHashedFile(oldHash, prefixedPath, removeDerivedFiles);
AddHashedFile(fileCache);
}
@@ -767,7 +774,7 @@ public sealed class FileCacheManager : IHostedService
{
try
{
RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath);
RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath, removeDerivedFiles: false);
var extensionPath = fileCache.ResolvedFilepath.ToUpper(CultureInfo.InvariantCulture) + "." + ext;
File.Move(fileCache.ResolvedFilepath, extensionPath, overwrite: true);
var newHashedEntity = new FileCacheEntity(fileCache.Hash, fileCache.PrefixedFilePath + "." + ext, DateTime.UtcNow.Ticks.ToString(CultureInfo.InvariantCulture));
@@ -784,6 +791,33 @@ public sealed class FileCacheManager : IHostedService
}
}
private void RemoveDerivedCacheFiles(string hash)
{
var cacheFolder = _configService.Current.CacheFolder;
if (string.IsNullOrWhiteSpace(cacheFolder))
{
return;
}
TryDeleteDerivedCacheFile(Path.Combine(cacheFolder, "downscaled", $"{hash}.tex"));
TryDeleteDerivedCacheFile(Path.Combine(cacheFolder, "decimated", $"{hash}.mdl"));
}
private void TryDeleteDerivedCacheFile(string path)
{
try
{
if (File.Exists(path))
{
File.Delete(path);
}
}
catch (Exception ex)
{
_logger.LogTrace(ex, "Failed to delete derived cache file {path}", path);
}
}
private void AddHashedFile(FileCacheEntity fileCache)
{
var normalizedPath = NormalizePrefixedPathKey(fileCache.PrefixedFilePath);