Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 51 Wiki Activity

Added debug mode for lightfinder IMGUI, added caching of file cache entries to reduce load of loading all entries again.

Browse Source
This commit is contained in:
cake
2025-12-28 03:01:02 +01:00
parent 8f32b375dd
commit deb99628f6
8 changed files with 723 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
LightlessSync/FileCache/FileCacheManager.cs
View File

@@ -7,6 +7,8 @@ using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
using System.Globalization;
using System.IO;
using System.Runtime.InteropServices;
using System.Text;
namespace LightlessSync.FileCache;
@@ -31,6 +33,14 @@ public sealed class FileCacheManager : IHostedService
private bool _csvHeaderEnsured;
public string CacheFolder => _configService.Current.CacheFolder;
private const string _compressedCacheExtension = ".llz4";
private readonly ConcurrentDictionary<string, SemaphoreSlim> _compressLocks = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, SizeInfo> _sizeCache =
new(StringComparer.OrdinalIgnoreCase);
[StructLayout(LayoutKind.Auto)]
public readonly record struct SizeInfo(long Original, long Compressed);
public FileCacheManager(ILogger<FileCacheManager> logger, IpcManager ipcManager, LightlessConfigService configService, LightlessMediator lightlessMediator)
{
_logger = logger;
@@ -45,6 +55,18 @@ public sealed class FileCacheManager : IHostedService
private static string NormalizeSeparators(string path) => path.Replace("/", "\\", StringComparison.Ordinal)
.Replace("\\\\", "\\", StringComparison.Ordinal);
private SemaphoreSlim GetCompressLock(string hash)
=> _compressLocks.GetOrAdd(hash, _ => new SemaphoreSlim(1, 1));
public void SetSizeInfo(string hash, long original, long compressed)
=> _sizeCache[hash] = new SizeInfo(original, compressed);
public bool TryGetSizeInfo(string hash, out SizeInfo info)
=> _sizeCache.TryGetValue(hash, out info);
private string GetCompressedCachePath(string hash)
=> Path.Combine(CacheFolder, hash + _compressedCacheExtension);
private static string NormalizePrefixedPathKey(string prefixedPath)
{
if (string.IsNullOrEmpty(prefixedPath))
@@ -111,6 +133,114 @@ public sealed class FileCacheManager : IHostedService
return int.TryParse(versionSpan, NumberStyles.Integer, CultureInfo.InvariantCulture, out version);
}
public void UpdateSizeInfo(string hash, long? original = null, long? compressed = null)
{
_sizeCache.AddOrUpdate(
hash,
_ => new SizeInfo(original ?? 0, compressed ?? 0),
(_, old) => new SizeInfo(original ?? old.Original, compressed ?? old.Compressed));
}
private void UpdateEntitiesSizes(string hash, long original, long compressed)
{
if (_fileCaches.TryGetValue(hash, out var dict))
{
foreach (var e in dict.Values)
{
e.Size = original;
e.CompressedSize = compressed;
}
}
}
public static void ApplySizesToEntries(IEnumerable<FileCacheEntity?> entries, long original, long compressed)
{
foreach (var e in entries)
{
if (e == null) continue;
e.Size = original;
e.CompressedSize = compressed > 0 ? compressed : null;
}
}
public async Task<long> GetCompressedSizeAsync(string hash, CancellationToken token)
{
if (_sizeCache.TryGetValue(hash, out var info) && info.Compressed > 0)
return info.Compressed;
if (_fileCaches.TryGetValue(hash, out var dict))
{
var any = dict.Values.FirstOrDefault();
if (any != null && any.CompressedSize > 0)
{
UpdateSizeInfo(hash, original: any.Size > 0 ? any.Size : null, compressed: any.CompressedSize);
return (long)any.CompressedSize;
}
}
if (!string.IsNullOrWhiteSpace(CacheFolder))
{
var path = GetCompressedCachePath(hash);
if (File.Exists(path))
{
var len = new FileInfo(path).Length;
UpdateSizeInfo(hash, compressed: len);
return len;
}
var bytes = await EnsureCompressedCacheBytesAsync(hash, token).ConfigureAwait(false);
return bytes.LongLength;
}
var fallback = await GetCompressedFileData(hash, token).ConfigureAwait(false);
return fallback.Item2.LongLength;
}
private async Task<byte[]> EnsureCompressedCacheBytesAsync(string hash, CancellationToken token)
{
if (string.IsNullOrWhiteSpace(CacheFolder))
throw new InvalidOperationException("CacheFolder is not set; cannot persist compressed cache.");
Directory.CreateDirectory(CacheFolder);
var compressedPath = GetCompressedCachePath(hash);
if (File.Exists(compressedPath))
return await File.ReadAllBytesAsync(compressedPath, token).ConfigureAwait(false);
var sem = GetCompressLock(hash);
await sem.WaitAsync(token).ConfigureAwait(false);
try
{
if (File.Exists(compressedPath))
return await File.ReadAllBytesAsync(compressedPath, token).ConfigureAwait(false);
var entity = GetFileCacheByHash(hash);
if (entity == null || string.IsNullOrWhiteSpace(entity.ResolvedFilepath))
throw new InvalidOperationException($"No local file cache found for hash {hash}.");
var sourcePath = entity.ResolvedFilepath;
var originalSize = new FileInfo(sourcePath).Length;
var raw = await File.ReadAllBytesAsync(sourcePath, token).ConfigureAwait(false);
var compressed = LZ4Wrapper.WrapHC(raw, 0, raw.Length);
var tmpPath = compressedPath + ".tmp";
await File.WriteAllBytesAsync(tmpPath, compressed, token).ConfigureAwait(false);
File.Move(tmpPath, compressedPath, overwrite: true);
var compressedSize = compressed.LongLength;
SetSizeInfo(hash, originalSize, compressedSize);
UpdateEntitiesSizes(hash, originalSize, compressedSize);
return compressed;
}
finally
{
sem.Release();
}
}
private string NormalizeToPrefixedPath(string path)
{
if (string.IsNullOrEmpty(path)) return string.Empty;
@@ -318,9 +448,18 @@ public sealed class FileCacheManager : IHostedService
public async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken)
{
if (!string.IsNullOrWhiteSpace(CacheFolder))
{
var bytes = await EnsureCompressedCacheBytesAsync(fileHash, uploadToken).ConfigureAwait(false);
UpdateSizeInfo(fileHash, compressed: bytes.LongLength);
return (fileHash, bytes);
}
var fileCache = GetFileCacheByHash(fileHash)!.ResolvedFilepath;
return (fileHash, LZ4Wrapper.WrapHC(await File.ReadAllBytesAsync(fileCache, uploadToken).ConfigureAwait(false), 0,
(int)new FileInfo(fileCache).Length));
var raw = await File.ReadAllBytesAsync(fileCache, uploadToken).ConfigureAwait(false);
var compressed = LZ4Wrapper.WrapHC(raw, 0, raw.Length);
UpdateSizeInfo(fileHash, original: raw.LongLength, compressed: compressed.LongLength);
return (fileHash, compressed);
}
public FileCacheEntity? GetFileCacheByHash(string hash)
@@ -891,6 +1030,14 @@ public sealed class FileCacheManager : IHostedService
compressed = resultCompressed;
}
}
if (size > 0 || compressed > 0)
{
UpdateSizeInfo(hash,
original: size > 0 ? size : null,
compressed: compressed > 0 ? compressed : null);
}
AddHashedFile(ReplacePathPrefixes(new FileCacheEntity(hash, path, time, size, compressed)));
}
catch (Exception ex)