Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 53 Wiki Activity

Merged Cake and Abel branched into 2.0.3 (#131)

Browse Source 
Co-authored-by: azyges <aaaaaa@aaa.aaa>
Co-authored-by: cake <admin@cakeandbanana.nl>
Co-authored-by: defnotken <itsdefnotken@gmail.com>
Reviewed-on: #131
This commit was merged in pull request #131.
This commit is contained in:
cake
2026-01-05 00:45:14 +00:00
parent e0b8070aa8
commit 30717ba200
67 changed files with 13247 additions and 802 deletions
Download Patch File Download Diff File Expand all files Collapse all files

215
LightlessSync/FileCache/CacheMonitor.cs
View File

@@ -103,6 +103,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
}
record WatcherChange(WatcherChangeTypes ChangeType, string? OldPath = null);
private readonly record struct CacheEvictionCandidate(string FullPath, long Size, DateTime LastAccessTime);
private readonly Dictionary<string, WatcherChange> _watcherChanges = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, WatcherChange> _lightlessChanges = new(StringComparer.OrdinalIgnoreCase);
@@ -441,116 +442,40 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
Logger.LogWarning(ex, "Could not determine drive size for storage folder {folder}", _configService.Current.CacheFolder);
}
var files = Directory.EnumerateFiles(_configService.Current.CacheFolder)
.Select(f => new FileInfo(f))
.OrderBy(f => f.LastAccessTime)
.ToList();
var cacheFolder = _configService.Current.CacheFolder;
var candidates = new List<CacheEvictionCandidate>();
long totalSize = 0;
foreach (var f in files)
{
token.ThrowIfCancellationRequested();
try
{
long size = 0;
if (!isWine)
{
try
{
size = _fileCompactor.GetFileSizeOnDisk(f);
}
catch (Exception ex)
{
Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName);
size = f.Length;
}
}
else
{
size = f.Length;
}
totalSize += size;
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Error getting size for {file}", f.FullName);
}
}
totalSize += AddFolderCandidates(cacheFolder, candidates, token, isWine);
totalSize += AddFolderCandidates(Path.Combine(cacheFolder, "downscaled"), candidates, token, isWine);
totalSize += AddFolderCandidates(Path.Combine(cacheFolder, "decimated"), candidates, token, isWine);
FileCacheSize = totalSize;
if (Directory.Exists(_configService.Current.CacheFolder + "/downscaled"))
{
var filesDownscaled = Directory.EnumerateFiles(_configService.Current.CacheFolder + "/downscaled").Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList();
long totalSizeDownscaled = 0;
foreach (var f in filesDownscaled)
{
token.ThrowIfCancellationRequested();
try
{
long size = 0;
if (!isWine)
{
try
{
size = _fileCompactor.GetFileSizeOnDisk(f);
}
catch (Exception ex)
{
Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName);
size = f.Length;
}
}
else
{
size = f.Length;
}
totalSizeDownscaled += size;
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Error getting size for {file}", f.FullName);
}
}
FileCacheSize = (totalSize + totalSizeDownscaled);
}
else
{
FileCacheSize = totalSize;
}
var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d);
if (FileCacheSize < maxCacheInBytes)
return;
var maxCacheBuffer = maxCacheInBytes * 0.05d;
while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer && files.Count > 0)
candidates.Sort(static (a, b) => a.LastAccessTime.CompareTo(b.LastAccessTime));
var evictionTarget = maxCacheInBytes - (long)maxCacheBuffer;
var index = 0;
while (FileCacheSize > evictionTarget && index < candidates.Count)
{
var oldestFile = files[0];
var oldestFile = candidates[index];
try
{
long fileSize = oldestFile.Length;
File.Delete(oldestFile.FullName);
FileCacheSize -= fileSize;
EvictCacheCandidate(oldestFile, cacheFolder);
FileCacheSize -= oldestFile.Size;
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullName);
Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullPath);
}
files.RemoveAt(0);
index++;
}
}
@@ -559,6 +484,114 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
HaltScanLocks.Clear();
}
private long AddFolderCandidates(string directory, List<CacheEvictionCandidate> candidates, CancellationToken token, bool isWine)
{
if (!Directory.Exists(directory))
{
return 0;
}
long totalSize = 0;
foreach (var path in Directory.EnumerateFiles(directory))
{
token.ThrowIfCancellationRequested();
try
{
var file = new FileInfo(path);
var size = GetFileSizeOnDisk(file, isWine);
totalSize += size;
candidates.Add(new CacheEvictionCandidate(file.FullName, size, file.LastAccessTime));
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Error getting size for {file}", path);
}
}
return totalSize;
}
private long GetFileSizeOnDisk(FileInfo file, bool isWine)
{
if (isWine)
{
return file.Length;
}
try
{
return _fileCompactor.GetFileSizeOnDisk(file);
}
catch (Exception ex)
{
Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", file.FullName);
return file.Length;
}
}
private void EvictCacheCandidate(CacheEvictionCandidate candidate, string cacheFolder)
{
if (TryGetCacheHashAndPrefixedPath(candidate.FullPath, cacheFolder, out var hash, out var prefixedPath))
{
_fileDbManager.RemoveHashedFile(hash, prefixedPath);
}
try
{
if (File.Exists(candidate.FullPath))
{
File.Delete(candidate.FullPath);
}
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Failed to delete old file {file}", candidate.FullPath);
}
}
private static bool TryGetCacheHashAndPrefixedPath(string filePath, string cacheFolder, out string hash, out string prefixedPath)
{
hash = string.Empty;
prefixedPath = string.Empty;
if (string.IsNullOrEmpty(cacheFolder))
{
return false;
}
var fileName = Path.GetFileNameWithoutExtension(filePath);
if (string.IsNullOrEmpty(fileName) || !IsSha1Hash(fileName))
{
return false;
}
var relative = Path.GetRelativePath(cacheFolder, filePath)
.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
var sanitizedRelative = relative.TrimStart(Path.DirectorySeparatorChar);
prefixedPath = Path.Combine(FileCacheManager.CachePrefix, sanitizedRelative);
hash = fileName;
return true;
}
private static bool IsSha1Hash(string value)
{
if (value.Length != 40)
{
return false;
}
foreach (var ch in value)
{
if (!Uri.IsHexDigit(ch))
{
return false;
}
}
return true;
}
public void ResumeScan(string source)
{
if (!HaltScanLocks.ContainsKey(source)) HaltScanLocks[source] = 0;

158
LightlessSync/FileCache/FileCacheManager.cs
View File

@@ -27,6 +27,7 @@ public sealed class FileCacheManager : IHostedService
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, FileCacheEntity> _fileCachesByPrefixedPath = new(StringComparer.OrdinalIgnoreCase);
private readonly SemaphoreSlim _getCachesByPathsSemaphore = new(1, 1);
private readonly SemaphoreSlim _evictSemaphore = new(1, 1);
private readonly Lock _fileWriteLock = new();
private readonly IpcManager _ipcManager;
private readonly ILogger<FileCacheManager> _logger;
@@ -226,13 +227,23 @@ public sealed class FileCacheManager : IHostedService
var compressed = LZ4Wrapper.WrapHC(raw, 0, raw.Length);
var tmpPath = compressedPath + ".tmp";
await File.WriteAllBytesAsync(tmpPath, compressed, token).ConfigureAwait(false);
File.Move(tmpPath, compressedPath, overwrite: true);
try
{
await File.WriteAllBytesAsync(tmpPath, compressed, token).ConfigureAwait(false);
File.Move(tmpPath, compressedPath, overwrite: true);
}
finally
{
try { if (File.Exists(tmpPath)) File.Delete(tmpPath); } catch { /* ignore */ }
}
var compressedSize = compressed.LongLength;
var compressedSize = new FileInfo(compressedPath).Length;
SetSizeInfo(hash, originalSize, compressedSize);
UpdateEntitiesSizes(hash, originalSize, compressedSize);
var maxBytes = GiBToBytes(_configService.Current.MaxLocalCacheInGiB);
await EnforceCacheLimitAsync(maxBytes, token).ConfigureAwait(false);
return compressed;
}
finally
@@ -280,6 +291,26 @@ public sealed class FileCacheManager : IHostedService
return CreateFileEntity(cacheFolder, CachePrefix, fi);
}
public FileCacheEntity? CreateCacheEntryWithKnownHash(string path, string hash)
{
if (string.IsNullOrWhiteSpace(hash))
{
return CreateCacheEntry(path);
}
FileInfo fi = new(path);
if (!fi.Exists) return null;
_logger.LogTrace("Creating cache entry for {path} using provided hash", path);
var cacheFolder = _configService.Current.CacheFolder;
if (string.IsNullOrEmpty(cacheFolder)) return null;
if (!TryBuildPrefixedPath(fi.FullName, cacheFolder, CachePrefix, out var prefixedPath, out _))
{
return null;
}
return CreateFileCacheEntity(fi, prefixedPath, hash);
}
public FileCacheEntity? CreateFileEntry(string path)
{
FileInfo fi = new(path);
@@ -562,9 +593,10 @@ public sealed class FileCacheManager : IHostedService
}
}
public void RemoveHashedFile(string hash, string prefixedFilePath)
public void RemoveHashedFile(string hash, string prefixedFilePath, bool removeDerivedFiles = true)
{
var normalizedPath = NormalizePrefixedPathKey(prefixedFilePath);
var removedHash = false;
if (_fileCaches.TryGetValue(hash, out var caches))
{
@@ -577,11 +609,16 @@ public sealed class FileCacheManager : IHostedService
if (caches.IsEmpty)
{
_fileCaches.TryRemove(hash, out _);
removedHash = _fileCaches.TryRemove(hash, out _);
}
}
_fileCachesByPrefixedPath.TryRemove(normalizedPath, out _);
if (removeDerivedFiles && removedHash)
{
RemoveDerivedCacheFiles(hash);
}
}
public void UpdateHashedFile(FileCacheEntity fileCache, bool computeProperties = true)
@@ -597,7 +634,8 @@ public sealed class FileCacheManager : IHostedService
fileCache.Hash = Crypto.ComputeFileHash(fileCache.ResolvedFilepath, Crypto.HashAlgo.Sha1);
fileCache.LastModifiedDateTicks = fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
}
RemoveHashedFile(oldHash, prefixedPath);
var removeDerivedFiles = !string.Equals(oldHash, fileCache.Hash, StringComparison.OrdinalIgnoreCase);
RemoveHashedFile(oldHash, prefixedPath, removeDerivedFiles);
AddHashedFile(fileCache);
}
@@ -747,7 +785,7 @@ public sealed class FileCacheManager : IHostedService
{
try
{
RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath);
RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath, removeDerivedFiles: false);
var extensionPath = fileCache.ResolvedFilepath.ToUpper(CultureInfo.InvariantCulture) + "." + ext;
File.Move(fileCache.ResolvedFilepath, extensionPath, overwrite: true);
var newHashedEntity = new FileCacheEntity(fileCache.Hash, fileCache.PrefixedFilePath + "." + ext, DateTime.UtcNow.Ticks.ToString(CultureInfo.InvariantCulture));
@@ -764,6 +802,33 @@ public sealed class FileCacheManager : IHostedService
}
}
private void RemoveDerivedCacheFiles(string hash)
{
var cacheFolder = _configService.Current.CacheFolder;
if (string.IsNullOrWhiteSpace(cacheFolder))
{
return;
}
TryDeleteDerivedCacheFile(Path.Combine(cacheFolder, "downscaled", $"{hash}.tex"));
TryDeleteDerivedCacheFile(Path.Combine(cacheFolder, "decimated", $"{hash}.mdl"));
}
private void TryDeleteDerivedCacheFile(string path)
{
try
{
if (File.Exists(path))
{
File.Delete(path);
}
}
catch (Exception ex)
{
_logger.LogTrace(ex, "Failed to delete derived cache file {path}", path);
}
}
private void AddHashedFile(FileCacheEntity fileCache)
{
var normalizedPath = NormalizePrefixedPathKey(fileCache.PrefixedFilePath);
@@ -877,6 +942,83 @@ public sealed class FileCacheManager : IHostedService
}, token).ConfigureAwait(false);
}
private async Task EnforceCacheLimitAsync(long maxBytes, CancellationToken token)
{
if (string.IsNullOrWhiteSpace(CacheFolder) || maxBytes <= 0) return;
await _evictSemaphore.WaitAsync(token).ConfigureAwait(false);
try
{
Directory.CreateDirectory(CacheFolder);
foreach (var tmp in Directory.EnumerateFiles(CacheFolder, "*" + _compressedCacheExtension + ".tmp"))
{
try { File.Delete(tmp); } catch { /* ignore */ }
}
var files = Directory.EnumerateFiles(CacheFolder, "*" + _compressedCacheExtension, SearchOption.TopDirectoryOnly)
.Select(p => new FileInfo(p))
.Where(fi => fi.Exists)
.OrderBy(fi => fi.LastWriteTimeUtc)
.ToList();
long total = files.Sum(f => f.Length);
if (total <= maxBytes) return;
foreach (var fi in files)
{
token.ThrowIfCancellationRequested();
if (total <= maxBytes) break;
var hash = Path.GetFileNameWithoutExtension(fi.Name);
try
{
var len = fi.Length;
fi.Delete();
total -= len;
_sizeCache.TryRemove(hash, out _);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to evict cache file {file}", fi.FullName);
}
}
}
finally
{
_evictSemaphore.Release();
}
}
private static long GiBToBytes(double gib)
{
if (double.IsNaN(gib) || double.IsInfinity(gib) || gib <= 0)
return 0;
var bytes = gib * 1024d * 1024d * 1024d;
if (bytes >= long.MaxValue) return long.MaxValue;
return (long)Math.Round(bytes, MidpointRounding.AwayFromZero);
}
private void CleanupOrphanCompressedCache()
{
if (string.IsNullOrWhiteSpace(CacheFolder) || !Directory.Exists(CacheFolder))
return;
foreach (var path in Directory.EnumerateFiles(CacheFolder, "*" + _compressedCacheExtension))
{
var hash = Path.GetFileNameWithoutExtension(path);
if (!_fileCaches.ContainsKey(hash))
{
try { File.Delete(path); }
catch (Exception ex) { _logger.LogWarning(ex, "Failed deleting orphan {file}", path); }
}
}
}
public async Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Starting FileCacheManager");
@@ -1060,6 +1202,8 @@ public sealed class FileCacheManager : IHostedService
{
await WriteOutFullCsvAsync(cancellationToken).ConfigureAwait(false);
}
CleanupOrphanCompressedCache();
}
_logger.LogInformation("Started FileCacheManager");

90
LightlessSync/FileCache/TransientResourceManager.cs
View File

@@ -297,7 +297,7 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
private void DalamudUtil_FrameworkUpdate()
{
RefreshPlayerRelatedAddressMap();
_ = Task.Run(() => RefreshPlayerRelatedAddressMap());
lock (_cacheAdditionLock)
{
@@ -306,20 +306,64 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
if (_lastClassJobId != _dalamudUtil.ClassJobId)
{
_lastClassJobId = _dalamudUtil.ClassJobId;
if (SemiTransientResources.TryGetValue(ObjectKind.Pet, out HashSet<string>? value))
{
value?.Clear();
}
PlayerConfig.JobSpecificCache.TryGetValue(_dalamudUtil.ClassJobId, out var jobSpecificData);
SemiTransientResources[ObjectKind.Player] = PlayerConfig.GlobalPersistentCache.Concat(jobSpecificData ?? []).ToHashSet(StringComparer.OrdinalIgnoreCase);
PlayerConfig.JobSpecificPetCache.TryGetValue(_dalamudUtil.ClassJobId, out var petSpecificData);
SemiTransientResources[ObjectKind.Pet] = new HashSet<string>(
petSpecificData ?? [],
StringComparer.OrdinalIgnoreCase);
UpdateClassJobCache();
}
CleanupAbsentObjects();
}
private void RefreshPlayerRelatedAddressMap()
{
var tempMap = new ConcurrentDictionary<nint, GameObjectHandler>();
var updatedFrameAddresses = new ConcurrentDictionary<nint, ObjectKind>();
lock (_playerRelatedLock)
{
foreach (var handler in _playerRelatedPointers)
{
var address = (nint)handler.Address;
if (address != nint.Zero)
{
tempMap[address] = handler;
updatedFrameAddresses[address] = handler.ObjectKind;
}
}
}
_playerRelatedByAddress.Clear();
foreach (var kvp in tempMap)
{
_playerRelatedByAddress[kvp.Key] = kvp.Value;
}
_cachedFrameAddresses.Clear();
foreach (var kvp in updatedFrameAddresses)
{
_cachedFrameAddresses[kvp.Key] = kvp.Value;
}
}
private void UpdateClassJobCache()
{
_lastClassJobId = _dalamudUtil.ClassJobId;
if (SemiTransientResources.TryGetValue(ObjectKind.Pet, out HashSet<string>? value))
{
value?.Clear();
}
PlayerConfig.JobSpecificCache.TryGetValue(_dalamudUtil.ClassJobId, out var jobSpecificData);
SemiTransientResources[ObjectKind.Player] = PlayerConfig.GlobalPersistentCache
.Concat(jobSpecificData ?? [])
.ToHashSet(StringComparer.OrdinalIgnoreCase);
PlayerConfig.JobSpecificPetCache.TryGetValue(_dalamudUtil.ClassJobId, out var petSpecificData);
SemiTransientResources[ObjectKind.Pet] = new HashSet<string>(
petSpecificData ?? [],
StringComparer.OrdinalIgnoreCase);
}
private void CleanupAbsentObjects()
{
foreach (var kind in Enum.GetValues(typeof(ObjectKind)).Cast<ObjectKind>())
{
if (!_cachedFrameAddresses.Any(k => k.Value == kind) && TransientResources.Remove(kind, out _))
@@ -349,26 +393,6 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
_semiTransientResources = null;
}
private void RefreshPlayerRelatedAddressMap()
{
_playerRelatedByAddress.Clear();
var updatedFrameAddresses = new ConcurrentDictionary<nint, ObjectKind>();
lock (_playerRelatedLock)
{
foreach (var handler in _playerRelatedPointers)
{
var address = (nint)handler.Address;
if (address != nint.Zero)
{
_playerRelatedByAddress[address] = handler;
updatedFrameAddresses[address] = handler.ObjectKind;
}
}
}
_cachedFrameAddresses = updatedFrameAddresses;
}
private void HandleActorTracked(ActorObjectService.ActorDescriptor descriptor)
{
if (descriptor.IsInGpose)