Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 2 Actions Packages Projects Releases 51 Wiki Activity

update cache

Browse Source
This commit is contained in:
azyges
2025-10-11 08:20:50 +09:00
parent 37ec0961d9
commit d7c9df54cb
1 changed files with 232 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

284
LightlessSync/FileCache/FileCacheManager.cs
View File

@@ -16,6 +16,8 @@ public sealed class FileCacheManager : IHostedService
public const string CachePrefix = "{cache}";
public const string CsvSplit = "|";
public const string PenumbraPrefix = "{penumbra}";
private const int FileCacheVersion = 1;
private const string FileCacheVersionHeaderPrefix = "#lightless-file-cache-version:";
private readonly LightlessConfigService _configService;
private readonly LightlessMediator _lightlessMediator;
private readonly string _csvPath;
@@ -54,6 +56,62 @@ public sealed class FileCacheManager : IHostedService
return NormalizeSeparators(prefixedPath).ToLowerInvariant();
}
private static bool TryBuildPrefixedPath(string path, string? baseDirectory, string prefix, out string prefixedPath, out int matchedLength)
{
prefixedPath = string.Empty;
matchedLength = 0;
if (string.IsNullOrEmpty(path) || string.IsNullOrEmpty(baseDirectory))
{
return false;
}
var normalizedPath = NormalizeSeparators(path).ToLowerInvariant();
var normalizedBase = NormalizeSeparators(baseDirectory).TrimEnd('\\').ToLowerInvariant();
if (!normalizedPath.StartsWith(normalizedBase, StringComparison.Ordinal))
{
return false;
}
if (normalizedPath.Length > normalizedBase.Length)
{
if (normalizedPath[normalizedBase.Length] != '\\')
{
return false;
}
prefixedPath = prefix + normalizedPath.Substring(normalizedBase.Length);
}
else
{
prefixedPath = prefix;
}
prefixedPath = prefixedPath.Replace("\\\\", "\\", StringComparison.Ordinal);
matchedLength = normalizedBase.Length;
return true;
}
private static string BuildVersionHeader() => $"{FileCacheVersionHeaderPrefix}{FileCacheVersion}";
private static bool TryParseVersionHeader(string? line, out int version)
{
version = 0;
if (string.IsNullOrWhiteSpace(line))
{
return false;
}
if (!line.StartsWith(FileCacheVersionHeaderPrefix, StringComparison.OrdinalIgnoreCase))
{
return false;
}
var versionSpan = line.AsSpan(FileCacheVersionHeaderPrefix.Length);
return int.TryParse(versionSpan, NumberStyles.Integer, CultureInfo.InvariantCulture, out version);
}
private string NormalizeToPrefixedPath(string path)
{
if (string.IsNullOrEmpty(path)) return string.Empty;
@@ -66,27 +124,25 @@ public sealed class FileCacheManager : IHostedService
return NormalizePrefixedPathKey(normalized);
}
var penumbraDir = _ipcManager.Penumbra.ModDirectory;
if (!string.IsNullOrEmpty(penumbraDir))
string? chosenPrefixed = null;
var chosenLength = -1;
if (TryBuildPrefixedPath(normalized, _ipcManager.Penumbra.ModDirectory, PenumbraPrefix, out var penumbraPrefixed, out var penumbraMatch))
{
var normalizedPenumbra = NormalizeSeparators(penumbraDir);
var replacement = normalizedPenumbra.EndsWith("\\", StringComparison.Ordinal)
? PenumbraPrefix + "\\"
: PenumbraPrefix;
normalized = normalized.Replace(normalizedPenumbra, replacement, StringComparison.OrdinalIgnoreCase);
chosenPrefixed = penumbraPrefixed;
chosenLength = penumbraMatch;
}
var cacheFolder = _configService.Current.CacheFolder;
if (!string.IsNullOrEmpty(cacheFolder))
if (TryBuildPrefixedPath(normalized, _configService.Current.CacheFolder, CachePrefix, out var cachePrefixed, out var cacheMatch))
{
var normalizedCache = NormalizeSeparators(cacheFolder);
var replacement = normalizedCache.EndsWith("\\", StringComparison.Ordinal)
? CachePrefix + "\\"
: CachePrefix;
normalized = normalized.Replace(normalizedCache, replacement, StringComparison.OrdinalIgnoreCase);
if (cacheMatch > chosenLength)
{
chosenPrefixed = cachePrefixed;
chosenLength = cacheMatch;
}
}
return NormalizePrefixedPathKey(normalized);
return NormalizePrefixedPathKey(chosenPrefixed ?? normalized);
}
public FileCacheEntity? CreateCacheEntry(string path)
@@ -94,7 +150,9 @@ public sealed class FileCacheManager : IHostedService
FileInfo fi = new(path);
if (!fi.Exists) return null;
_logger.LogTrace("Creating cache entry for {path}", path);
return CreateFileEntity(_configService.Current.CacheFolder.ToLowerInvariant(), CachePrefix, fi);
var cacheFolder = _configService.Current.CacheFolder;
if (string.IsNullOrEmpty(cacheFolder)) return null;
return CreateFileEntity(cacheFolder, CachePrefix, fi);
}
public FileCacheEntity? CreateFileEntry(string path)
@@ -102,14 +160,18 @@ public sealed class FileCacheManager : IHostedService
FileInfo fi = new(path);
if (!fi.Exists) return null;
_logger.LogTrace("Creating file entry for {path}", path);
return CreateFileEntity(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), PenumbraPrefix, fi);
var modDirectory = _ipcManager.Penumbra.ModDirectory;
if (string.IsNullOrEmpty(modDirectory)) return null;
return CreateFileEntity(modDirectory, PenumbraPrefix, fi);
}
private FileCacheEntity? CreateFileEntity(string directory, string prefix, FileInfo fi)
{
var fullName = fi.FullName.ToLowerInvariant();
if (!fullName.Contains(directory, StringComparison.Ordinal)) return null;
string prefixedPath = fullName.Replace(directory, prefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
if (!TryBuildPrefixedPath(fi.FullName, directory, prefix, out var prefixedPath, out _))
{
return null;
}
return CreateFileCacheEntity(fi, prefixedPath);
}
@@ -367,6 +429,7 @@ public sealed class FileCacheManager : IHostedService
lock (_fileWriteLock)
{
StringBuilder sb = new();
sb.AppendLine(BuildVersionHeader());
foreach (var entry in _fileCaches.Values.SelectMany(k => k.Values).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase))
{
sb.AppendLine(entry.CsvEntry);
@@ -389,6 +452,53 @@ public sealed class FileCacheManager : IHostedService
}
}
private void EnsureCsvHeaderLocked()
{
if (!File.Exists(_csvPath))
{
return;
}
string[] existingLines = File.ReadAllLines(_csvPath);
if (existingLines.Length > 0 && TryParseVersionHeader(existingLines[0], out var existingVersion) && existingVersion == FileCacheVersion)
{
return;
}
StringBuilder rebuilt = new();
rebuilt.AppendLine(BuildVersionHeader());
foreach (var line in existingLines)
{
if (TryParseVersionHeader(line, out _))
{
continue;
}
if (!string.IsNullOrEmpty(line))
{
rebuilt.AppendLine(line);
}
}
File.WriteAllText(_csvPath, rebuilt.ToString());
}
private void BackupUnsupportedCache(string suffix)
{
var sanitizedSuffix = string.IsNullOrWhiteSpace(suffix) ? "unsupported" : $"{suffix}.unsupported";
var backupPath = _csvPath + "." + sanitizedSuffix;
try
{
File.Move(_csvPath, backupPath, overwrite: true);
_logger.LogWarning("Backed up unsupported file cache to {path}", backupPath);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to back up unsupported file cache to {path}", backupPath);
}
}
internal FileCacheEntity MigrateFileHashToExtension(FileCacheEntity fileCache, string ext)
{
try
@@ -427,7 +537,15 @@ public sealed class FileCacheManager : IHostedService
AddHashedFile(entity);
lock (_fileWriteLock)
{
File.AppendAllLines(_csvPath, new[] { entity.CsvEntry });
if (!File.Exists(_csvPath))
{
File.WriteAllLines(_csvPath, new[] { BuildVersionHeader(), entity.CsvEntry });
}
else
{
EnsureCsvHeaderLocked();
File.AppendAllLines(_csvPath, new[] { entity.CsvEntry });
}
}
var result = GetFileCacheByPath(fileInfo.FullName);
_logger.LogTrace("Creating cache entity for {name} success: {success}", fileInfo.FullName, (result != null));
@@ -546,49 +664,111 @@ public sealed class FileCacheManager : IHostedService
_logger.LogWarning("Could not load entries from {path}, continuing with empty file cache", _csvPath);
}
_logger.LogInformation("Found {amount} files in {path}", entries.Length, _csvPath);
bool rewriteRequired = false;
bool parseEntries = entries.Length > 0;
int startIndex = 0;
Dictionary<string, bool> processedFiles = new(StringComparer.OrdinalIgnoreCase);
foreach (var entry in entries)
if (entries.Length > 0)
{
var splittedEntry = entry.Split(CsvSplit, StringSplitOptions.None);
try
var headerLine = entries[0];
var hasHeader = !string.IsNullOrEmpty(headerLine) &&
headerLine.StartsWith(FileCacheVersionHeaderPrefix, StringComparison.OrdinalIgnoreCase);
if (hasHeader)
{
var hash = splittedEntry[0];
if (hash.Length != 40) throw new InvalidOperationException("Expected Hash length of 40, received " + hash.Length);
var path = splittedEntry[1];
var time = splittedEntry[2];
if (processedFiles.ContainsKey(path))
if (!TryParseVersionHeader(headerLine, out var parsedVersion))
{
_logger.LogWarning("Already processed {file}, ignoring", path);
continue;
_logger.LogWarning("Failed to parse file cache version header \"{header}\". Backing up existing cache.", headerLine);
BackupUnsupportedCache("invalid-version");
parseEntries = false;
rewriteRequired = true;
entries = Array.Empty<string>();
}
processedFiles.Add(path, value: true);
long size = -1;
long compressed = -1;
if (splittedEntry.Length > 3)
else if (parsedVersion != FileCacheVersion)
{
if (long.TryParse(splittedEntry[3], CultureInfo.InvariantCulture, out long result))
{
size = result;
}
if (long.TryParse(splittedEntry[4], CultureInfo.InvariantCulture, out long resultCompressed))
{
compressed = resultCompressed;
}
_logger.LogWarning("Unsupported file cache version {version} detected (expected {expected}). Backing up existing cache.", parsedVersion, FileCacheVersion);
BackupUnsupportedCache($"v{parsedVersion}");
parseEntries = false;
rewriteRequired = true;
entries = Array.Empty<string>();
}
else
{
startIndex = 1;
}
AddHashedFile(ReplacePathPrefixes(new FileCacheEntity(hash, path, time, size, compressed)));
}
catch (Exception ex)
else if (entries.Length > 0)
{
_logger.LogWarning(ex, "Failed to initialize entry {entry}, ignoring", entry);
_logger.LogInformation("File cache missing version header, scheduling rewrite.");
rewriteRequired = true;
}
}
if (processedFiles.Count != entries.Length)
var totalEntries = Math.Max(0, entries.Length - startIndex);
Dictionary<string, bool> processedFiles = new(StringComparer.OrdinalIgnoreCase);
if (parseEntries && totalEntries > 0)
{
_logger.LogInformation("Found {amount} files in {path}", totalEntries, _csvPath);
for (var index = startIndex; index < entries.Length; index++)
{
var entry = entries[index];
if (string.IsNullOrWhiteSpace(entry))
{
continue;
}
var splittedEntry = entry.Split(CsvSplit, StringSplitOptions.None);
try
{
var hash = splittedEntry[0];
if (hash.Length != 40)
throw new InvalidOperationException("Expected Hash length of 40, received " + hash.Length);
var path = splittedEntry[1];
var time = splittedEntry[2];
if (processedFiles.ContainsKey(path))
{
_logger.LogWarning("Already processed {file}, ignoring", path);
continue;
}
processedFiles.Add(path, value: true);
long size = -1;
long compressed = -1;
if (splittedEntry.Length > 3)
{
if (long.TryParse(splittedEntry[3], CultureInfo.InvariantCulture, out long result))
{
size = result;
}
if (splittedEntry.Length > 4 &&
long.TryParse(splittedEntry[4], CultureInfo.InvariantCulture, out long resultCompressed))
{
compressed = resultCompressed;
}
}
AddHashedFile(ReplacePathPrefixes(new FileCacheEntity(hash, path, time, size, compressed)));
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to initialize entry {entry}, ignoring", entry);
}
}
if (processedFiles.Count != totalEntries)
{
rewriteRequired = true;
}
}
else if (!parseEntries && entries.Length > 0)
{
_logger.LogInformation("Skipping existing file cache entries due to incompatible version.");
}
if (rewriteRequired)
{
WriteOutFullCsv();
}