From d7c9df54cb5ffb0c3919311b3d8edbeb83211fdb Mon Sep 17 00:00:00 2001 From: azyges <229218900+azyges@users.noreply.github.com> Date: Sat, 11 Oct 2025 08:20:50 +0900 Subject: [PATCH] update cache --- LightlessSync/FileCache/FileCacheManager.cs | 284 ++++++++++++++++---- 1 file changed, 232 insertions(+), 52 deletions(-) diff --git a/LightlessSync/FileCache/FileCacheManager.cs b/LightlessSync/FileCache/FileCacheManager.cs index 94dd658..ed57656 100644 --- a/LightlessSync/FileCache/FileCacheManager.cs +++ b/LightlessSync/FileCache/FileCacheManager.cs @@ -16,6 +16,8 @@ public sealed class FileCacheManager : IHostedService public const string CachePrefix = "{cache}"; public const string CsvSplit = "|"; public const string PenumbraPrefix = "{penumbra}"; + private const int FileCacheVersion = 1; + private const string FileCacheVersionHeaderPrefix = "#lightless-file-cache-version:"; private readonly LightlessConfigService _configService; private readonly LightlessMediator _lightlessMediator; private readonly string _csvPath; @@ -54,6 +56,62 @@ public sealed class FileCacheManager : IHostedService return NormalizeSeparators(prefixedPath).ToLowerInvariant(); } + private static bool TryBuildPrefixedPath(string path, string? baseDirectory, string prefix, out string prefixedPath, out int matchedLength) + { + prefixedPath = string.Empty; + matchedLength = 0; + + if (string.IsNullOrEmpty(path) || string.IsNullOrEmpty(baseDirectory)) + { + return false; + } + + var normalizedPath = NormalizeSeparators(path).ToLowerInvariant(); + var normalizedBase = NormalizeSeparators(baseDirectory).TrimEnd('\\').ToLowerInvariant(); + + if (!normalizedPath.StartsWith(normalizedBase, StringComparison.Ordinal)) + { + return false; + } + + if (normalizedPath.Length > normalizedBase.Length) + { + if (normalizedPath[normalizedBase.Length] != '\\') + { + return false; + } + + prefixedPath = prefix + normalizedPath.Substring(normalizedBase.Length); + } + else + { + prefixedPath = prefix; + } + + prefixedPath = prefixedPath.Replace("\\\\", "\\", StringComparison.Ordinal); + matchedLength = normalizedBase.Length; + return true; + } + + private static string BuildVersionHeader() => $"{FileCacheVersionHeaderPrefix}{FileCacheVersion}"; + + private static bool TryParseVersionHeader(string? line, out int version) + { + version = 0; + if (string.IsNullOrWhiteSpace(line)) + { + return false; + } + + if (!line.StartsWith(FileCacheVersionHeaderPrefix, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var versionSpan = line.AsSpan(FileCacheVersionHeaderPrefix.Length); + return int.TryParse(versionSpan, NumberStyles.Integer, CultureInfo.InvariantCulture, out version); + } + private string NormalizeToPrefixedPath(string path) { if (string.IsNullOrEmpty(path)) return string.Empty; @@ -66,27 +124,25 @@ public sealed class FileCacheManager : IHostedService return NormalizePrefixedPathKey(normalized); } - var penumbraDir = _ipcManager.Penumbra.ModDirectory; - if (!string.IsNullOrEmpty(penumbraDir)) + string? chosenPrefixed = null; + var chosenLength = -1; + + if (TryBuildPrefixedPath(normalized, _ipcManager.Penumbra.ModDirectory, PenumbraPrefix, out var penumbraPrefixed, out var penumbraMatch)) { - var normalizedPenumbra = NormalizeSeparators(penumbraDir); - var replacement = normalizedPenumbra.EndsWith("\\", StringComparison.Ordinal) - ? PenumbraPrefix + "\\" - : PenumbraPrefix; - normalized = normalized.Replace(normalizedPenumbra, replacement, StringComparison.OrdinalIgnoreCase); + chosenPrefixed = penumbraPrefixed; + chosenLength = penumbraMatch; } - var cacheFolder = _configService.Current.CacheFolder; - if (!string.IsNullOrEmpty(cacheFolder)) + if (TryBuildPrefixedPath(normalized, _configService.Current.CacheFolder, CachePrefix, out var cachePrefixed, out var cacheMatch)) { - var normalizedCache = NormalizeSeparators(cacheFolder); - var replacement = normalizedCache.EndsWith("\\", StringComparison.Ordinal) - ? CachePrefix + "\\" - : CachePrefix; - normalized = normalized.Replace(normalizedCache, replacement, StringComparison.OrdinalIgnoreCase); + if (cacheMatch > chosenLength) + { + chosenPrefixed = cachePrefixed; + chosenLength = cacheMatch; + } } - return NormalizePrefixedPathKey(normalized); + return NormalizePrefixedPathKey(chosenPrefixed ?? normalized); } public FileCacheEntity? CreateCacheEntry(string path) @@ -94,7 +150,9 @@ public sealed class FileCacheManager : IHostedService FileInfo fi = new(path); if (!fi.Exists) return null; _logger.LogTrace("Creating cache entry for {path}", path); - return CreateFileEntity(_configService.Current.CacheFolder.ToLowerInvariant(), CachePrefix, fi); + var cacheFolder = _configService.Current.CacheFolder; + if (string.IsNullOrEmpty(cacheFolder)) return null; + return CreateFileEntity(cacheFolder, CachePrefix, fi); } public FileCacheEntity? CreateFileEntry(string path) @@ -102,14 +160,18 @@ public sealed class FileCacheManager : IHostedService FileInfo fi = new(path); if (!fi.Exists) return null; _logger.LogTrace("Creating file entry for {path}", path); - return CreateFileEntity(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), PenumbraPrefix, fi); + var modDirectory = _ipcManager.Penumbra.ModDirectory; + if (string.IsNullOrEmpty(modDirectory)) return null; + return CreateFileEntity(modDirectory, PenumbraPrefix, fi); } private FileCacheEntity? CreateFileEntity(string directory, string prefix, FileInfo fi) { - var fullName = fi.FullName.ToLowerInvariant(); - if (!fullName.Contains(directory, StringComparison.Ordinal)) return null; - string prefixedPath = fullName.Replace(directory, prefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal); + if (!TryBuildPrefixedPath(fi.FullName, directory, prefix, out var prefixedPath, out _)) + { + return null; + } + return CreateFileCacheEntity(fi, prefixedPath); } @@ -367,6 +429,7 @@ public sealed class FileCacheManager : IHostedService lock (_fileWriteLock) { StringBuilder sb = new(); + sb.AppendLine(BuildVersionHeader()); foreach (var entry in _fileCaches.Values.SelectMany(k => k.Values).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase)) { sb.AppendLine(entry.CsvEntry); @@ -389,6 +452,53 @@ public sealed class FileCacheManager : IHostedService } } + private void EnsureCsvHeaderLocked() + { + if (!File.Exists(_csvPath)) + { + return; + } + + string[] existingLines = File.ReadAllLines(_csvPath); + if (existingLines.Length > 0 && TryParseVersionHeader(existingLines[0], out var existingVersion) && existingVersion == FileCacheVersion) + { + return; + } + + StringBuilder rebuilt = new(); + rebuilt.AppendLine(BuildVersionHeader()); + foreach (var line in existingLines) + { + if (TryParseVersionHeader(line, out _)) + { + continue; + } + + if (!string.IsNullOrEmpty(line)) + { + rebuilt.AppendLine(line); + } + } + + File.WriteAllText(_csvPath, rebuilt.ToString()); + } + + private void BackupUnsupportedCache(string suffix) + { + var sanitizedSuffix = string.IsNullOrWhiteSpace(suffix) ? "unsupported" : $"{suffix}.unsupported"; + var backupPath = _csvPath + "." + sanitizedSuffix; + + try + { + File.Move(_csvPath, backupPath, overwrite: true); + _logger.LogWarning("Backed up unsupported file cache to {path}", backupPath); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to back up unsupported file cache to {path}", backupPath); + } + } + internal FileCacheEntity MigrateFileHashToExtension(FileCacheEntity fileCache, string ext) { try @@ -427,7 +537,15 @@ public sealed class FileCacheManager : IHostedService AddHashedFile(entity); lock (_fileWriteLock) { - File.AppendAllLines(_csvPath, new[] { entity.CsvEntry }); + if (!File.Exists(_csvPath)) + { + File.WriteAllLines(_csvPath, new[] { BuildVersionHeader(), entity.CsvEntry }); + } + else + { + EnsureCsvHeaderLocked(); + File.AppendAllLines(_csvPath, new[] { entity.CsvEntry }); + } } var result = GetFileCacheByPath(fileInfo.FullName); _logger.LogTrace("Creating cache entity for {name} success: {success}", fileInfo.FullName, (result != null)); @@ -546,49 +664,111 @@ public sealed class FileCacheManager : IHostedService _logger.LogWarning("Could not load entries from {path}, continuing with empty file cache", _csvPath); } - _logger.LogInformation("Found {amount} files in {path}", entries.Length, _csvPath); + bool rewriteRequired = false; + bool parseEntries = entries.Length > 0; + int startIndex = 0; - Dictionary processedFiles = new(StringComparer.OrdinalIgnoreCase); - foreach (var entry in entries) + if (entries.Length > 0) { - var splittedEntry = entry.Split(CsvSplit, StringSplitOptions.None); - try + var headerLine = entries[0]; + var hasHeader = !string.IsNullOrEmpty(headerLine) && + headerLine.StartsWith(FileCacheVersionHeaderPrefix, StringComparison.OrdinalIgnoreCase); + + if (hasHeader) { - var hash = splittedEntry[0]; - if (hash.Length != 40) throw new InvalidOperationException("Expected Hash length of 40, received " + hash.Length); - var path = splittedEntry[1]; - var time = splittedEntry[2]; - - if (processedFiles.ContainsKey(path)) + if (!TryParseVersionHeader(headerLine, out var parsedVersion)) { - _logger.LogWarning("Already processed {file}, ignoring", path); - continue; + _logger.LogWarning("Failed to parse file cache version header \"{header}\". Backing up existing cache.", headerLine); + BackupUnsupportedCache("invalid-version"); + parseEntries = false; + rewriteRequired = true; + entries = Array.Empty(); } - - processedFiles.Add(path, value: true); - - long size = -1; - long compressed = -1; - if (splittedEntry.Length > 3) + else if (parsedVersion != FileCacheVersion) { - if (long.TryParse(splittedEntry[3], CultureInfo.InvariantCulture, out long result)) - { - size = result; - } - if (long.TryParse(splittedEntry[4], CultureInfo.InvariantCulture, out long resultCompressed)) - { - compressed = resultCompressed; - } + _logger.LogWarning("Unsupported file cache version {version} detected (expected {expected}). Backing up existing cache.", parsedVersion, FileCacheVersion); + BackupUnsupportedCache($"v{parsedVersion}"); + parseEntries = false; + rewriteRequired = true; + entries = Array.Empty(); + } + else + { + startIndex = 1; } - AddHashedFile(ReplacePathPrefixes(new FileCacheEntity(hash, path, time, size, compressed))); } - catch (Exception ex) + else if (entries.Length > 0) { - _logger.LogWarning(ex, "Failed to initialize entry {entry}, ignoring", entry); + _logger.LogInformation("File cache missing version header, scheduling rewrite."); + rewriteRequired = true; } } - if (processedFiles.Count != entries.Length) + var totalEntries = Math.Max(0, entries.Length - startIndex); + Dictionary processedFiles = new(StringComparer.OrdinalIgnoreCase); + + if (parseEntries && totalEntries > 0) + { + _logger.LogInformation("Found {amount} files in {path}", totalEntries, _csvPath); + + for (var index = startIndex; index < entries.Length; index++) + { + var entry = entries[index]; + if (string.IsNullOrWhiteSpace(entry)) + { + continue; + } + + var splittedEntry = entry.Split(CsvSplit, StringSplitOptions.None); + try + { + var hash = splittedEntry[0]; + if (hash.Length != 40) + throw new InvalidOperationException("Expected Hash length of 40, received " + hash.Length); + var path = splittedEntry[1]; + var time = splittedEntry[2]; + + if (processedFiles.ContainsKey(path)) + { + _logger.LogWarning("Already processed {file}, ignoring", path); + continue; + } + + processedFiles.Add(path, value: true); + + long size = -1; + long compressed = -1; + if (splittedEntry.Length > 3) + { + if (long.TryParse(splittedEntry[3], CultureInfo.InvariantCulture, out long result)) + { + size = result; + } + if (splittedEntry.Length > 4 && + long.TryParse(splittedEntry[4], CultureInfo.InvariantCulture, out long resultCompressed)) + { + compressed = resultCompressed; + } + } + AddHashedFile(ReplacePathPrefixes(new FileCacheEntity(hash, path, time, size, compressed))); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to initialize entry {entry}, ignoring", entry); + } + } + + if (processedFiles.Count != totalEntries) + { + rewriteRequired = true; + } + } + else if (!parseEntries && entries.Length > 0) + { + _logger.LogInformation("Skipping existing file cache entries due to incompatible version."); + } + + if (rewriteRequired) { WriteOutFullCsv(); }