1.11.10 (#31)

Co-authored-by: CakeAndBanana <admin@cakeandbanana.nl>
Co-authored-by: defnotken <itsdefnotken@gmail.com>
Reviewed-on: #31

LightlessSync/FileCache/FileCacheManager.cs

@@ -21,7 +21,7 @@ public sealed class FileCacheManager : IHostedService
     private readonly string _csvPath;
     private readonly ConcurrentDictionary<string, List<FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
     private readonly SemaphoreSlim _getCachesByPathsSemaphore = new(1, 1);
-    private readonly object _fileWriteLock = new();
+    private readonly Lock _fileWriteLock = new();
     private readonly IpcManager _ipcManager;
     private readonly ILogger<FileCacheManager> _logger;
     public string CacheFolder => _configService.Current.CacheFolder;
@@ -42,10 +42,7 @@ public sealed class FileCacheManager : IHostedService
         FileInfo fi = new(path);
         if (!fi.Exists) return null;
         _logger.LogTrace("Creating cache entry for {path}", path);
-        var fullName = fi.FullName.ToLowerInvariant();
-        if (!fullName.Contains(_configService.Current.CacheFolder.ToLowerInvariant(), StringComparison.Ordinal)) return null;
-        string prefixedPath = fullName.Replace(_configService.Current.CacheFolder.ToLowerInvariant(), CachePrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
-        return CreateFileCacheEntity(fi, prefixedPath);
+        return CreateFileEntity(_configService.Current.CacheFolder.ToLowerInvariant(), CachePrefix, fi);
     }
 
     public FileCacheEntity? CreateFileEntry(string path)
@@ -53,9 +50,14 @@ public sealed class FileCacheManager : IHostedService
         FileInfo fi = new(path);
         if (!fi.Exists) return null;
         _logger.LogTrace("Creating file entry for {path}", path);
+        return CreateFileEntity(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), PenumbraPrefix, fi);
+    }
+
+    private FileCacheEntity? CreateFileEntity(string directory, string prefix, FileInfo fi)
+    {
         var fullName = fi.FullName.ToLowerInvariant();
-        if (!fullName.Contains(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), StringComparison.Ordinal)) return null;
-        string prefixedPath = fullName.Replace(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), PenumbraPrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
+        if (!fullName.Contains(_configService.Current.CacheFolder.ToLowerInvariant(), StringComparison.Ordinal)) return null;
+        string prefixedPath = fullName.Replace(directory, prefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
         return CreateFileCacheEntity(fi, prefixedPath);
     }
 
@@ -66,7 +68,7 @@ public sealed class FileCacheManager : IHostedService
         List<FileCacheEntity> output = [];
         if (_fileCaches.TryGetValue(hash, out var fileCacheEntities))
         {
-            foreach (var fileCache in fileCacheEntities.Where(c => ignoreCacheEntries ? !c.IsCacheEntry : true).ToList())
+            foreach (var fileCache in fileCacheEntities.Where(c => !ignoreCacheEntries || !c.IsCacheEntry).ToList())
             {
                 if (!validate) output.Add(fileCache);
                 else
@@ -106,7 +108,7 @@ public sealed class FileCacheManager : IHostedService
                 var computedHash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
                 if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal))
                 {
-                    _logger.LogInformation("Failed to validate {file}, got hash {hash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
+                    _logger.LogInformation("Failed to validate {file}, got hash {computedHash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
                     brokenEntities.Add(fileCache);
                 }
             }
@@ -151,7 +153,7 @@ public sealed class FileCacheManager : IHostedService
     {
         if (_fileCaches.TryGetValue(hash, out var hashes))
         {
-            var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix) ? 0 : 1).FirstOrDefault();
+            var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix, StringComparison.Ordinal) ? 0 : 1).FirstOrDefault();
             if (item != null) return GetValidatedFileCache(item);
         }
         return null;
@@ -180,48 +182,66 @@ public sealed class FileCacheManager : IHostedService
 
         try
         {
-            var cleanedPaths = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
-            foreach (var p in paths)
+            var allEntities = _fileCaches.SelectMany(f => f.Value).ToArray();
+
+            var cacheDict = new ConcurrentDictionary<string, FileCacheEntity>(
+                StringComparer.OrdinalIgnoreCase);
+
+            Parallel.ForEach(allEntities, entity =>
+            {
+                cacheDict[entity.PrefixedFilePath] = entity;
+            });
+
+            var cleanedPaths = new ConcurrentDictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+            var seenCleaned = new ConcurrentDictionary<string, byte>(StringComparer.OrdinalIgnoreCase);
+
+            Parallel.ForEach(paths, p =>
             {
                 var cleaned = p.Replace("/", "\\", StringComparison.OrdinalIgnoreCase)
-                    .Replace(_ipcManager.Penumbra.ModDirectory!, _ipcManager.Penumbra.ModDirectory!.EndsWith('\\') ? PenumbraPrefix + '\\' : PenumbraPrefix, StringComparison.OrdinalIgnoreCase)
-                    .Replace(_configService.Current.CacheFolder, _configService.Current.CacheFolder.EndsWith('\\') ? CachePrefix + '\\' : CachePrefix, StringComparison.OrdinalIgnoreCase)
+                    .Replace(
+                        _ipcManager.Penumbra.ModDirectory!,
+                        _ipcManager.Penumbra.ModDirectory!.EndsWith('\\')
+                            ? PenumbraPrefix + '\\' : PenumbraPrefix,
+                        StringComparison.OrdinalIgnoreCase)
+                    .Replace(
+                        _configService.Current.CacheFolder,
+                        _configService.Current.CacheFolder.EndsWith('\\')
+                            ? CachePrefix + '\\' : CachePrefix,
+                        StringComparison.OrdinalIgnoreCase)
                     .Replace("\\\\", "\\", StringComparison.Ordinal);
 
-                if (!cleanedPaths.ContainsValue(cleaned))
+                if (seenCleaned.TryAdd(cleaned, 0))
                 {
                     _logger.LogDebug("Adding to cleanedPaths: {cleaned}", cleaned);
                     cleanedPaths[p] = cleaned;
-                } else
-                {
-                    _logger.LogWarning("Duplicated found: {cleaned}", cleaned);
                 }
-            }
+                else
+                {
+                    _logger.LogWarning("Duplicate found: {cleaned}", cleaned);
+                }
+            });
 
-            Dictionary<string, FileCacheEntity?> result = new(StringComparer.OrdinalIgnoreCase);
+            var result = new ConcurrentDictionary<string, FileCacheEntity?>(StringComparer.OrdinalIgnoreCase);
 
-            var dict = _fileCaches.SelectMany(f => f.Value).Distinct()
-                .ToDictionary(d => d.PrefixedFilePath, d => d, StringComparer.OrdinalIgnoreCase);
-
-            foreach (var entry in cleanedPaths)
+            Parallel.ForEach(cleanedPaths, entry =>
             {
                 _logger.LogDebug("Checking if in cache: {path}", entry.Value);
 
-                if (dict.TryGetValue(entry.Value, out var entity))
+                if (cacheDict.TryGetValue(entry.Value, out var entity))
                 {
                     var validatedCache = GetValidatedFileCache(entity);
-                    result.Add(entry.Key, validatedCache);
+                    result[entry.Key] = validatedCache;
                 }
                 else
                 {
                     if (!entry.Value.Contains(CachePrefix, StringComparison.Ordinal))
-                        result.Add(entry.Key, CreateFileEntry(entry.Key));
+                        result[entry.Key] = CreateFileEntry(entry.Key);
                     else
-                        result.Add(entry.Key, CreateCacheEntry(entry.Key));
+                        result[entry.Key] = CreateCacheEntry(entry.Key);
                 }
-            }
+            });
 
-            return result;
+            return new Dictionary<string, FileCacheEntity?>(result, StringComparer.OrdinalIgnoreCase);
         }
         finally
         {
@@ -450,7 +470,7 @@ public sealed class FileCacheManager : IHostedService
                 {
                     attempts++;
                     _logger.LogWarning(ex, "Could not open {file}, trying again", _csvPath);
-                    Thread.Sleep(100);
+                    Task.Delay(100, cancellationToken);
                 }
             }