2.0.0 (#92)

2.0.0 Changes:

- Reworked shell finder UI with compact or list view with profile tags showing with the listing, allowing moderators to broadcast the syncshell as well to have it be used more.
- Reworked user list in syncshell admin screen to have filter visible and moved away from table to its own thing, allowing to copy uid/note/alias when clicking on the name.
- Reworked download bars and download box to make it look more modern, removed the jitter around, so it shouldn't vibrate around much.
- Chat has been added to the top menu, working in Zone or in Syncshells to be used there.
- Paired system has been revamped to make pausing and unpausing faster, and loading people should be faster as well.
- Moved to the internal object table to have faster load times for users; people should load in faster
- Compactor is running on a multi-threaded level instead of single-threaded; this should increase the speed of compacting files
- Nameplate Service has been reworked so it wouldn't use the nameplate handler anymore.
- Files can be resized when downloading to reduce load on users if they aren't compressed. (can be toggled to resize all).
- Penumbra Collections are now only made when people are visible, reducing the load on boot-up when having many syncshells in your list.
- Lightfinder plates have been moved away from using Nameplates, but will use an overlay.
- Main UI has been changed a bit with a gradient, and on hover will glow up now.
- Reworked Profile UI for Syncshell and Users to be more user-facing with more customizable items.
- Reworked Settings UI to look more modern.
- Performance should be better due to new systems that would dispose of the collections and better caching of items.

Co-authored-by: defnotken <itsdefnotken@gmail.com>
Co-authored-by: azyges <aaaaaa@aaa.aaa>
Co-authored-by: choco <choco@patat.nl>
Co-authored-by: cake <admin@cakeandbanana.nl>
Co-authored-by: Minmoose <KennethBohr@outlook.com>
Reviewed-on: #92

LightlessSync/FileCache/FileCacheManager.cs

@@ -18,6 +18,7 @@ public sealed class FileCacheManager : IHostedService
     public const string PenumbraPrefix = "{penumbra}";
     private const int FileCacheVersion = 1;
     private const string FileCacheVersionHeaderPrefix = "#lightless-file-cache-version:";
+    private readonly SemaphoreSlim _fileWriteSemaphore = new(1, 1);
     private readonly LightlessConfigService _configService;
     private readonly LightlessMediator _lightlessMediator;
     private readonly string _csvPath;
@@ -41,11 +42,8 @@ public sealed class FileCacheManager : IHostedService
 
     private string CsvBakPath => _csvPath + ".bak";
 
-    private static string NormalizeSeparators(string path)
-    {
-        return path.Replace("/", "\\", StringComparison.Ordinal)
+    private static string NormalizeSeparators(string path) => path.Replace("/", "\\", StringComparison.Ordinal)
             .Replace("\\\\", "\\", StringComparison.Ordinal);
-    }
 
     private static string NormalizePrefixedPathKey(string prefixedPath)
     {
@@ -134,13 +132,9 @@ public sealed class FileCacheManager : IHostedService
             chosenLength = penumbraMatch;
         }
 
-        if (TryBuildPrefixedPath(normalized, _configService.Current.CacheFolder, CachePrefix, out var cachePrefixed, out var cacheMatch))
+        if (TryBuildPrefixedPath(normalized, _configService.Current.CacheFolder, CachePrefix, out var cachePrefixed, out var cacheMatch) && cacheMatch > chosenLength)
         {
-            if (cacheMatch > chosenLength)
-            {
-                chosenPrefixed = cachePrefixed;
-                chosenLength = cacheMatch;
-            }
+            chosenPrefixed = cachePrefixed;
         }
 
         return NormalizePrefixedPathKey(chosenPrefixed ?? normalized);
@@ -176,27 +170,53 @@ public sealed class FileCacheManager : IHostedService
         return CreateFileCacheEntity(fi, prefixedPath);
     }
 
-    public List<FileCacheEntity> GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v.Values.Where(e => e != null)).ToList();
+    public List<FileCacheEntity> GetAllFileCaches() => [.. _fileCaches.Values.SelectMany(v => v.Values.Where(e => e != null))];
 
     public List<FileCacheEntity> GetAllFileCachesByHash(string hash, bool ignoreCacheEntries = false, bool validate = true)
     {
-        List<FileCacheEntity> output = [];
-        if (_fileCaches.TryGetValue(hash, out var fileCacheEntities))
+        var output = new List<FileCacheEntity>();
+
+        if (!_fileCaches.TryGetValue(hash, out var fileCacheEntities))
+            return output;
+
+        foreach (var fileCache in fileCacheEntities.Values
+                     .Where(c => !ignoreCacheEntries || !c.IsCacheEntry))
         {
-            foreach (var fileCache in fileCacheEntities.Values.Where(c => !ignoreCacheEntries || !c.IsCacheEntry).ToList())
+            if (!validate)
             {
-                if (!validate)
-                {
-                    output.Add(fileCache);
-                }
-                else
-                {
-                    var validated = GetValidatedFileCache(fileCache);
-                    if (validated != null)
-                    {
-                        output.Add(validated);
-                    }
-                }
+                output.Add(fileCache);
+                continue;
+            }
+
+            var validated = GetValidatedFileCache(fileCache);
+            if (validated != null)
+                output.Add(validated);
+        }
+
+        return output;
+    }
+
+    public async Task<List<FileCacheEntity>> GetAllFileCachesByHashAsync(string hash, bool ignoreCacheEntries = false, bool validate = true,CancellationToken token = default)
+    {
+        var output = new List<FileCacheEntity>();
+
+        if (!_fileCaches.TryGetValue(hash, out var fileCacheEntities))
+            return output;
+
+        foreach (var fileCache in fileCacheEntities.Values.Where(c => !ignoreCacheEntries || !c.IsCacheEntry))
+        {
+            token.ThrowIfCancellationRequested();
+
+            if (!validate)
+            {
+                output.Add(fileCache);
+            }
+            else
+            {
+                var validated = await GetValidatedFileCacheAsync(fileCache, token).ConfigureAwait(false);
+
+                if (validated != null)
+                    output.Add(validated);
             }
         }
 
@@ -238,10 +258,11 @@ public sealed class FileCacheManager : IHostedService
                     return;
                 }
 
+                var algo = Crypto.DetectAlgo(fileCache.Hash);
                 string computedHash;
                 try
                 {
-                    computedHash = await Crypto.GetFileHashAsync(fileCache.ResolvedFilepath, token).ConfigureAwait(false);
+                    computedHash = await Crypto.ComputeFileHashAsync(fileCache.ResolvedFilepath, Crypto.HashAlgo.Sha1, token).ConfigureAwait(false);
                 }
                 catch (Exception ex)
                 {
@@ -253,8 +274,8 @@ public sealed class FileCacheManager : IHostedService
                 if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal))
                 {
                     _logger.LogInformation(
-                        "Hash mismatch: {file} (got {computedHash}, expected {expected})",
-                        fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
+                        "Hash mismatch: {file} (got {computedHash}, expected {expected} : hash {hash})",
+                        fileCache.ResolvedFilepath, computedHash, fileCache.Hash, algo);
 
                     brokenEntities.Add(fileCache);
                 }
@@ -434,7 +455,7 @@ public sealed class FileCacheManager : IHostedService
             var fi = new FileInfo(fileCache.ResolvedFilepath);
             fileCache.Size = fi.Length;
             fileCache.CompressedSize = null;
-            fileCache.Hash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
+            fileCache.Hash = Crypto.ComputeFileHash(fileCache.ResolvedFilepath, Crypto.HashAlgo.Sha1);
             fileCache.LastModifiedDateTicks = fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
         }
         RemoveHashedFile(oldHash, prefixedPath);
@@ -485,6 +506,44 @@ public sealed class FileCacheManager : IHostedService
         }
     }
 
+    public async Task WriteOutFullCsvAsync(CancellationToken cancellationToken = default)
+    {
+        await _fileWriteSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
+
+        try
+        {
+            var sb = new StringBuilder();
+            sb.AppendLine(BuildVersionHeader());
+
+            foreach (var entry in _fileCaches.Values
+                         .SelectMany(k => k.Values)
+                         .OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase))
+            {
+                sb.AppendLine(entry.CsvEntry);
+            }
+
+            if (File.Exists(_csvPath))
+            {
+                File.Copy(_csvPath, CsvBakPath, overwrite: true);
+            }
+
+            try
+            {
+                await File.WriteAllTextAsync(_csvPath, sb.ToString(), cancellationToken).ConfigureAwait(false);
+
+                File.Delete(CsvBakPath);
+            }
+            catch
+            {
+                await File.WriteAllTextAsync(CsvBakPath, sb.ToString(), cancellationToken).ConfigureAwait(false);
+            }
+        }
+        finally
+        {
+            _fileWriteSemaphore.Release();
+        }
+    }
+
     private void EnsureCsvHeaderLocked()
     {
         if (!File.Exists(_csvPath))
@@ -577,7 +636,7 @@ public sealed class FileCacheManager : IHostedService
 
     private FileCacheEntity? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath, string? hash = null)
     {
-        hash ??= Crypto.GetFileHash(fileInfo.FullName);
+        hash ??= Crypto.ComputeFileHash(fileInfo.FullName, Crypto.HashAlgo.Sha1);
         var entity = new FileCacheEntity(hash, prefixedPath, fileInfo.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), fileInfo.Length);
         entity = ReplacePathPrefixes(entity);
         AddHashedFile(entity);
@@ -585,13 +644,13 @@ public sealed class FileCacheManager : IHostedService
         {
             if (!File.Exists(_csvPath))
             {
-                File.WriteAllLines(_csvPath, new[] { BuildVersionHeader(), entity.CsvEntry });
+                File.WriteAllLines(_csvPath, [BuildVersionHeader(), entity.CsvEntry]);
                 _csvHeaderEnsured = true;
             }
             else
             {
                 EnsureCsvHeaderLockedCached();
-                File.AppendAllLines(_csvPath, new[] { entity.CsvEntry });
+                File.AppendAllLines(_csvPath, [entity.CsvEntry]);
             }
         }
         var result = GetFileCacheByPath(fileInfo.FullName);
@@ -602,11 +661,17 @@ public sealed class FileCacheManager : IHostedService
     private FileCacheEntity? GetValidatedFileCache(FileCacheEntity fileCache)
     {
         var resultingFileCache = ReplacePathPrefixes(fileCache);
-        //_logger.LogTrace("Validating {path}", fileCache.PrefixedFilePath);
         resultingFileCache = Validate(resultingFileCache);
         return resultingFileCache;
     }
 
+    private async Task<FileCacheEntity?> GetValidatedFileCacheAsync(FileCacheEntity fileCache, CancellationToken token = default)
+    {
+        var resultingFileCache = ReplacePathPrefixes(fileCache);
+        resultingFileCache = await ValidateAsync(resultingFileCache, token).ConfigureAwait(false);
+        return resultingFileCache;
+    }
+
     private FileCacheEntity ReplacePathPrefixes(FileCacheEntity fileCache)
     {
         if (fileCache.PrefixedFilePath.StartsWith(PenumbraPrefix, StringComparison.OrdinalIgnoreCase))
@@ -629,6 +694,7 @@ public sealed class FileCacheManager : IHostedService
             RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath);
             return null;
         }
+
         var file = new FileInfo(fileCache.ResolvedFilepath);
         if (!file.Exists)
         {
@@ -636,7 +702,8 @@ public sealed class FileCacheManager : IHostedService
             return null;
         }
 
-        if (!string.Equals(file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), fileCache.LastModifiedDateTicks, StringComparison.Ordinal))
+        var lastWriteTicks = file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
+        if (!string.Equals(lastWriteTicks, fileCache.LastModifiedDateTicks, StringComparison.Ordinal))
         {
             UpdateHashedFile(fileCache);
         }
@@ -644,7 +711,34 @@ public sealed class FileCacheManager : IHostedService
         return fileCache;
     }
 
-    public Task StartAsync(CancellationToken cancellationToken)
+    private async Task<FileCacheEntity?> ValidateAsync(FileCacheEntity fileCache, CancellationToken token)
+    {
+        if (string.IsNullOrWhiteSpace(fileCache.ResolvedFilepath))
+        {
+            _logger.LogWarning("FileCacheEntity has empty ResolvedFilepath for hash {hash}, prefixed path {prefixed}", fileCache.Hash, fileCache.PrefixedFilePath);
+            RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath);
+            return null;
+        }
+
+        return await Task.Run(() =>
+        {
+            var file = new FileInfo(fileCache.ResolvedFilepath);
+            if (!file.Exists)
+            {
+                RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath);
+                return null;
+            }
+
+            if (!string.Equals(file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), fileCache.LastModifiedDateTicks, StringComparison.Ordinal))
+            {
+                UpdateHashedFile(fileCache);
+            }
+
+            return fileCache;
+        }, token).ConfigureAwait(false);
+    }
+
+    public async Task StartAsync(CancellationToken cancellationToken)
     {
         _logger.LogInformation("Starting FileCacheManager");
 
@@ -695,14 +789,14 @@ public sealed class FileCacheManager : IHostedService
                 try
                 {
                     _logger.LogInformation("Attempting to read {csvPath}", _csvPath);
-                    entries = File.ReadAllLines(_csvPath);
+                    entries = await File.ReadAllLinesAsync(_csvPath, cancellationToken).ConfigureAwait(false);
                     success = true;
                 }
                 catch (Exception ex)
                 {
                     attempts++;
                     _logger.LogWarning(ex, "Could not open {file}, trying again", _csvPath);
-                    Task.Delay(100, cancellationToken);
+                    await Task.Delay(100, cancellationToken).ConfigureAwait(false);
                 }
             }
 
@@ -729,7 +823,7 @@ public sealed class FileCacheManager : IHostedService
                         BackupUnsupportedCache("invalid-version");
                         parseEntries = false;
                         rewriteRequired = true;
-                        entries = Array.Empty<string>();
+                        entries = [];
                     }
                     else if (parsedVersion != FileCacheVersion)
                     {
@@ -737,7 +831,7 @@ public sealed class FileCacheManager : IHostedService
                         BackupUnsupportedCache($"v{parsedVersion}");
                         parseEntries = false;
                         rewriteRequired = true;
-                        entries = Array.Empty<string>();
+                        entries = [];
                     }
                     else
                     {
@@ -817,18 +911,18 @@ public sealed class FileCacheManager : IHostedService
 
             if (rewriteRequired)
             {
-                WriteOutFullCsv();
+                await WriteOutFullCsvAsync(cancellationToken).ConfigureAwait(false);
             }
         }
 
         _logger.LogInformation("Started FileCacheManager");
-
-        return Task.CompletedTask;
+		_lightlessMediator.Publish(new FileCacheInitializedMessage());
+		await Task.CompletedTask.ConfigureAwait(false);
     }
 
-    public Task StopAsync(CancellationToken cancellationToken)
+    public async Task StopAsync(CancellationToken cancellationToken)
     {
-        WriteOutFullCsv();
-        return Task.CompletedTask;
+        await WriteOutFullCsvAsync(cancellationToken).ConfigureAwait(false);
+        await Task.CompletedTask.ConfigureAwait(false);
     }
 }