2.0.0 (#92)

2.0.0 Changes:

- Reworked shell finder UI with compact or list view with profile tags showing with the listing, allowing moderators to broadcast the syncshell as well to have it be used more.
- Reworked user list in syncshell admin screen to have filter visible and moved away from table to its own thing, allowing to copy uid/note/alias when clicking on the name.
- Reworked download bars and download box to make it look more modern, removed the jitter around, so it shouldn't vibrate around much.
- Chat has been added to the top menu, working in Zone or in Syncshells to be used there.
- Paired system has been revamped to make pausing and unpausing faster, and loading people should be faster as well.
- Moved to the internal object table to have faster load times for users; people should load in faster
- Compactor is running on a multi-threaded level instead of single-threaded; this should increase the speed of compacting files
- Nameplate Service has been reworked so it wouldn't use the nameplate handler anymore.
- Files can be resized when downloading to reduce load on users if they aren't compressed. (can be toggled to resize all).
- Penumbra Collections are now only made when people are visible, reducing the load on boot-up when having many syncshells in your list.
- Lightfinder plates have been moved away from using Nameplates, but will use an overlay.
- Main UI has been changed a bit with a gradient, and on hover will glow up now.
- Reworked Profile UI for Syncshell and Users to be more user-facing with more customizable items.
- Reworked Settings UI to look more modern.
- Performance should be better due to new systems that would dispose of the collections and better caching of items.

Co-authored-by: defnotken <itsdefnotken@gmail.com>
Co-authored-by: azyges <aaaaaa@aaa.aaa>
Co-authored-by: choco <choco@patat.nl>
Co-authored-by: cake <admin@cakeandbanana.nl>
Co-authored-by: Minmoose <KennethBohr@outlook.com>
Reviewed-on: #92

LightlessSync/Services/CharacterAnalyzer.cs

@@ -1,16 +1,14 @@
-using LightlessSync.API.Data;
+using LightlessSync.API.Data;
 using LightlessSync.API.Data.Enum;
 using LightlessSync.FileCache;
+using LightlessSync.Services.CharaData;
+using LightlessSync.Services.CharaData.Models;
 using LightlessSync.Services.Mediator;
 using LightlessSync.UI;
 using LightlessSync.Utils;
 using Lumina.Data.Files;
 using Microsoft.Extensions.Logging;
-using System.Collections.Generic;
 using System.Collections.Immutable;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
 namespace LightlessSync.Services;
 
 public sealed class CharacterAnalyzer : MediatorSubscriberBase, IDisposable
@@ -40,73 +38,97 @@ public sealed class CharacterAnalyzer : MediatorSubscriberBase, IDisposable
     public int TotalFiles { get; internal set; }
     internal Dictionary<ObjectKind, Dictionary<string, FileDataEntry>> LastAnalysis { get; } = [];
     public CharacterAnalysisSummary LatestSummary => _latestSummary;
-
     public void CancelAnalyze()
     {
         _analysisCts?.CancelDispose();
         _analysisCts = null;
     }
-
     public async Task ComputeAnalysis(bool print = true, bool recalculate = false)
     {
         Logger.LogDebug("=== Calculating Character Analysis ===");
-
         _analysisCts = _analysisCts?.CancelRecreate() ?? new();
-
         var cancelToken = _analysisCts.Token;
-
         var allFiles = LastAnalysis.SelectMany(v => v.Value.Select(d => d.Value)).ToList();
-        if (allFiles.Exists(c => !c.IsComputed || recalculate))
+
+        var remaining = allFiles.Where(c => !c.IsComputed || recalculate).ToList();
+
+        if (remaining.Count == 0)
+            return;
+
+        TotalFiles = remaining.Count;
+        CurrentFile = 0;
+
+        Logger.LogDebug("=== Computing {amount} remaining files ===", remaining.Count);
+
+        Mediator.Publish(new HaltScanMessage(nameof(CharacterAnalyzer)));
+
+        try
         {
-            var remaining = allFiles.Where(c => !c.IsComputed || recalculate).ToList();
-            TotalFiles = remaining.Count;
-            CurrentFile = 1;
-            Logger.LogDebug("=== Computing {amount} remaining files ===", remaining.Count);
-
-            Mediator.Publish(new HaltScanMessage(nameof(CharacterAnalyzer)));
-            try
+            foreach (var file in remaining)
             {
-                foreach (var file in remaining)
-                {
-                    Logger.LogDebug("Computing file {file}", file.FilePaths[0]);
-                    await file.ComputeSizes(_fileCacheManager, cancelToken).ConfigureAwait(false);
-                    CurrentFile++;
-                }
+                cancelToken.ThrowIfCancellationRequested();
 
-                _fileCacheManager.WriteOutFullCsv();
+                var path = file.FilePaths.FirstOrDefault() ?? "<unknown>";
+                Logger.LogDebug("Computing file {file}", path);
 
+                await file.ComputeSizes(_fileCacheManager, cancelToken).ConfigureAwait(false);
+
+                CurrentFile++;
             }
-            catch (Exception ex)
-            {
-                Logger.LogWarning(ex, "Failed to analyze files");
-            }
-            finally
-            {
-                Mediator.Publish(new ResumeScanMessage(nameof(CharacterAnalyzer)));
-            }
+
+            await _fileCacheManager.WriteOutFullCsvAsync(cancelToken).ConfigureAwait(false);
+        }
+        catch (OperationCanceledException)
+        {
+            Logger.LogInformation("File analysis cancelled");
+            throw;
+        }
+        catch (Exception ex)
+        {
+            Logger.LogWarning(ex, "Failed to analyze files");
+        }
+        finally
+        {
+            Mediator.Publish(new ResumeScanMessage(nameof(CharacterAnalyzer)));
         }
 
         RecalculateSummary();
-
         Mediator.Publish(new CharacterDataAnalyzedMessage());
-
         _analysisCts.CancelDispose();
         _analysisCts = null;
-
         if (print) PrintAnalysis();
     }
-
     public void Dispose()
     {
         _analysisCts.CancelDispose();
+        _baseAnalysisCts.Dispose();
+    }
+    public async Task UpdateFileEntriesAsync(IEnumerable<string> filePaths, CancellationToken token)
+    {
+        var normalized = new HashSet<string>(
+            filePaths.Where(path => !string.IsNullOrWhiteSpace(path)),
+            StringComparer.OrdinalIgnoreCase);
+        if (normalized.Count == 0)
+        {
+            return;
+        }
+        foreach (var objectEntries in LastAnalysis.Values)
+        {
+            foreach (var entry in objectEntries.Values)
+            {
+                if (!entry.FilePaths.Exists(path => normalized.Contains(path)))
+                {
+                    continue;
+                }
+                token.ThrowIfCancellationRequested();
+                await entry.ComputeSizes(_fileCacheManager, token).ConfigureAwait(false);
+            }
+        }
     }
-
     private async Task BaseAnalysis(CharacterData charaData, CancellationToken token)
     {
         if (string.Equals(charaData.DataHash.Value, _lastDataHash, StringComparison.Ordinal)) return;
-
         LastAnalysis.Clear();
-
         foreach (var obj in charaData.FileReplacements)
         {
             Dictionary<string, FileDataEntry> data = new(StringComparer.OrdinalIgnoreCase);
@@ -114,9 +136,8 @@ public sealed class CharacterAnalyzer : MediatorSubscriberBase, IDisposable
             {
                 token.ThrowIfCancellationRequested();
 
-                var fileCacheEntries = _fileCacheManager.GetAllFileCachesByHash(fileEntry.Hash, ignoreCacheEntries: true, validate: false).ToList();
+                var fileCacheEntries = (await _fileCacheManager.GetAllFileCachesByHashAsync(fileEntry.Hash, ignoreCacheEntries: true, validate: false, token).ConfigureAwait(false)).ToList();
                 if (fileCacheEntries.Count == 0) continue;
-
                 var filePath = fileCacheEntries[0].ResolvedFilepath;
                 FileInfo fi = new(filePath);
                 string ext = "unk?";
@@ -128,30 +149,24 @@ public sealed class CharacterAnalyzer : MediatorSubscriberBase, IDisposable
                 {
                     Logger.LogWarning(ex, "Could not identify extension for {path}", filePath);
                 }
-
                 var tris = await _xivDataAnalyzer.GetTrianglesByHash(fileEntry.Hash).ConfigureAwait(false);
-
                 foreach (var entry in fileCacheEntries)
                 {
                     data[fileEntry.Hash] = new FileDataEntry(fileEntry.Hash, ext,
                         [.. fileEntry.GamePaths],
-                        fileCacheEntries.Select(c => c.ResolvedFilepath).Distinct().ToList(),
+                        [.. fileCacheEntries.Select(c => c.ResolvedFilepath).Distinct(StringComparer.Ordinal)],
                         entry.Size > 0 ? entry.Size.Value : 0,
                         entry.CompressedSize > 0 ? entry.CompressedSize.Value : 0,
                         tris);
                 }
             }
-
             LastAnalysis[obj.Key] = data;
         }
 
         RecalculateSummary();
-
         Mediator.Publish(new CharacterDataAnalyzedMessage());
-
         _lastDataHash = charaData.DataHash.Value;
     }
-
     private void RecalculateSummary()
     {
         var builder = ImmutableDictionary.CreateBuilder<ObjectKind, CharacterAnalysisObjectSummary>();
@@ -177,7 +192,6 @@ public sealed class CharacterAnalyzer : MediatorSubscriberBase, IDisposable
 
         _latestSummary = new CharacterAnalysisSummary(builder.ToImmutable());
     }
-
     private void PrintAnalysis()
     {
         if (LastAnalysis.Count == 0) return;
@@ -186,7 +200,6 @@ public sealed class CharacterAnalyzer : MediatorSubscriberBase, IDisposable
             int fileCounter = 1;
             int totalFiles = kvp.Value.Count;
             Logger.LogInformation("=== Analysis for {obj} ===", kvp.Key);
-
             foreach (var entry in kvp.Value.OrderBy(b => b.Value.GamePaths.OrderBy(p => p, StringComparer.Ordinal).First(), StringComparer.Ordinal))
             {
                 Logger.LogInformation("File {x}/{y}: {hash}", fileCounter++, totalFiles, entry.Key);
@@ -215,7 +228,6 @@ public sealed class CharacterAnalyzer : MediatorSubscriberBase, IDisposable
             Logger.LogInformation("Total files: {count}, size extracted: {size}, size compressed: {sizeComp}", kvp.Value.Count,
             UiSharedService.ByteToString(kvp.Value.Sum(v => v.Value.OriginalSize)), UiSharedService.ByteToString(kvp.Value.Sum(v => v.Value.CompressedSize)));
         }
-
         Logger.LogInformation("=== Total summary for all currently present objects ===");
         Logger.LogInformation("Total files: {count}, size extracted: {size}, size compressed: {sizeComp}",
             LastAnalysis.Values.Sum(v => v.Values.Count),
@@ -223,7 +235,6 @@ public sealed class CharacterAnalyzer : MediatorSubscriberBase, IDisposable
             UiSharedService.ByteToString(LastAnalysis.Values.Sum(c => c.Values.Sum(v => v.CompressedSize))));
         Logger.LogInformation("IMPORTANT NOTES:\n\r- For Lightless up- and downloads only the compressed size is relevant.\n\r- An unusually high total files count beyond 200 and up will also increase your download time to others significantly.");
     }
-
     internal sealed record FileDataEntry(string Hash, string FileType, List<string> GamePaths, List<string> FilePaths, long OriginalSize, long CompressedSize, long Triangles)
     {
         public bool IsComputed => OriginalSize > 0 && CompressedSize > 0;
@@ -231,7 +242,7 @@ public sealed class CharacterAnalyzer : MediatorSubscriberBase, IDisposable
         {
             var compressedsize = await fileCacheManager.GetCompressedFileData(Hash, token).ConfigureAwait(false);
             var normalSize = new FileInfo(FilePaths[0]).Length;
-            var entries = fileCacheManager.GetAllFileCachesByHash(Hash, ignoreCacheEntries: true, validate: false);
+            var entries = await fileCacheManager.GetAllFileCachesByHashAsync(Hash, ignoreCacheEntries: true, validate: false, token).ConfigureAwait(false);
             foreach (var entry in entries)
             {
                 entry.Size = normalSize;
@@ -239,53 +250,40 @@ public sealed class CharacterAnalyzer : MediatorSubscriberBase, IDisposable
             }
             OriginalSize = normalSize;
             CompressedSize = compressedsize.Item2.LongLength;
+            RefreshFormat();
         }
         public long OriginalSize { get; private set; } = OriginalSize;
         public long CompressedSize { get; private set; } = CompressedSize;
         public long Triangles { get; private set; } = Triangles;
+        public Lazy<string> Format => _format ??= CreateFormatValue();
 
-        public Lazy<string> Format = new(() =>
+        private Lazy<string>? _format;
+
+        public void RefreshFormat()
         {
-            switch (FileType)
+            _format = CreateFormatValue();
+        }
+
+        private Lazy<string> CreateFormatValue()
+            => new(() =>
             {
-                case "tex":
-                    {
-                        try
-                        {
-                            using var stream = new FileStream(FilePaths[0], FileMode.Open, FileAccess.Read, FileShare.Read);
-                            using var reader = new BinaryReader(stream);
-                            reader.BaseStream.Position = 4;
-                            var format = (TexFile.TextureFormat)reader.ReadInt32();
-                            return format.ToString();
-                        }
-                        catch
-                        {
-                            return "Unknown";
-                        }
-                    }
-                default:
+                if (!string.Equals(FileType, "tex", StringComparison.Ordinal))
+                {
                     return string.Empty;
-            }
-        });
+                }
+
+                try
+                {
+                    using var stream = new FileStream(FilePaths[0], FileMode.Open, FileAccess.Read, FileShare.Read);
+                    using var reader = new BinaryReader(stream);
+                    reader.BaseStream.Position = 4;
+                    var format = (TexFile.TextureFormat)reader.ReadInt32();
+                    return format.ToString();
+                }
+                catch
+                {
+                    return "Unknown";
+                }
+            });
     }
 }
-
-public readonly record struct CharacterAnalysisObjectSummary(int EntryCount, long TotalTriangles, long TexOriginalBytes, long TexCompressedBytes)
-{
-    public bool HasEntries => EntryCount > 0;
-}
-
-public sealed class CharacterAnalysisSummary
-{
-    public static CharacterAnalysisSummary Empty { get; } =
-        new(ImmutableDictionary<ObjectKind, CharacterAnalysisObjectSummary>.Empty);
-
-    internal CharacterAnalysisSummary(IImmutableDictionary<ObjectKind, CharacterAnalysisObjectSummary> objects)
-    {
-        Objects = objects;
-    }
-
-    public IImmutableDictionary<ObjectKind, CharacterAnalysisObjectSummary> Objects { get; }
-
-    public bool HasData => Objects.Any(kvp => kvp.Value.HasEntries);
-}