sigma update

LightlessSync/FileCache/ExternalCompactionExecutor.cs

@@ -0,0 +1,241 @@
+using Microsoft.Extensions.Logging;
+using System.Diagnostics;
+using System.IO.Pipes;
+using System.Text.Json;
+
+namespace LightlessSync.FileCache;
+
+internal sealed class ExternalCompactionExecutor : ICompactionExecutor, IDisposable
+{
+    private readonly ILogger<ExternalCompactionExecutor> _logger;
+    private readonly ICompactorContext _context;
+    private readonly TimeSpan _timeout = TimeSpan.FromMinutes(5);
+    private readonly string _pipeName;
+    private Process? _workerProcess;
+    private bool _disposed;
+    private readonly object _sync = new();
+
+    public ExternalCompactionExecutor(ILogger<ExternalCompactionExecutor> logger, ICompactorContext context)
+    {
+        _logger = logger;
+        _context = context;
+        _pipeName = $"LightlessCompactor-{Environment.ProcessId}";
+    }
+
+    public bool TryCompact(string filePath)
+    {
+        if (string.IsNullOrWhiteSpace(filePath) || !File.Exists(filePath))
+            return false;
+
+        if (!EnsureWorkerRunning())
+            return false;
+
+        try
+        {
+            var request = new CompactorRequest
+            {
+                Type = "compact",
+                Path = filePath
+            };
+
+            return SendRequest(request, out var response) && response?.Success == true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "External compactor failed for {file}", filePath);
+            return false;
+        }
+    }
+
+    public void Dispose()
+    {
+        if (_disposed)
+            return;
+
+        _disposed = true;
+
+        try
+        {
+            SendRequest(new CompactorRequest { Type = "shutdown" }, out _);
+        }
+        catch
+        {
+            // ignore
+        }
+
+        lock (_sync)
+        {
+            if (_workerProcess is null)
+                return;
+
+            TryKill(_workerProcess);
+            _workerProcess.Dispose();
+            _workerProcess = null;
+        }
+    }
+
+    private bool EnsureWorkerRunning()
+    {
+        lock (_sync)
+        {
+            if (_workerProcess is { HasExited: false })
+                return true;
+
+            _workerProcess?.Dispose();
+            _workerProcess = null;
+
+            var workerPath = ResolveWorkerPath();
+            if (string.IsNullOrEmpty(workerPath))
+                return false;
+
+            var args = BuildArguments();
+            var startInfo = new ProcessStartInfo
+            {
+                FileName = workerPath,
+                Arguments = args,
+                CreateNoWindow = true,
+                UseShellExecute = false,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true
+            };
+
+            var process = new Process { StartInfo = startInfo };
+            if (!process.Start())
+                return false;
+
+            TrySetLowPriority(process);
+            _ = DrainAsync(process.StandardOutput, "stdout");
+            _ = DrainAsync(process.StandardError, "stderr");
+
+            _workerProcess = process;
+            return true;
+        }
+    }
+
+    private bool SendRequest(CompactorRequest request, out CompactorResponse? response)
+    {
+        response = null;
+        using var pipe = new NamedPipeClientStream(".", _pipeName, PipeDirection.InOut, PipeOptions.Asynchronous);
+
+        try
+        {
+            pipe.Connect((int)_timeout.TotalMilliseconds);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogDebug(ex, "Compactor pipe connection failed.");
+            return false;
+        }
+
+        using var writer = new StreamWriter(pipe) { AutoFlush = true };
+        using var reader = new StreamReader(pipe);
+
+        var payload = JsonSerializer.Serialize(request);
+        writer.WriteLine(payload);
+
+        var readTask = reader.ReadLineAsync();
+        if (!readTask.Wait(_timeout))
+        {
+            _logger.LogWarning("Compactor pipe timed out waiting for response.");
+            return false;
+        }
+
+        var line = readTask.Result;
+        if (string.IsNullOrWhiteSpace(line))
+            return false;
+
+        try
+        {
+            response = JsonSerializer.Deserialize<CompactorResponse>(line);
+            return response is not null;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogDebug(ex, "Failed to parse compactor response.");
+            return false;
+        }
+    }
+
+    private string? ResolveWorkerPath()
+    {
+        var baseDir = AppContext.BaseDirectory;
+        var exeName = OperatingSystem.IsWindows() || _context.IsWine
+            ? "LightlessCompactorWorker.exe"
+            : "LightlessCompactorWorker";
+        var path = Path.Combine(baseDir, exeName);
+        return File.Exists(path) ? path : null;
+    }
+
+    private string BuildArguments()
+    {
+        var args = new List<string> { "--pipe", Quote(_pipeName), "--parent", Environment.ProcessId.ToString() };
+        if (_context.IsWine)
+            args.Add("--wine");
+        return string.Join(' ', args);
+    }
+
+    private static string Quote(string value)
+    {
+        if (string.IsNullOrEmpty(value))
+            return "\"\"";
+
+        if (!value.Contains('"', StringComparison.Ordinal))
+            return "\"" + value + "\"";
+
+        return "\"" + value.Replace("\"", "\\\"", StringComparison.Ordinal) + "\"";
+    }
+
+    private static void TrySetLowPriority(Process process)
+    {
+        try
+        {
+            if (OperatingSystem.IsWindows())
+                process.PriorityClass = ProcessPriorityClass.BelowNormal;
+        }
+        catch
+        {
+            // ignore
+        }
+    }
+
+    private async Task DrainAsync(StreamReader reader, string label)
+    {
+        try
+        {
+            string? line;
+            while ((line = await reader.ReadLineAsync().ConfigureAwait(false)) != null)
+            {
+                if (_logger.IsEnabled(LogLevel.Trace))
+                    _logger.LogTrace("Compactor {label}: {line}", label, line);
+            }
+        }
+        catch
+        {
+            // ignore
+        }
+    }
+
+    private static void TryKill(Process process)
+    {
+        try
+        {
+            process.Kill(entireProcessTree: true);
+        }
+        catch
+        {
+            // ignore
+        }
+    }
+
+    private sealed class CompactorRequest
+    {
+        public string Type { get; init; } = "compact";
+        public string? Path { get; init; }
+    }
+
+    private sealed class CompactorResponse
+    {
+        public bool Success { get; init; }
+        public string? Error { get; init; }
+    }
+}

LightlessSync/FileCache/FileCacheManager.cs

@@ -115,6 +115,35 @@ public sealed class FileCacheManager : IHostedService
         return true;
     }
 
+    private static bool TryGetHashFromFileName(FileInfo fileInfo, out string hash)
+    {
+        hash = Path.GetFileNameWithoutExtension(fileInfo.Name);
+        if (string.IsNullOrWhiteSpace(hash))
+        {
+            return false;
+        }
+
+        if (hash.Length is not (40 or 64))
+        {
+            return false;
+        }
+
+        for (var i = 0; i < hash.Length; i++)
+        {
+            var c = hash[i];
+            var isHex = (c >= '0' && c <= '9')
+                || (c >= 'a' && c <= 'f')
+                || (c >= 'A' && c <= 'F');
+            if (!isHex)
+            {
+                return false;
+            }
+        }
+
+        hash = hash.ToUpperInvariant();
+        return true;
+    }
+
     private static string BuildVersionHeader() => $"{FileCacheVersionHeaderPrefix}{FileCacheVersion}";
 
     private static bool TryParseVersionHeader(string? line, out int version)
@@ -288,6 +317,11 @@ public sealed class FileCacheManager : IHostedService
         _logger.LogTrace("Creating cache entry for {path}", path);
         var cacheFolder = _configService.Current.CacheFolder;
         if (string.IsNullOrEmpty(cacheFolder)) return null;
+        if (TryGetHashFromFileName(fi, out var hash))
+        {
+            return CreateCacheEntryWithKnownHash(fi.FullName, hash);
+        }
+
         return CreateFileEntity(cacheFolder, CachePrefix, fi);
     }
 

LightlessSync/FileCache/PluginCompactorContext.cs

@@ -0,0 +1,20 @@
+using LightlessSync.LightlessConfiguration;
+using LightlessSync.Services;
+
+namespace LightlessSync.FileCache;
+
+internal sealed class PluginCompactorContext : ICompactorContext
+{
+    private readonly LightlessConfigService _configService;
+    private readonly DalamudUtilService _dalamudUtilService;
+
+    public PluginCompactorContext(LightlessConfigService configService, DalamudUtilService dalamudUtilService)
+    {
+        _configService = configService;
+        _dalamudUtilService = dalamudUtilService;
+    }
+
+    public bool UseCompactor => _configService.Current.UseCompactor;
+    public string CacheFolder => _configService.Current.CacheFolder;
+    public bool IsWine => _dalamudUtilService.IsWine;
+}

LightlessSync/FileCache/TransientResourceManager.cs

@@ -25,7 +25,6 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
     private readonly object _ownedHandlerLock = new();
     private readonly string[] _handledFileTypes = ["tmb", "pap", "avfx", "atex", "sklb", "eid", "phyb", "scd", "skp", "shpk", "kdb"];
     private readonly string[] _handledRecordingFileTypes = ["tex", "mdl", "mtrl"];
-    private readonly string[] _handledFileTypesWithRecording;
     private readonly HashSet<GameObjectHandler> _playerRelatedPointers = [];
     private readonly object _playerRelatedLock = new();
     private readonly ConcurrentDictionary<nint, GameObjectHandler> _playerRelatedByAddress = new();
@@ -42,8 +41,6 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
         _dalamudUtil = dalamudUtil;
         _actorObjectService = actorObjectService;
         _gameObjectHandlerFactory = gameObjectHandlerFactory;
-        _handledFileTypesWithRecording = _handledRecordingFileTypes.Concat(_handledFileTypes).ToArray();
-
         Mediator.Subscribe<PenumbraResourceLoadMessage>(this, Manager_PenumbraResourceLoadEvent);
         Mediator.Subscribe<ActorTrackedMessage>(this, msg => HandleActorTracked(msg.Descriptor));
         Mediator.Subscribe<ActorUntrackedMessage>(this, msg => HandleActorUntracked(msg.Descriptor));
@@ -523,46 +520,51 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
 
     private void Manager_PenumbraResourceLoadEvent(PenumbraResourceLoadMessage msg)
     {
+        var gamePath = msg.GamePath.ToLowerInvariant();
         var gameObjectAddress = msg.GameObject;
-        if (!_cachedFrameAddresses.TryGetValue(gameObjectAddress, out var objectKind))
-        {
-            if (_actorObjectService.TryGetOwnedKind(gameObjectAddress, out var ownedKind))
-            {
-                objectKind = ownedKind;
-            }
-            else
-            {
-                return;
-            }
-        }
-
-        var gamePath = NormalizeGamePath(msg.GamePath);
-        if (string.IsNullOrEmpty(gamePath))
-        {
-            return;
-        }
+        var filePath = msg.FilePath;
 
         // ignore files already processed this frame
+        if (_cachedHandledPaths.Contains(gamePath)) return;
+
         lock (_cacheAdditionLock)
         {
-            if (!_cachedHandledPaths.Add(gamePath))
-            {
-                return;
-            }
+            _cachedHandledPaths.Add(gamePath);
+        }
+
+        // replace individual mtrl stuff
+        if (filePath.StartsWith("|", StringComparison.OrdinalIgnoreCase))
+        {
+            filePath = filePath.Split("|")[2];
+        }
+        // replace filepath
+        filePath = filePath.ToLowerInvariant().Replace("\\", "/", StringComparison.OrdinalIgnoreCase);
+
+        // ignore files that are the same
+        var replacedGamePath = gamePath.ToLowerInvariant().Replace("\\", "/", StringComparison.OrdinalIgnoreCase);
+        if (string.Equals(filePath, replacedGamePath, StringComparison.OrdinalIgnoreCase))
+        {
+            return;
         }
 
         // ignore files to not handle
-        var handledTypes = IsTransientRecording ? _handledFileTypesWithRecording : _handledFileTypes;
-        if (!HasHandledFileType(gamePath, handledTypes))
+        var handledTypes = IsTransientRecording ? _handledRecordingFileTypes.Concat(_handledFileTypes) : _handledFileTypes;
+        if (!handledTypes.Any(type => gamePath.EndsWith(type, StringComparison.OrdinalIgnoreCase)))
         {
+            lock (_cacheAdditionLock)
+            {
+                _cachedHandledPaths.Add(gamePath);
+            }
             return;
         }
 
-        var filePath = NormalizeFilePath(msg.FilePath);
-
-        // ignore files that are the same
-        if (string.Equals(filePath, gamePath, StringComparison.Ordinal))
+        // ignore files not belonging to anything player related
+        if (!_cachedFrameAddresses.TryGetValue(gameObjectAddress, out var objectKind))
         {
+            lock (_cacheAdditionLock)
+            {
+                _cachedHandledPaths.Add(gamePath);
+            }
             return;
         }
 
@@ -577,12 +579,13 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
         _playerRelatedByAddress.TryGetValue(gameObjectAddress, out var owner);
         bool alreadyTransient = false;
 
-        bool transientContains = transientResources.Contains(gamePath);
-        bool semiTransientContains = SemiTransientResources.Values.Any(value => value.Contains(gamePath));
+        bool transientContains = transientResources.Contains(replacedGamePath);
+        bool semiTransientContains = SemiTransientResources.SelectMany(k => k.Value)
+            .Any(f => string.Equals(f, gamePath, StringComparison.OrdinalIgnoreCase));
         if (transientContains || semiTransientContains)
         {
             if (!IsTransientRecording)
-                Logger.LogTrace("Not adding {replacedPath} => {filePath}, Reason: Transient: {contains}, SemiTransient: {contains2}", gamePath, filePath,
+                Logger.LogTrace("Not adding {replacedPath} => {filePath}, Reason: Transient: {contains}, SemiTransient: {contains2}", replacedGamePath, filePath,
                     transientContains, semiTransientContains);
             alreadyTransient = true;
         }
@@ -590,10 +593,10 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
         {
             if (!IsTransientRecording)
             {
-                bool isAdded = transientResources.Add(gamePath);
+                bool isAdded = transientResources.Add(replacedGamePath);
                 if (isAdded)
                 {
-                    Logger.LogDebug("Adding {replacedGamePath} for {gameObject} ({filePath})", gamePath, owner?.ToString() ?? gameObjectAddress.ToString("X"), filePath);
+                    Logger.LogDebug("Adding {replacedGamePath} for {gameObject} ({filePath})", replacedGamePath, owner?.ToString() ?? gameObjectAddress.ToString("X"), filePath);
                     SendTransients(gameObjectAddress, objectKind);
                 }
             }
@@ -601,7 +604,7 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
 
         if (owner != null && IsTransientRecording)
         {
-            _recordedTransients.Add(new TransientRecord(owner, gamePath, filePath, alreadyTransient) { AddTransient = !alreadyTransient });
+            _recordedTransients.Add(new TransientRecord(owner, replacedGamePath, filePath, alreadyTransient) { AddTransient = !alreadyTransient });
         }
     }
 
@@ -700,4 +703,4 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
     {
         public bool AddTransient { get; set; }
     }
-}
+}