Implemented compactor to work on BTRFS, redid cache a bit for better function on linux. Removed error for websockets, it will be forced on wine again.

LightlessSync/FileCache/FileCompactor.cs

@@ -1,11 +1,15 @@
 using LightlessSync.LightlessConfiguration;
 using LightlessSync.Services;
+using LightlessSync.Utils;
 using Microsoft.Extensions.Logging;
-using System.Runtime.InteropServices;
 using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.IO;
+using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Channels;
 using System.Threading.Tasks;
+using static LightlessSync.Utils.FileSystemHelper;
 
 namespace LightlessSync.FileCache;
 
@@ -87,25 +91,51 @@ public sealed class FileCompactor : IDisposable
 
     public long GetFileSizeOnDisk(FileInfo fileInfo, bool? isNTFS = null)
     {
-        bool ntfs = isNTFS ?? string.Equals(new DriveInfo(fileInfo.Directory!.Root.FullName).DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase);
+        var fsType = FileSystemHelper.GetFilesystemType(fileInfo.FullName);
 
-        if (_dalamudUtilService.IsWine || !ntfs) return fileInfo.Length;
+        bool ntfs = isNTFS ?? fsType == FileSystemHelper.FilesystemType.NTFS;
 
-        var clusterSize = GetClusterSize(fileInfo);
-        if (clusterSize == -1) return fileInfo.Length;
-        var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize);
-        var size = (long)hosize << 32 | losize;
-        return ((size + clusterSize - 1) / clusterSize) * clusterSize;
+        if (fsType != FileSystemHelper.FilesystemType.Btrfs && !ntfs)
+        {
+            return fileInfo.Length;
+        }
+
+        if (ntfs && !_dalamudUtilService.IsWine)
+        {
+            var clusterSize = GetClusterSize(fileInfo);
+            if (clusterSize == -1) return fileInfo.Length;
+            var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize);
+            var size = (long)hosize << 32 | losize;
+            return ((size + clusterSize - 1) / clusterSize) * clusterSize;
+        }
+
+        if (fsType == FileSystemHelper.FilesystemType.Btrfs)
+        {
+            try
+            {
+                long blocks = RunStatGetBlocks(fileInfo.FullName);
+                return blocks * 512L;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogDebug(ex, "Failed to get on-disk size via stat for {file}, falling back to Length", fileInfo.FullName);
+                return fileInfo.Length;
+            }
+        }
+
+        return fileInfo.Length;
     }
 
     public async Task WriteAllBytesAsync(string filePath, byte[] decompressedFile, CancellationToken token)
     {
+        var dir = Path.GetDirectoryName(filePath);
+        if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir))
+            Directory.CreateDirectory(dir);
+
         await File.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false);
 
-        if (_dalamudUtilService.IsWine || !_lightlessConfigService.Current.UseCompactor)
-        {
+        if (!_lightlessConfigService.Current.UseCompactor)
             return;
-        }
 
         EnqueueCompaction(filePath);
     }
@@ -153,56 +183,178 @@ public sealed class FileCompactor : IDisposable
 
     private void CompactFile(string filePath)
     {
-        var fs = new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName);
-        bool isNTFS = string.Equals(fs.DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase);
-        if (!isNTFS)
+        var fi = new FileInfo(filePath);
+        if (!fi.Exists)
         {
-            _logger.LogWarning("Drive for file {file} is not NTFS", filePath);
+            _logger.LogDebug("Skipping compaction for missing file {file}", filePath);
             return;
         }
 
-        var fi = new FileInfo(filePath);
+        var fsType = FileSystemHelper.GetFilesystemType(filePath);
         var oldSize = fi.Length;
-        var clusterSize = GetClusterSize(fi);
 
+        int clusterSize = GetClusterSize(fi);
         if (oldSize < Math.Max(clusterSize, 8 * 1024))
         {
             _logger.LogDebug("File {file} is smaller than cluster size ({size}), ignoring", filePath, clusterSize);
             return;
         }
 
-        if (!IsCompactedFile(filePath))
+        // NTFS Compression.
+        if (fsType == FileSystemHelper.FilesystemType.NTFS && !_dalamudUtilService.IsWine)
         {
-            _logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath);
+            if (!IsWOFCompactedFile(filePath))
+            {
+                _logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath);
+                var success = WOFCompressFile(filePath);
 
-            WOFCompressFile(filePath);
+                if (success)
+                {
+                    var newSize = GetFileSizeOnDisk(fi);
+                    _logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize);
+                }
+                else
+                {
+                    _logger.LogWarning("NTFS compression failed or not available for {file}", filePath);
+                }
 
-            var newSize = GetFileSizeOnDisk(fi);
-
-            _logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize);
+            }
+            else
+            {
+                _logger.LogDebug("File {file} already compressed (NTFS)", filePath);
+            }
         }
-        else
+
+        // BTRFS Compression
+        if (fsType == FileSystemHelper.FilesystemType.Btrfs)
         {
-            _logger.LogDebug("File {file} already compressed", filePath);
+            if (!IsBtrfsCompressedFile(filePath))
+            {
+                _logger.LogDebug("Attempting btrfs compression for {file}", filePath);
+                var success = BtrfsCompressFile(filePath);
+
+                if (success)
+                {
+                    var newSize = GetFileSizeOnDisk(fi);
+                    _logger.LogDebug("Btrfs-compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize);
+                }
+                else
+                {
+                    _logger.LogWarning("Btrfs compression failed or not available for {file}", filePath);
+                }
+            } 
+            else
+            {
+                _logger.LogDebug("File {file} already compressed (Btrfs)", filePath);
+            }
         }
     }
 
+    private static long RunStatGetBlocks(string path)
+    {
+        var psi = new ProcessStartInfo("stat", $"-c %b \"{path}\"")
+        {
+            RedirectStandardOutput = true,
+            RedirectStandardError = true,
+            UseShellExecute = false,
+            CreateNoWindow = true
+        };
+
+        using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat process");
+        var outp = proc.StandardOutput.ReadToEnd();
+        var err = proc.StandardError.ReadToEnd();
+        proc.WaitForExit();
+        if (proc.ExitCode != 0)
+        {
+            throw new InvalidOperationException($"stat failed: {err}");
+        }
+
+        if (!long.TryParse(outp.Trim(), out var blocks))
+        {
+            throw new InvalidOperationException($"invalid stat output: {outp}");
+        }
+
+        return blocks;
+    }
+
     private void DecompressFile(string path)
     {
         _logger.LogDebug("Removing compression from {file}", path);
-        try
+        var fsType = FileSystemHelper.GetFilesystemType(path);
+        if (fsType == null) return;
+
+        //NTFS Decompression
+        if (fsType == FileSystemHelper.FilesystemType.NTFS && !_dalamudUtilService.IsWine)
         {
-            using (var fs = new FileStream(path, FileMode.Open))
+            try
             {
+                using (var fs = new FileStream(path, FileMode.Open))
+                {
 #pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called
-                var hDevice = fs.SafeFileHandle.DangerousGetHandle();
+                    var hDevice = fs.SafeFileHandle.DangerousGetHandle();
 #pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called
-                _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _);
+                    _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _);
+                }
             }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Error decompressing file {path}", path);
+            }
+            return;
         }
-        catch (Exception ex)
+
+        //BTRFS Decompression
+        if (fsType == FileSystemHelper.FilesystemType.Btrfs)
         {
-            _logger.LogWarning(ex, "Error decompressing file {path}", path);
+            try
+            {
+                var mountOptions = GetMountOptionsForPath(path);
+                if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase))
+                {
+                    _logger.LogWarning(
+                        "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " +
+                        "Remount with 'compress=no' before running decompression.", path, mountOptions);
+                    return;
+                }
+
+                _logger.LogDebug("Rewriting {file} to remove btrfs compression...", path);
+
+                var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -- \"{path}\"")
+                {
+                    RedirectStandardOutput = true,
+                    RedirectStandardError = true,
+                    UseShellExecute = false,
+                    CreateNoWindow = true
+                };
+
+                using var proc = Process.Start(psi);
+                if (proc == null)
+                {
+                    _logger.LogWarning("Failed to start btrfs defragment for decompression of {file}", path);
+                    return;
+                }
+
+                var stdout = proc.StandardOutput.ReadToEnd();
+                var stderr = proc.StandardError.ReadToEnd();
+                proc.WaitForExit();
+
+                if (proc.ExitCode != 0)
+                {
+                    _logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr);
+                }
+                else
+                {
+                    // Log output only in debug mode to avoid clutter
+                    if (!string.IsNullOrWhiteSpace(stdout))
+                        _logger.LogDebug("btrfs defragment output for {file}: {out}", path, stdout.Trim());
+
+                    _logger.LogInformation("Decompressed (rewritten uncompressed) btrfs file: {file}", path);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Error rewriting {file} for decompression", path);
+            }
         }
     }
 
@@ -220,7 +372,7 @@ public sealed class FileCompactor : IDisposable
         return _clusterSizes[root];
     }
 
-    private static bool IsCompactedFile(string filePath)
+    private static bool IsWOFCompactedFile(string filePath)
     {
         uint buf = 8;
         _ = WofIsExternalFile(filePath, out int isExtFile, out uint _, out var info, ref buf);
@@ -228,40 +380,151 @@ public sealed class FileCompactor : IDisposable
         return info.Algorithm == CompressionAlgorithm.XPRESS8K;
     }
 
-    private void WOFCompressFile(string path)
+    private bool IsBtrfsCompressedFile(string path)
+    {
+        try
+        {
+            var psi = new ProcessStartInfo("filefrag", $"-v \"{path}\"")
+            {
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+                CreateNoWindow = true
+            };
+
+            using var proc = Process.Start(psi);
+            if (proc == null)
+            {
+                _logger.LogWarning("Failed to start filefrag for {file}", path);
+                return false;
+            }
+
+            string output = proc.StandardOutput.ReadToEnd();
+            proc.WaitForExit();
+
+            // look for "flags: compressed" in the output
+            if (output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase))
+            {
+                return true;
+            }
+
+            return false;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogDebug(ex, "Failed to detect btrfs compression for {file}", path);
+            return false;
+        }
+    }
+
+    private bool WOFCompressFile(string path)
     {
         var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo));
         Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: true);
         ulong length = (ulong)Marshal.SizeOf(_efInfo);
         try
         {
-            using (var fs = new FileStream(path, FileMode.Open))
+            using var fs = new FileStream(path, FileMode.Open);
+            #pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called
+            var hFile = fs.SafeFileHandle.DangerousGetHandle();
+            #pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called
+            if (fs.SafeFileHandle.IsInvalid)
             {
-#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called
-                var hFile = fs.SafeFileHandle.DangerousGetHandle();
-#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called
-                if (fs.SafeFileHandle.IsInvalid)
-                {
-                    _logger.LogWarning("Invalid file handle to {file}", path);
-                }
-                else
-                {
-                    var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length);
-                    if (!(ret == 0 || ret == unchecked((int)0x80070158)))
-                    {
-                        _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X"));
-                    }
-                }
+                _logger.LogWarning("Invalid file handle to {file}", path);
+                return false;
+            }
+
+            var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length);
+            if (!(ret == 0 || ret == unchecked((int)0x80070158)))
+            {
+                _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X"));
+                return false;
             }
         }
         catch (Exception ex)
         {
             _logger.LogWarning(ex, "Error compacting file {path}", path);
+            return false;
         }
         finally
         {
             Marshal.FreeHGlobal(efInfoPtr);
         }
+        return true;
+    }
+
+    private bool BtrfsCompressFile(string path)
+    {
+        try
+        {
+            var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -czstd -- \"{path}\"")
+            {
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+                CreateNoWindow = true
+            };
+
+            using var proc = Process.Start(psi);
+            if (proc == null)
+            {
+                _logger.LogWarning("Failed to start btrfs process for {file}", path);
+                return false;
+            }
+
+            var stdout = proc.StandardOutput.ReadToEnd();
+            var stderr = proc.StandardError.ReadToEnd();
+            proc.WaitForExit();
+
+            if (proc.ExitCode != 0)
+            {
+                _logger.LogWarning("btrfs defrag returned {code} for {file}: {err}", proc.ExitCode, path, stderr);
+                return false;
+            }
+
+            _logger.LogDebug("btrfs output: {out}", stdout);
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path);
+            return false;
+        }
+    }
+
+    private string GetMountOptionsForPath(string path)
+    {
+        try
+        {
+            var fullPath = Path.GetFullPath(path);
+            var mounts = File.ReadAllLines("/proc/mounts");
+            string bestMount = string.Empty;
+            string mountOptions = string.Empty;
+
+            foreach (var line in mounts)
+            {
+                var parts = line.Split(' ');
+                if (parts.Length < 4) continue;
+                var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); // unescape spaces
+                string normalized;
+                try { normalized = Path.GetFullPath(mountPoint); }
+                catch { normalized = mountPoint; }
+
+                if (fullPath.StartsWith(normalized, StringComparison.Ordinal) &&
+                    normalized.Length > bestMount.Length)
+                {
+                    bestMount = normalized;
+                    mountOptions = parts[3];
+                }
+            }
+
+            return mountOptions;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogDebug(ex, "Failed to get mount options for {path}", path);
+            return string.Empty;
+        }
     }
 
     private struct WOF_FILE_COMPRESSION_INFO_V1
@@ -273,7 +536,14 @@ public sealed class FileCompactor : IDisposable
     private void EnqueueCompaction(string filePath)
     {
         if (!_pendingCompactions.TryAdd(filePath, 0))
+            return;
+        
+        var fsType = GetFilesystemType(filePath);
+
+        if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs)
         {
+            _logger.LogTrace("Skipping compaction enqueue for unsupported filesystem {fs} ({file})", fsType, filePath);
+            _pendingCompactions.TryRemove(filePath, out _);
             return;
         }
 
@@ -282,6 +552,10 @@ public sealed class FileCompactor : IDisposable
             _pendingCompactions.TryRemove(filePath, out _);
             _logger.LogDebug("Failed to enqueue compaction job for {file}", filePath);
         }
+        else
+        {
+            _logger.LogTrace("Queued compaction job for {file} (fs={fs})", filePath, fsType);
+        }
     }
 
     private async Task ProcessQueueAsync(CancellationToken token)
@@ -299,7 +573,7 @@ public sealed class FileCompactor : IDisposable
                             return;
                         }
 
-                        if (_dalamudUtilService.IsWine || !_lightlessConfigService.Current.UseCompactor)
+                        if (!_lightlessConfigService.Current.UseCompactor)
                         {
                             continue;
                         }