From 177534d78b32269c8c96f65d5c01043cf5cdca1b Mon Sep 17 00:00:00 2001 From: cake Date: Wed, 29 Oct 2025 04:37:24 +0100 Subject: [PATCH 01/13] Implemented compactor to work on BTRFS, redid cache a bit for better function on linux. Removed error for websockets, it will be forced on wine again. --- LightlessSync/FileCache/CacheMonitor.cs | 17 +- LightlessSync/FileCache/FileCacheManager.cs | 80 ++-- LightlessSync/FileCache/FileCompactor.cs | 374 +++++++++++++++--- .../Models/ServerStorage.cs | 1 - LightlessSync/UI/SettingsUi.cs | 36 +- LightlessSync/Utils/Crypto.cs | 20 + LightlessSync/Utils/FileSystemHelper.cs | 143 +++++++ LightlessSync/WebAPI/SignalR/HubFactory.cs | 7 - 8 files changed, 572 insertions(+), 106 deletions(-) create mode 100644 LightlessSync/Utils/FileSystemHelper.cs diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 3b41d85..23f5c19 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -115,6 +115,8 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase public bool StorageisNTFS { get; private set; } = false; + public bool StorageIsBtrfs { get ; private set; } = false; + public void StartLightlessWatcher(string? lightlessPath) { LightlessWatcher?.Dispose(); @@ -124,10 +126,19 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase Logger.LogWarning("Lightless file path is not set, cannot start the FSW for Lightless."); return; } + var fsType = FileSystemHelper.GetFilesystemType(_configService.Current.CacheFolder); - DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName); - StorageisNTFS = string.Equals("NTFS", di.DriveFormat, StringComparison.OrdinalIgnoreCase); - Logger.LogInformation("Lightless Storage is on NTFS drive: {isNtfs}", StorageisNTFS); + if (fsType == FileSystemHelper.FilesystemType.NTFS) + { + StorageisNTFS = true; + Logger.LogInformation("Lightless Storage is on NTFS drive: {isNtfs}", StorageisNTFS); + } + + if (fsType == FileSystemHelper.FilesystemType.Btrfs) + { + StorageIsBtrfs = true; + Logger.LogInformation("Lightless Storage is on BTRFS drive: {isNtfs}", StorageIsBtrfs); + } Logger.LogDebug("Initializing Lightless FSW on {path}", lightlessPath); LightlessWatcher = new() diff --git a/LightlessSync/FileCache/FileCacheManager.cs b/LightlessSync/FileCache/FileCacheManager.cs index 972c4d9..7ee6c99 100644 --- a/LightlessSync/FileCache/FileCacheManager.cs +++ b/LightlessSync/FileCache/FileCacheManager.cs @@ -203,42 +203,72 @@ public sealed class FileCacheManager : IHostedService return output; } - public Task> ValidateLocalIntegrity(IProgress<(int, int, FileCacheEntity)> progress, CancellationToken cancellationToken) + public async Task> ValidateLocalIntegrity(IProgress<(int completed, int total, FileCacheEntity current)> progress, CancellationToken cancellationToken) { _lightlessMediator.Publish(new HaltScanMessage(nameof(ValidateLocalIntegrity))); _logger.LogInformation("Validating local storage"); - var cacheEntries = _fileCaches.Values.SelectMany(v => v.Values.Where(e => e != null)).Where(v => v.IsCacheEntry).ToList(); - List brokenEntities = []; - int i = 0; - foreach (var fileCache in cacheEntries) + + var cacheEntries = _fileCaches.Values + .SelectMany(v => v.Values) + .Where(v => v.IsCacheEntry) + .ToList(); + + int total = cacheEntries.Count; + int processed = 0; + var brokenEntities = new ConcurrentBag(); + + _logger.LogInformation("Checking {count} cache entries...", total); + + await Parallel.ForEachAsync(cacheEntries, new ParallelOptions + { + MaxDegreeOfParallelism = Environment.ProcessorCount, + CancellationToken = cancellationToken + }, + async (fileCache, token) => { - if (cancellationToken.IsCancellationRequested) break; - - _logger.LogInformation("Validating {file}", fileCache.ResolvedFilepath); - - progress.Report((i, cacheEntries.Count, fileCache)); - i++; - if (!File.Exists(fileCache.ResolvedFilepath)) - { - brokenEntities.Add(fileCache); - continue; - } - try { - var computedHash = Crypto.GetFileHash(fileCache.ResolvedFilepath); + int current = Interlocked.Increment(ref processed); + if (current % 10 == 0) + progress.Report((current, total, fileCache)); + + if (!File.Exists(fileCache.ResolvedFilepath)) + { + brokenEntities.Add(fileCache); + return; + } + + string computedHash; + try + { + computedHash = await Crypto.GetFileHashAsync(fileCache.ResolvedFilepath, token).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error hashing {file}", fileCache.ResolvedFilepath); + brokenEntities.Add(fileCache); + return; + } + if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal)) { - _logger.LogInformation("Failed to validate {file}, got hash {computedHash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash); + _logger.LogInformation( + "Hash mismatch: {file} (got {computedHash}, expected {expected})", + fileCache.ResolvedFilepath, computedHash, fileCache.Hash); + brokenEntities.Add(fileCache); } } - catch (Exception e) + catch (OperationCanceledException) { - _logger.LogWarning(e, "Error during validation of {file}", fileCache.ResolvedFilepath); + _logger.LogError("Validation got cancelled for {file}", fileCache.ResolvedFilepath); + } + catch (Exception ex) + { + _logger.LogError(ex, "Unexpected error validating {file}", fileCache.ResolvedFilepath); brokenEntities.Add(fileCache); } - } + }).ConfigureAwait(false); foreach (var brokenEntity in brokenEntities) { @@ -250,12 +280,14 @@ public sealed class FileCacheManager : IHostedService } catch (Exception ex) { - _logger.LogWarning(ex, "Could not delete {file}", brokenEntity.ResolvedFilepath); + _logger.LogWarning(ex, "Failed to delete invalid cache file {file}", brokenEntity.ResolvedFilepath); } } _lightlessMediator.Publish(new ResumeScanMessage(nameof(ValidateLocalIntegrity))); - return Task.FromResult(brokenEntities); + _logger.LogInformation("Validation complete. Found {count} invalid entries.", brokenEntities.Count); + + return [.. brokenEntities]; } public string GetCacheFilePath(string hash, string extension) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 1a35ad6..e5d219e 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,11 +1,15 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; +using LightlessSync.Utils; using Microsoft.Extensions.Logging; -using System.Runtime.InteropServices; using System.Collections.Concurrent; +using System.Diagnostics; +using System.IO; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; +using static LightlessSync.Utils.FileSystemHelper; namespace LightlessSync.FileCache; @@ -87,25 +91,51 @@ public sealed class FileCompactor : IDisposable public long GetFileSizeOnDisk(FileInfo fileInfo, bool? isNTFS = null) { - bool ntfs = isNTFS ?? string.Equals(new DriveInfo(fileInfo.Directory!.Root.FullName).DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase); + var fsType = FileSystemHelper.GetFilesystemType(fileInfo.FullName); - if (_dalamudUtilService.IsWine || !ntfs) return fileInfo.Length; + bool ntfs = isNTFS ?? fsType == FileSystemHelper.FilesystemType.NTFS; - var clusterSize = GetClusterSize(fileInfo); - if (clusterSize == -1) return fileInfo.Length; - var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); - var size = (long)hosize << 32 | losize; - return ((size + clusterSize - 1) / clusterSize) * clusterSize; + if (fsType != FileSystemHelper.FilesystemType.Btrfs && !ntfs) + { + return fileInfo.Length; + } + + if (ntfs && !_dalamudUtilService.IsWine) + { + var clusterSize = GetClusterSize(fileInfo); + if (clusterSize == -1) return fileInfo.Length; + var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); + var size = (long)hosize << 32 | losize; + return ((size + clusterSize - 1) / clusterSize) * clusterSize; + } + + if (fsType == FileSystemHelper.FilesystemType.Btrfs) + { + try + { + long blocks = RunStatGetBlocks(fileInfo.FullName); + return blocks * 512L; + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed to get on-disk size via stat for {file}, falling back to Length", fileInfo.FullName); + return fileInfo.Length; + } + } + + return fileInfo.Length; } public async Task WriteAllBytesAsync(string filePath, byte[] decompressedFile, CancellationToken token) { + var dir = Path.GetDirectoryName(filePath); + if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) + Directory.CreateDirectory(dir); + await File.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false); - if (_dalamudUtilService.IsWine || !_lightlessConfigService.Current.UseCompactor) - { + if (!_lightlessConfigService.Current.UseCompactor) return; - } EnqueueCompaction(filePath); } @@ -153,56 +183,178 @@ public sealed class FileCompactor : IDisposable private void CompactFile(string filePath) { - var fs = new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName); - bool isNTFS = string.Equals(fs.DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase); - if (!isNTFS) + var fi = new FileInfo(filePath); + if (!fi.Exists) { - _logger.LogWarning("Drive for file {file} is not NTFS", filePath); + _logger.LogDebug("Skipping compaction for missing file {file}", filePath); return; } - var fi = new FileInfo(filePath); + var fsType = FileSystemHelper.GetFilesystemType(filePath); var oldSize = fi.Length; - var clusterSize = GetClusterSize(fi); + int clusterSize = GetClusterSize(fi); if (oldSize < Math.Max(clusterSize, 8 * 1024)) { _logger.LogDebug("File {file} is smaller than cluster size ({size}), ignoring", filePath, clusterSize); return; } - if (!IsCompactedFile(filePath)) + // NTFS Compression. + if (fsType == FileSystemHelper.FilesystemType.NTFS && !_dalamudUtilService.IsWine) { - _logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath); + if (!IsWOFCompactedFile(filePath)) + { + _logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath); + var success = WOFCompressFile(filePath); - WOFCompressFile(filePath); + if (success) + { + var newSize = GetFileSizeOnDisk(fi); + _logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); + } + else + { + _logger.LogWarning("NTFS compression failed or not available for {file}", filePath); + } - var newSize = GetFileSizeOnDisk(fi); - - _logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); + } + else + { + _logger.LogDebug("File {file} already compressed (NTFS)", filePath); + } } - else + + // BTRFS Compression + if (fsType == FileSystemHelper.FilesystemType.Btrfs) { - _logger.LogDebug("File {file} already compressed", filePath); + if (!IsBtrfsCompressedFile(filePath)) + { + _logger.LogDebug("Attempting btrfs compression for {file}", filePath); + var success = BtrfsCompressFile(filePath); + + if (success) + { + var newSize = GetFileSizeOnDisk(fi); + _logger.LogDebug("Btrfs-compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); + } + else + { + _logger.LogWarning("Btrfs compression failed or not available for {file}", filePath); + } + } + else + { + _logger.LogDebug("File {file} already compressed (Btrfs)", filePath); + } } } + private static long RunStatGetBlocks(string path) + { + var psi = new ProcessStartInfo("stat", $"-c %b \"{path}\"") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat process"); + var outp = proc.StandardOutput.ReadToEnd(); + var err = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + if (proc.ExitCode != 0) + { + throw new InvalidOperationException($"stat failed: {err}"); + } + + if (!long.TryParse(outp.Trim(), out var blocks)) + { + throw new InvalidOperationException($"invalid stat output: {outp}"); + } + + return blocks; + } + private void DecompressFile(string path) { _logger.LogDebug("Removing compression from {file}", path); - try + var fsType = FileSystemHelper.GetFilesystemType(path); + if (fsType == null) return; + + //NTFS Decompression + if (fsType == FileSystemHelper.FilesystemType.NTFS && !_dalamudUtilService.IsWine) { - using (var fs = new FileStream(path, FileMode.Open)) + try { + using (var fs = new FileStream(path, FileMode.Open)) + { #pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called - var hDevice = fs.SafeFileHandle.DangerousGetHandle(); + var hDevice = fs.SafeFileHandle.DangerousGetHandle(); #pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called - _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _); + _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _); + } } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error decompressing file {path}", path); + } + return; } - catch (Exception ex) + + //BTRFS Decompression + if (fsType == FileSystemHelper.FilesystemType.Btrfs) { - _logger.LogWarning(ex, "Error decompressing file {path}", path); + try + { + var mountOptions = GetMountOptionsForPath(path); + if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) + { + _logger.LogWarning( + "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " + + "Remount with 'compress=no' before running decompression.", path, mountOptions); + return; + } + + _logger.LogDebug("Rewriting {file} to remove btrfs compression...", path); + + var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -- \"{path}\"") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using var proc = Process.Start(psi); + if (proc == null) + { + _logger.LogWarning("Failed to start btrfs defragment for decompression of {file}", path); + return; + } + + var stdout = proc.StandardOutput.ReadToEnd(); + var stderr = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + + if (proc.ExitCode != 0) + { + _logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr); + } + else + { + // Log output only in debug mode to avoid clutter + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogDebug("btrfs defragment output for {file}: {out}", path, stdout.Trim()); + + _logger.LogInformation("Decompressed (rewritten uncompressed) btrfs file: {file}", path); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error rewriting {file} for decompression", path); + } } } @@ -220,7 +372,7 @@ public sealed class FileCompactor : IDisposable return _clusterSizes[root]; } - private static bool IsCompactedFile(string filePath) + private static bool IsWOFCompactedFile(string filePath) { uint buf = 8; _ = WofIsExternalFile(filePath, out int isExtFile, out uint _, out var info, ref buf); @@ -228,40 +380,151 @@ public sealed class FileCompactor : IDisposable return info.Algorithm == CompressionAlgorithm.XPRESS8K; } - private void WOFCompressFile(string path) + private bool IsBtrfsCompressedFile(string path) + { + try + { + var psi = new ProcessStartInfo("filefrag", $"-v \"{path}\"") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using var proc = Process.Start(psi); + if (proc == null) + { + _logger.LogWarning("Failed to start filefrag for {file}", path); + return false; + } + + string output = proc.StandardOutput.ReadToEnd(); + proc.WaitForExit(); + + // look for "flags: compressed" in the output + if (output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return false; + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed to detect btrfs compression for {file}", path); + return false; + } + } + + private bool WOFCompressFile(string path) { var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo)); Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: true); ulong length = (ulong)Marshal.SizeOf(_efInfo); try { - using (var fs = new FileStream(path, FileMode.Open)) + using var fs = new FileStream(path, FileMode.Open); + #pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called + var hFile = fs.SafeFileHandle.DangerousGetHandle(); + #pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called + if (fs.SafeFileHandle.IsInvalid) { -#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called - var hFile = fs.SafeFileHandle.DangerousGetHandle(); -#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called - if (fs.SafeFileHandle.IsInvalid) - { - _logger.LogWarning("Invalid file handle to {file}", path); - } - else - { - var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length); - if (!(ret == 0 || ret == unchecked((int)0x80070158))) - { - _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); - } - } + _logger.LogWarning("Invalid file handle to {file}", path); + return false; + } + + var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length); + if (!(ret == 0 || ret == unchecked((int)0x80070158))) + { + _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); + return false; } } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting file {path}", path); + return false; } finally { Marshal.FreeHGlobal(efInfoPtr); } + return true; + } + + private bool BtrfsCompressFile(string path) + { + try + { + var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -czstd -- \"{path}\"") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using var proc = Process.Start(psi); + if (proc == null) + { + _logger.LogWarning("Failed to start btrfs process for {file}", path); + return false; + } + + var stdout = proc.StandardOutput.ReadToEnd(); + var stderr = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + + if (proc.ExitCode != 0) + { + _logger.LogWarning("btrfs defrag returned {code} for {file}: {err}", proc.ExitCode, path, stderr); + return false; + } + + _logger.LogDebug("btrfs output: {out}", stdout); + return true; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); + return false; + } + } + + private string GetMountOptionsForPath(string path) + { + try + { + var fullPath = Path.GetFullPath(path); + var mounts = File.ReadAllLines("/proc/mounts"); + string bestMount = string.Empty; + string mountOptions = string.Empty; + + foreach (var line in mounts) + { + var parts = line.Split(' '); + if (parts.Length < 4) continue; + var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); // unescape spaces + string normalized; + try { normalized = Path.GetFullPath(mountPoint); } + catch { normalized = mountPoint; } + + if (fullPath.StartsWith(normalized, StringComparison.Ordinal) && + normalized.Length > bestMount.Length) + { + bestMount = normalized; + mountOptions = parts[3]; + } + } + + return mountOptions; + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed to get mount options for {path}", path); + return string.Empty; + } } private struct WOF_FILE_COMPRESSION_INFO_V1 @@ -273,7 +536,14 @@ public sealed class FileCompactor : IDisposable private void EnqueueCompaction(string filePath) { if (!_pendingCompactions.TryAdd(filePath, 0)) + return; + + var fsType = GetFilesystemType(filePath); + + if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) { + _logger.LogTrace("Skipping compaction enqueue for unsupported filesystem {fs} ({file})", fsType, filePath); + _pendingCompactions.TryRemove(filePath, out _); return; } @@ -282,6 +552,10 @@ public sealed class FileCompactor : IDisposable _pendingCompactions.TryRemove(filePath, out _); _logger.LogDebug("Failed to enqueue compaction job for {file}", filePath); } + else + { + _logger.LogTrace("Queued compaction job for {file} (fs={fs})", filePath, fsType); + } } private async Task ProcessQueueAsync(CancellationToken token) @@ -299,7 +573,7 @@ public sealed class FileCompactor : IDisposable return; } - if (_dalamudUtilService.IsWine || !_lightlessConfigService.Current.UseCompactor) + if (!_lightlessConfigService.Current.UseCompactor) { continue; } diff --git a/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs b/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs index 2b003bb..20302fe 100644 --- a/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs +++ b/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs @@ -13,5 +13,4 @@ public class ServerStorage public bool UseOAuth2 { get; set; } = false; public string? OAuthToken { get; set; } = null; public HttpTransportType HttpTransportType { get; set; } = HttpTransportType.WebSockets; - public bool ForceWebSockets { get; set; } = false; } \ No newline at end of file diff --git a/LightlessSync/UI/SettingsUi.cs b/LightlessSync/UI/SettingsUi.cs index febc142..d686a75 100644 --- a/LightlessSync/UI/SettingsUi.cs +++ b/LightlessSync/UI/SettingsUi.cs @@ -1227,16 +1227,16 @@ public class SettingsUi : WindowMediatorSubscriberBase ImGui.TextUnformatted($"Currently utilized local storage: Calculating..."); ImGui.TextUnformatted( $"Remaining space free on drive: {UiSharedService.ByteToString(_cacheMonitor.FileCacheDriveFree)}"); + bool useFileCompactor = _configService.Current.UseCompactor; - bool isLinux = _dalamudUtilService.IsWine; - if (!useFileCompactor && !isLinux) + if (!useFileCompactor) { UiSharedService.ColorTextWrapped( "Hint: To free up space when using Lightless consider enabling the File Compactor", UIColors.Get("LightlessYellow")); } - if (isLinux || !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); + if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); if (ImGui.Checkbox("Use file compactor", ref useFileCompactor)) { _configService.Current.UseCompactor = useFileCompactor; @@ -1281,10 +1281,20 @@ public class SettingsUi : WindowMediatorSubscriberBase UIColors.Get("LightlessYellow")); } - if (isLinux || !_cacheMonitor.StorageisNTFS) + if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) { ImGui.EndDisabled(); - ImGui.TextUnformatted("The file compactor is only available on Windows and NTFS drives."); + ImGui.TextUnformatted("The file compactor is only available on BTRFS and NTFS drives."); + } + + if (_cacheMonitor.StorageisNTFS) + { + ImGui.TextUnformatted("The file compactor is running on NTFS Drive."); + } + + if (_cacheMonitor.StorageIsBtrfs) + { + ImGui.TextUnformatted("The file compactor is running on Btrfs Drive."); } ImGuiHelpers.ScaledDummy(new Vector2(10, 10)); @@ -3113,22 +3123,6 @@ public class SettingsUi : WindowMediatorSubscriberBase UiSharedService.TooltipSeparator + "Note: if the server does not support a specific Transport Type it will fall through to the next automatically: WebSockets > ServerSentEvents > LongPolling"); - if (_dalamudUtilService.IsWine) - { - bool forceWebSockets = selectedServer.ForceWebSockets; - if (ImGui.Checkbox("[wine only] Force WebSockets", ref forceWebSockets)) - { - selectedServer.ForceWebSockets = forceWebSockets; - _serverConfigurationManager.Save(); - } - - _uiShared.DrawHelpText( - "On wine, Lightless will automatically fall back to ServerSentEvents/LongPolling, even if WebSockets is selected. " - + "WebSockets are known to crash XIV entirely on wine 8.5 shipped with Dalamud. " - + "Only enable this if you are not running wine 8.5." + Environment.NewLine - + "Note: If the issue gets resolved at some point this option will be removed."); - } - ImGuiHelpers.ScaledDummy(5); if (ImGui.Checkbox("Use Discord OAuth2 Authentication", ref useOauth)) diff --git a/LightlessSync/Utils/Crypto.cs b/LightlessSync/Utils/Crypto.cs index de04d26..87d6883 100644 --- a/LightlessSync/Utils/Crypto.cs +++ b/LightlessSync/Utils/Crypto.cs @@ -21,6 +21,26 @@ public static class Crypto return BitConverter.ToString(sha1.ComputeHash(stream)).Replace("-", "", StringComparison.Ordinal); } + public static async Task GetFileHashAsync(string filePath, CancellationToken cancellationToken = default) + { + var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, bufferSize: 65536, options: FileOptions.Asynchronous); + await using (stream.ConfigureAwait(false)) + { + using var sha1 = SHA1.Create(); + + var buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0) + { + sha1.TransformBlock(buffer, 0, bytesRead, outputBuffer: null, 0); + } + + sha1.TransformFinalBlock([], 0, 0); + + return BitConverter.ToString(sha1.Hash!).Replace("-", "", StringComparison.Ordinal); + } + } + public static string GetHash256(this (string, ushort) playerToHash) { if (_hashListPlayersSHA256.TryGetValue(playerToHash, out var hash)) diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs new file mode 100644 index 0000000..a80e922 --- /dev/null +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -0,0 +1,143 @@ +using System.Collections.Concurrent; +using System.Runtime.InteropServices; + +namespace LightlessSync.Utils +{ + public static class FileSystemHelper + { + public enum FilesystemType + { + Unknown = 0, + NTFS, + Btrfs, + Ext4, + Xfs, + Apfs, + HfsPlus, + Fat, + Exfat, + Zfs + } + + private static readonly ConcurrentDictionary _filesystemTypeCache = new(StringComparer.OrdinalIgnoreCase); + + public static FilesystemType GetFilesystemType(string filePath) + { + try + { + string rootPath; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + var info = new FileInfo(filePath); + var dir = info.Directory ?? new DirectoryInfo(filePath); + rootPath = dir.Root.FullName; + } + else + { + rootPath = GetMountPoint(filePath); + if (string.IsNullOrEmpty(rootPath)) + rootPath = "/"; + } + + if (_filesystemTypeCache.TryGetValue(rootPath, out var cachedType)) + return cachedType; + + FilesystemType detected; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + var root = new DriveInfo(rootPath); + var format = root.DriveFormat?.ToUpperInvariant() ?? string.Empty; + detected = format switch + { + "NTFS" => FilesystemType.NTFS, + "FAT32" => FilesystemType.Fat, + "EXFAT" => FilesystemType.Exfat, + _ => FilesystemType.Unknown + }; + } + else + { + detected = GetLinuxFilesystemType(filePath); + } + + _filesystemTypeCache[rootPath] = detected; + return detected; + } + catch (Exception ex) + { + return FilesystemType.Unknown; + } + } + + private static string GetMountPoint(string filePath) + { + try + { + var path = Path.GetFullPath(filePath); + if (!File.Exists("/proc/mounts")) return "/"; + var mounts = File.ReadAllLines("/proc/mounts"); + + string bestMount = "/"; + foreach (var line in mounts) + { + var parts = line.Split(' '); + if (parts.Length < 3) continue; + var mountPoint = parts[1].Replace("\\040", " "); // unescape spaces + + string normalizedMount; + try { normalizedMount = Path.GetFullPath(mountPoint); } + catch { normalizedMount = mountPoint; } + + if (path.StartsWith(normalizedMount, StringComparison.Ordinal) && + normalizedMount.Length > bestMount.Length) + { + bestMount = normalizedMount; + } + } + + return bestMount; + } + catch + { + return "/"; + } + } + + private static FilesystemType GetLinuxFilesystemType(string filePath) + { + try + { + var mountPoint = GetMountPoint(filePath); + var mounts = File.ReadAllLines("/proc/mounts"); + + foreach (var line in mounts) + { + var parts = line.Split(' '); + if (parts.Length < 3) continue; + var mount = parts[1].Replace("\\040", " "); + if (string.Equals(mount, mountPoint, StringComparison.Ordinal)) + { + var fstype = parts[2].ToLowerInvariant(); + return fstype switch + { + "btrfs" => FilesystemType.Btrfs, + "ext4" => FilesystemType.Ext4, + "xfs" => FilesystemType.Xfs, + "zfs" => FilesystemType.Zfs, + "apfs" => FilesystemType.Apfs, + "hfsplus" => FilesystemType.HfsPlus, + _ => FilesystemType.Unknown + }; + } + } + + return FilesystemType.Unknown; + } + catch + { + return FilesystemType.Unknown; + } + } + } +} diff --git a/LightlessSync/WebAPI/SignalR/HubFactory.cs b/LightlessSync/WebAPI/SignalR/HubFactory.cs index ef9f919..1d5a0c8 100644 --- a/LightlessSync/WebAPI/SignalR/HubFactory.cs +++ b/LightlessSync/WebAPI/SignalR/HubFactory.cs @@ -70,13 +70,6 @@ public class HubFactory : MediatorSubscriberBase _ => HttpTransportType.WebSockets | HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling }; - if (_isWine && !_serverConfigurationManager.CurrentServer.ForceWebSockets - && transportType.HasFlag(HttpTransportType.WebSockets)) - { - Logger.LogDebug("Wine detected, falling back to ServerSentEvents / LongPolling"); - transportType = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; - } - Logger.LogDebug("Building new HubConnection using transport {transport}", transportType); _instance = new HubConnectionBuilder() From 9a846a37d40c1d3115dc7bf21438fa39a489bdac Mon Sep 17 00:00:00 2001 From: cake Date: Wed, 29 Oct 2025 04:43:18 +0100 Subject: [PATCH 02/13] Redone handling of windows compactor handling. --- LightlessSync/FileCache/FileCompactor.cs | 40 +++++++++++++++++------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index e5d219e..dbc2574 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -372,6 +372,22 @@ public sealed class FileCompactor : IDisposable return _clusterSizes[root]; } + public static bool UseSafeHandle(SafeHandle handle, Func action) + { + bool addedRef = false; + try + { + handle.DangerousAddRef(ref addedRef); + IntPtr ptr = handle.DangerousGetHandle(); + return action(ptr); + } + finally + { + if (addedRef) + handle.DangerousRelease(); + } + } + private static bool IsWOFCompactedFile(string filePath) { uint buf = 8; @@ -424,22 +440,25 @@ public sealed class FileCompactor : IDisposable ulong length = (ulong)Marshal.SizeOf(_efInfo); try { - using var fs = new FileStream(path, FileMode.Open); - #pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called - var hFile = fs.SafeFileHandle.DangerousGetHandle(); - #pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called - if (fs.SafeFileHandle.IsInvalid) + using var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None); + var handle = fs.SafeFileHandle; + + if (handle.IsInvalid) { _logger.LogWarning("Invalid file handle to {file}", path); return false; } - var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length); - if (!(ret == 0 || ret == unchecked((int)0x80070158))) + return UseSafeHandle(handle, hFile => { - _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); - return false; - } + int ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length); + if (ret != 0 && ret != unchecked((int)0x80070158)) + { + _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); + return false; + } + return true; + }); } catch (Exception ex) { @@ -450,7 +469,6 @@ public sealed class FileCompactor : IDisposable { Marshal.FreeHGlobal(efInfoPtr); } - return true; } private bool BtrfsCompressFile(string path) From 3e626c5e47042c7941a9c52460c72ec98a92ac21 Mon Sep 17 00:00:00 2001 From: cake Date: Wed, 29 Oct 2025 04:49:46 +0100 Subject: [PATCH 03/13] Cleanup some code, removed ntfs usage on cache monitor --- LightlessSync/FileCache/CacheMonitor.cs | 2 +- LightlessSync/FileCache/FileCompactor.cs | 143 ++++++++++++++++++----- 2 files changed, 112 insertions(+), 33 deletions(-) diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 23f5c19..c724225 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -429,7 +429,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase try { - return _fileCompactor.GetFileSizeOnDisk(f, StorageisNTFS); + return _fileCompactor.GetFileSizeOnDisk(f); } catch { diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index dbc2574..e961fbd 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -4,11 +4,8 @@ using LightlessSync.Utils; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Diagnostics; -using System.IO; using System.Runtime.InteropServices; -using System.Threading; using System.Threading.Channels; -using System.Threading.Tasks; using static LightlessSync.Utils.FileSystemHelper; namespace LightlessSync.FileCache; @@ -89,27 +86,21 @@ public sealed class FileCompactor : IDisposable MassCompactRunning = false; } - public long GetFileSizeOnDisk(FileInfo fileInfo, bool? isNTFS = null) + public long GetFileSizeOnDisk(FileInfo fileInfo) { - var fsType = FileSystemHelper.GetFilesystemType(fileInfo.FullName); + var fsType = GetFilesystemType(fileInfo.FullName); - bool ntfs = isNTFS ?? fsType == FileSystemHelper.FilesystemType.NTFS; - - if (fsType != FileSystemHelper.FilesystemType.Btrfs && !ntfs) + if (fsType != FilesystemType.Btrfs && fsType != FilesystemType.NTFS) { return fileInfo.Length; } - if (ntfs && !_dalamudUtilService.IsWine) + if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { - var clusterSize = GetClusterSize(fileInfo); - if (clusterSize == -1) return fileInfo.Length; - var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); - var size = (long)hosize << 32 | losize; - return ((size + clusterSize - 1) / clusterSize) * clusterSize; + return GetFileSizeOnDisk(fileInfo, GetClusterSize); } - if (fsType == FileSystemHelper.FilesystemType.Btrfs) + if (fsType == FilesystemType.Btrfs) { try { @@ -163,6 +154,41 @@ public sealed class FileCompactor : IDisposable GC.SuppressFinalize(this); } + [DllImport("libc", SetLastError = true)] + private static extern int statvfs(string path, out Statvfs buf); + + [StructLayout(LayoutKind.Sequential)] + private struct Statvfs + { + public ulong f_bsize; + public ulong f_frsize; + public ulong f_blocks; + public ulong f_bfree; + public ulong f_bavail; + public ulong f_files; + public ulong f_ffree; + public ulong f_favail; + public ulong f_fsid; + public ulong f_flag; + public ulong f_namemax; + } + + private static int GetLinuxBlockSize(string path) + { + try + { + int result = statvfs(path, out var buf); + if (result != 0) + return -1; + + return (int)buf.f_frsize; + } + catch + { + return -1; + } + } + [DllImport("kernel32.dll")] private static extern int DeviceIoControl(IntPtr hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out IntPtr lpBytesReturned, out IntPtr lpOverlapped); @@ -190,7 +216,7 @@ public sealed class FileCompactor : IDisposable return; } - var fsType = FileSystemHelper.GetFilesystemType(filePath); + var fsType = GetFilesystemType(filePath); var oldSize = fi.Length; int clusterSize = GetClusterSize(fi); @@ -201,7 +227,7 @@ public sealed class FileCompactor : IDisposable } // NTFS Compression. - if (fsType == FileSystemHelper.FilesystemType.NTFS && !_dalamudUtilService.IsWine) + if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { if (!IsWOFCompactedFile(filePath)) { @@ -250,6 +276,17 @@ public sealed class FileCompactor : IDisposable } } + private static long GetFileSizeOnDisk(FileInfo fileInfo, Func getClusterSize) + { + int clusterSize = getClusterSize(fileInfo); + if (clusterSize <= 0) + return fileInfo.Length; + + uint low = GetCompressedFileSizeW(fileInfo.FullName, out uint high); + long compressed = ((long)high << 32) | low; + return ((compressed + clusterSize - 1) / clusterSize) * clusterSize; + } + private static long RunStatGetBlocks(string path) { var psi = new ProcessStartInfo("stat", $"-c %b \"{path}\"") @@ -280,11 +317,10 @@ public sealed class FileCompactor : IDisposable private void DecompressFile(string path) { _logger.LogDebug("Removing compression from {file}", path); - var fsType = FileSystemHelper.GetFilesystemType(path); - if (fsType == null) return; + var fsType = GetFilesystemType(path); //NTFS Decompression - if (fsType == FileSystemHelper.FilesystemType.NTFS && !_dalamudUtilService.IsWine) + if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { try { @@ -304,7 +340,7 @@ public sealed class FileCompactor : IDisposable } //BTRFS Decompression - if (fsType == FileSystemHelper.FilesystemType.Btrfs) + if (fsType == FilesystemType.Btrfs) { try { @@ -360,16 +396,60 @@ public sealed class FileCompactor : IDisposable private int GetClusterSize(FileInfo fi) { - if (!fi.Exists) return -1; - var root = fi.Directory?.Root.FullName.ToLower() ?? string.Empty; - if (string.IsNullOrEmpty(root)) return -1; - if (_clusterSizes.TryGetValue(root, out int value)) return value; - _logger.LogDebug("Getting Cluster Size for {path}, root {root}", fi.FullName, root); - int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _); - if (result == 0) return -1; - _clusterSizes[root] = (int)(sectorsPerCluster * bytesPerSector); - _logger.LogDebug("Determined Cluster Size for root {root}: {cluster}", root, _clusterSizes[root]); - return _clusterSizes[root]; + try + { + if (!fi.Exists) + return -1; + + var root = fi.Directory?.Root.FullName; + if (string.IsNullOrEmpty(root)) + return -1; + + root = root.ToLowerInvariant(); + + if (_clusterSizes.TryGetValue(root, out int cached)) + return cached; + + _logger.LogDebug("Determining cluster/block size for {path} (root: {root})", fi.FullName, root); + + int clusterSize; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + int result = GetDiskFreeSpaceW( + root, + out uint sectorsPerCluster, + out uint bytesPerSector, + out _, + out _); + + if (result == 0) + { + _logger.LogWarning("GetDiskFreeSpaceW failed for {root}", root); + return -1; + } + + clusterSize = (int)(sectorsPerCluster * bytesPerSector); + } + else + { + clusterSize = GetLinuxBlockSize(root); + if (clusterSize <= 0) + { + _logger.LogWarning("Failed to determine block size for {root}", root); + return -1; + } + } + + _clusterSizes[root] = clusterSize; + _logger.LogDebug("Determined cluster/block size for {root}: {cluster} bytes", root, clusterSize); + return clusterSize; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error determining cluster size for {file}", fi.FullName); + return -1; + } } public static bool UseSafeHandle(SafeHandle handle, Func action) @@ -418,7 +498,6 @@ public sealed class FileCompactor : IDisposable string output = proc.StandardOutput.ReadToEnd(); proc.WaitForExit(); - // look for "flags: compressed" in the output if (output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase)) { return true; From 3f85852618c6c3143c4ebe5fa14bc687f6269d05 Mon Sep 17 00:00:00 2001 From: cake Date: Wed, 29 Oct 2025 04:52:17 +0100 Subject: [PATCH 04/13] Added string comparisons --- LightlessSync/Utils/FileSystemHelper.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs index a80e922..a80c900 100644 --- a/LightlessSync/Utils/FileSystemHelper.cs +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -83,7 +83,7 @@ namespace LightlessSync.Utils { var parts = line.Split(' '); if (parts.Length < 3) continue; - var mountPoint = parts[1].Replace("\\040", " "); // unescape spaces + var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); string normalizedMount; try { normalizedMount = Path.GetFullPath(mountPoint); } @@ -115,7 +115,7 @@ namespace LightlessSync.Utils { var parts = line.Split(' '); if (parts.Length < 3) continue; - var mount = parts[1].Replace("\\040", " "); + var mount = parts[1].Replace("\\040", " ", StringComparison.Ordinal); if (string.Equals(mount, mountPoint, StringComparison.Ordinal)) { var fstype = parts[2].ToLowerInvariant(); From c37e3badf16da5a55c536d4a4621f669ebafcd71 Mon Sep 17 00:00:00 2001 From: cake Date: Wed, 29 Oct 2025 06:09:44 +0100 Subject: [PATCH 05/13] Check if wine is used. --- LightlessSync/FileCache/CacheMonitor.cs | 2 +- LightlessSync/FileCache/FileCompactor.cs | 26 +++++++++++------------- LightlessSync/Utils/FileSystemHelper.cs | 8 +++++--- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index c724225..95c3150 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -126,7 +126,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase Logger.LogWarning("Lightless file path is not set, cannot start the FSW for Lightless."); return; } - var fsType = FileSystemHelper.GetFilesystemType(_configService.Current.CacheFolder); + var fsType = FileSystemHelper.GetFilesystemType(_configService.Current.CacheFolder, _dalamudUtil.IsWine); if (fsType == FileSystemHelper.FilesystemType.NTFS) { diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index e961fbd..d1aa60a 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -4,6 +4,7 @@ using LightlessSync.Utils; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Diagnostics; +using System.IO; using System.Runtime.InteropServices; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; @@ -88,7 +89,7 @@ public sealed class FileCompactor : IDisposable public long GetFileSizeOnDisk(FileInfo fileInfo) { - var fsType = GetFilesystemType(fileInfo.FullName); + var fsType = GetFilesystemType(fileInfo.FullName, _dalamudUtilService.IsWine); if (fsType != FilesystemType.Btrfs && fsType != FilesystemType.NTFS) { @@ -216,7 +217,7 @@ public sealed class FileCompactor : IDisposable return; } - var fsType = GetFilesystemType(filePath); + var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); var oldSize = fi.Length; int clusterSize = GetClusterSize(fi); @@ -252,7 +253,7 @@ public sealed class FileCompactor : IDisposable } // BTRFS Compression - if (fsType == FileSystemHelper.FilesystemType.Btrfs) + if (fsType == FilesystemType.Btrfs) { if (!IsBtrfsCompressedFile(filePath)) { @@ -317,20 +318,18 @@ public sealed class FileCompactor : IDisposable private void DecompressFile(string path) { _logger.LogDebug("Removing compression from {file}", path); - var fsType = GetFilesystemType(path); + var fsType = GetFilesystemType(path, _dalamudUtilService.IsWine); //NTFS Decompression if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { try { - using (var fs = new FileStream(path, FileMode.Open)) - { + using var fs = new FileStream(path, FileMode.Open); #pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called - var hDevice = fs.SafeFileHandle.DangerousGetHandle(); + var hDevice = fs.SafeFileHandle.DangerousGetHandle(); #pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called - _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _); - } + _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _); } catch (Exception ex) { @@ -380,7 +379,6 @@ public sealed class FileCompactor : IDisposable } else { - // Log output only in debug mode to avoid clutter if (!string.IsNullOrWhiteSpace(stdout)) _logger.LogDebug("btrfs defragment output for {file}: {out}", path, stdout.Trim()); @@ -414,7 +412,7 @@ public sealed class FileCompactor : IDisposable int clusterSize; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !_dalamudUtilService.IsWine) { int result = GetDiskFreeSpaceW( root, @@ -602,7 +600,7 @@ public sealed class FileCompactor : IDisposable { var parts = line.Split(' '); if (parts.Length < 4) continue; - var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); // unescape spaces + var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); string normalized; try { normalized = Path.GetFullPath(mountPoint); } catch { normalized = mountPoint; } @@ -635,7 +633,7 @@ public sealed class FileCompactor : IDisposable if (!_pendingCompactions.TryAdd(filePath, 0)) return; - var fsType = GetFilesystemType(filePath); + var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) { @@ -700,7 +698,7 @@ public sealed class FileCompactor : IDisposable } catch (OperationCanceledException) { - // expected during shutdown + _logger.LogDebug("Queue has been cancelled by token"); } } } diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs index a80c900..a5bb427 100644 --- a/LightlessSync/Utils/FileSystemHelper.cs +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -21,12 +21,12 @@ namespace LightlessSync.Utils private static readonly ConcurrentDictionary _filesystemTypeCache = new(StringComparer.OrdinalIgnoreCase); - public static FilesystemType GetFilesystemType(string filePath) + public static FilesystemType GetFilesystemType(string filePath, bool isWine = false) { try { string rootPath; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (!IsProbablyWine() || !isWine)) { var info = new FileInfo(filePath); var dir = info.Directory ?? new DirectoryInfo(filePath); @@ -44,7 +44,7 @@ namespace LightlessSync.Utils FilesystemType detected; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (!IsProbablyWine() || !isWine)) { var root = new DriveInfo(rootPath); var format = root.DriveFormat?.ToUpperInvariant() ?? string.Empty; @@ -139,5 +139,7 @@ namespace LightlessSync.Utils return FilesystemType.Unknown; } } + + private static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); } } From 7c4d0fd5e99a897c0bbaa6ae2e819cf98e17185e Mon Sep 17 00:00:00 2001 From: cake Date: Wed, 29 Oct 2025 22:54:50 +0100 Subject: [PATCH 06/13] Added comments, clean-up --- LightlessSync/FileCache/CacheMonitor.cs | 12 ++++++--- LightlessSync/FileCache/FileCompactor.cs | 31 ++++++++++++------------ LightlessSync/Utils/Crypto.cs | 27 ++++++++++----------- LightlessSync/Utils/FileSystemHelper.cs | 14 ++++++----- 4 files changed, 45 insertions(+), 39 deletions(-) diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 95c3150..64910f3 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -409,11 +409,17 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase return; } - FileCacheSize = -1; - DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName); + FileCacheSize = -1; + + var drive = DriveInfo.GetDrives().FirstOrDefault(d => _configService.Current.CacheFolder.StartsWith(d.Name, StringComparison.Ordinal)); + if (drive == null) + { + return; + } + try { - FileCacheDriveFree = di.AvailableFreeSpace; + FileCacheDriveFree = drive.AvailableFreeSpace; } catch (Exception ex) { diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index d1aa60a..9a73e81 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,10 +1,8 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; -using LightlessSync.Utils; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Diagnostics; -using System.IO; using System.Runtime.InteropServices; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; @@ -106,6 +104,7 @@ public sealed class FileCompactor : IDisposable try { long blocks = RunStatGetBlocks(fileInfo.FullName); + //st_blocks are always calculated in 512-byte units, hence we use 512L return blocks * 512L; } catch (Exception ex) @@ -161,17 +160,17 @@ public sealed class FileCompactor : IDisposable [StructLayout(LayoutKind.Sequential)] private struct Statvfs { - public ulong f_bsize; - public ulong f_frsize; - public ulong f_blocks; - public ulong f_bfree; - public ulong f_bavail; - public ulong f_files; - public ulong f_ffree; - public ulong f_favail; - public ulong f_fsid; - public ulong f_flag; - public ulong f_namemax; + public ulong f_bsize; /* Filesystem block size */ + public ulong f_frsize; /* Fragment size */ + public ulong f_blocks; /* Size of fs in f_frsize units */ + public ulong f_bfree; /* Number of free blocks */ + public ulong f_bavail; /* Number of free blocks for unprivileged users */ + public ulong f_files; /* Number of inodes */ + public ulong f_ffree; /* Number of free inodes */ + public ulong f_favail; /* Number of free inodes for unprivileged users */ + public ulong f_fsid; /* Filesystem ID */ + public ulong f_flag; /* Mount flags */ + public ulong f_namemax; /* Maximum filename length */ } private static int GetLinuxBlockSize(string path) @@ -182,6 +181,7 @@ public sealed class FileCompactor : IDisposable if (result != 0) return -1; + //return fragment size of linux return (int)buf.f_frsize; } catch @@ -346,9 +346,7 @@ public sealed class FileCompactor : IDisposable var mountOptions = GetMountOptionsForPath(path); if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) { - _logger.LogWarning( - "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " + - "Remount with 'compress=no' before running decompression.", path, mountOptions); + _logger.LogWarning("Cannot safely decompress {file}: filesystem mounted with compression ({opts}). Remount with 'compress=no' before running decompression.", path, mountOptions); return; } @@ -369,6 +367,7 @@ public sealed class FileCompactor : IDisposable return; } + //End stream of process to read the files var stdout = proc.StandardOutput.ReadToEnd(); var stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); diff --git a/LightlessSync/Utils/Crypto.cs b/LightlessSync/Utils/Crypto.cs index 87d6883..c31f82f 100644 --- a/LightlessSync/Utils/Crypto.cs +++ b/LightlessSync/Utils/Crypto.cs @@ -1,16 +1,15 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Security.Cryptography; +using System.Security.Cryptography; using System.Text; namespace LightlessSync.Utils; public static class Crypto { + //This buffersize seems to be the best sweetpoint for Linux and Windows + private const int _bufferSize = 65536; #pragma warning disable SYSLIB0021 // Type or member is obsolete - private static readonly Dictionary<(string, ushort), string> _hashListPlayersSHA256 = new(); + private static readonly Dictionary<(string, ushort), string> _hashListPlayersSHA256 = []; private static readonly Dictionary _hashListSHA256 = new(StringComparer.Ordinal); private static readonly SHA256CryptoServiceProvider _sha256CryptoProvider = new(); @@ -23,21 +22,21 @@ public static class Crypto public static async Task GetFileHashAsync(string filePath, CancellationToken cancellationToken = default) { - var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, bufferSize: 65536, options: FileOptions.Asynchronous); + var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, bufferSize: _bufferSize, options: FileOptions.Asynchronous); await using (stream.ConfigureAwait(false)) { using var sha1 = SHA1.Create(); - var buffer = new byte[8192]; - int bytesRead; - while ((bytesRead = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0) - { - sha1.TransformBlock(buffer, 0, bytesRead, outputBuffer: null, 0); - } + var buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0) + { + sha1.TransformBlock(buffer, 0, bytesRead, outputBuffer: null, 0); + } - sha1.TransformFinalBlock([], 0, 0); + sha1.TransformFinalBlock([], 0, 0); - return BitConverter.ToString(sha1.Hash!).Replace("-", "", StringComparison.Ordinal); + return Convert.ToHexString(sha1.Hash!); } } diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs index a5bb427..cda36c3 100644 --- a/LightlessSync/Utils/FileSystemHelper.cs +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -8,17 +8,18 @@ namespace LightlessSync.Utils public enum FilesystemType { Unknown = 0, - NTFS, - Btrfs, + NTFS, // Compressable + Btrfs, // Compressable Ext4, Xfs, Apfs, HfsPlus, Fat, Exfat, - Zfs + Zfs // Compressable } + private const string _mountPath = "/proc/mounts"; private static readonly ConcurrentDictionary _filesystemTypeCache = new(StringComparer.OrdinalIgnoreCase); public static FilesystemType GetFilesystemType(string filePath, bool isWine = false) @@ -75,8 +76,8 @@ namespace LightlessSync.Utils try { var path = Path.GetFullPath(filePath); - if (!File.Exists("/proc/mounts")) return "/"; - var mounts = File.ReadAllLines("/proc/mounts"); + if (!File.Exists(_mountPath)) return "/"; + var mounts = File.ReadAllLines(_mountPath); string bestMount = "/"; foreach (var line in mounts) @@ -109,7 +110,7 @@ namespace LightlessSync.Utils try { var mountPoint = GetMountPoint(filePath); - var mounts = File.ReadAllLines("/proc/mounts"); + var mounts = File.ReadAllLines(_mountPath); foreach (var line in mounts) { @@ -140,6 +141,7 @@ namespace LightlessSync.Utils } } + //Extra check on private static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); } } From b3cc41382f28f8edf6dd49ee023a0174ae297a66 Mon Sep 17 00:00:00 2001 From: cake Date: Thu, 30 Oct 2025 03:05:53 +0100 Subject: [PATCH 07/13] Refactored a bit, added comments on the file systems. --- LightlessSync/FileCache/FileCompactor.cs | 53 +++--------------------- LightlessSync/Utils/FileSystemHelper.cs | 52 +++++++++++++++++++---- 2 files changed, 49 insertions(+), 56 deletions(-) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 9a73e81..60cc78e 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,5 +1,6 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; +using LightlessSync.Utils; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Diagnostics; @@ -181,7 +182,7 @@ public sealed class FileCompactor : IDisposable if (result != 0) return -1; - //return fragment size of linux + //return fragment size of Linux file system return (int)buf.f_frsize; } catch @@ -413,12 +414,7 @@ public sealed class FileCompactor : IDisposable if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !_dalamudUtilService.IsWine) { - int result = GetDiskFreeSpaceW( - root, - out uint sectorsPerCluster, - out uint bytesPerSector, - out _, - out _); + int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _); if (result == 0) { @@ -495,12 +491,10 @@ public sealed class FileCompactor : IDisposable string output = proc.StandardOutput.ReadToEnd(); proc.WaitForExit(); - if (output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase)) - { - return true; - } + bool compressed = output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); - return false; + _logger.LogTrace("Btrfs compression check for {file}: {compressed}", path, compressed); + return compressed; } catch (Exception ex) { @@ -586,41 +580,6 @@ public sealed class FileCompactor : IDisposable } } - private string GetMountOptionsForPath(string path) - { - try - { - var fullPath = Path.GetFullPath(path); - var mounts = File.ReadAllLines("/proc/mounts"); - string bestMount = string.Empty; - string mountOptions = string.Empty; - - foreach (var line in mounts) - { - var parts = line.Split(' '); - if (parts.Length < 4) continue; - var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); - string normalized; - try { normalized = Path.GetFullPath(mountPoint); } - catch { normalized = mountPoint; } - - if (fullPath.StartsWith(normalized, StringComparison.Ordinal) && - normalized.Length > bestMount.Length) - { - bestMount = normalized; - mountOptions = parts[3]; - } - } - - return mountOptions; - } - catch (Exception ex) - { - _logger.LogDebug(ex, "Failed to get mount options for {path}", path); - return string.Empty; - } - } - private struct WOF_FILE_COMPRESSION_INFO_V1 { public CompressionAlgorithm Algorithm; diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs index cda36c3..4bbfc75 100644 --- a/LightlessSync/Utils/FileSystemHelper.cs +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -8,15 +8,15 @@ namespace LightlessSync.Utils public enum FilesystemType { Unknown = 0, - NTFS, // Compressable - Btrfs, // Compressable - Ext4, - Xfs, - Apfs, - HfsPlus, - Fat, - Exfat, - Zfs // Compressable + NTFS, // Compressable on file level + Btrfs, // Compressable on file level + Ext4, // Uncompressable + Xfs, // Uncompressable + Apfs, // Compressable on OS + HfsPlus, // Compressable on OS + Fat, // Uncompressable + Exfat, // Uncompressable + Zfs // Compressable, not on file level } private const string _mountPath = "/proc/mounts"; @@ -105,6 +105,40 @@ namespace LightlessSync.Utils } } + public static string GetMountOptionsForPath(string path) + { + try + { + var fullPath = Path.GetFullPath(path); + var mounts = File.ReadAllLines("/proc/mounts"); + string bestMount = string.Empty; + string mountOptions = string.Empty; + + foreach (var line in mounts) + { + var parts = line.Split(' '); + if (parts.Length < 4) continue; + var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); + string normalized; + try { normalized = Path.GetFullPath(mountPoint); } + catch { normalized = mountPoint; } + + if (fullPath.StartsWith(normalized, StringComparison.Ordinal) && + normalized.Length > bestMount.Length) + { + bestMount = normalized; + mountOptions = parts[3]; + } + } + + return mountOptions; + } + catch (Exception ex) + { + return string.Empty; + } + } + private static FilesystemType GetLinuxFilesystemType(string filePath) { try From bf139c128b75b8d744864cb9b574c74c91e65ad6 Mon Sep 17 00:00:00 2001 From: cake Date: Thu, 30 Oct 2025 03:11:38 +0100 Subject: [PATCH 08/13] Added fail safes in compact of WOF incase --- LightlessSync/FileCache/FileCompactor.cs | 32 ++++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 60cc78e..2e32a3e 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -138,7 +138,7 @@ public sealed class FileCompactor : IDisposable _compactionCts.Cancel(); try { - if (!_compactionWorker.Wait(TimeSpan.FromSeconds(5))) + if (!_compactionWorker.Wait(TimeSpan.FromSeconds(5), _compactionCts.Token)) { _logger.LogDebug("Compaction worker did not shut down within timeout"); } @@ -463,10 +463,32 @@ public sealed class FileCompactor : IDisposable private static bool IsWOFCompactedFile(string filePath) { - uint buf = 8; - _ = WofIsExternalFile(filePath, out int isExtFile, out uint _, out var info, ref buf); - if (isExtFile == 0) return false; - return info.Algorithm == CompressionAlgorithm.XPRESS8K; + try + { + uint buf = (uint)Marshal.SizeOf(); + int result = WofIsExternalFile(filePath, out int isExternal, out uint _, out var info, ref buf); + + if (result != 0 || isExternal == 0) + return false; + + return info.Algorithm == CompressionAlgorithm.XPRESS8K || info.Algorithm == CompressionAlgorithm.XPRESS4K + || info.Algorithm == CompressionAlgorithm.XPRESS16K || info.Algorithm == CompressionAlgorithm.LZX; + } + catch (DllNotFoundException) + { + // WofUtil.dll not available + return false; + } + catch (EntryPointNotFoundException) + { + // Running under Wine or non-NTFS systems + return false; + } + catch (Exception) + { + // Exception happened + return false; + } } private bool IsBtrfsCompressedFile(string path) From c1770528f3c79300ac7fc42f65f862cbb72e37d9 Mon Sep 17 00:00:00 2001 From: cake Date: Thu, 30 Oct 2025 03:34:56 +0100 Subject: [PATCH 09/13] Added wine checks, path fixing on wine -> linux --- LightlessSync/FileCache/FileCompactor.cs | 105 ++++++++++++++++++++--- 1 file changed, 91 insertions(+), 14 deletions(-) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 2e32a3e..a8170f4 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,6 +1,5 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; -using LightlessSync.Utils; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Diagnostics; @@ -191,6 +190,16 @@ public sealed class FileCompactor : IDisposable } } + private static string ConvertWinePathToLinux(string winePath) + { + if (winePath.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + return "/" + winePath.Substring(3).Replace('\\', '/'); + if (winePath.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), + winePath.Substring(3).Replace('\\', '/')).Replace('\\', '/'); + return winePath.Replace('\\', '/'); + } + [DllImport("kernel32.dll")] private static extern int DeviceIoControl(IntPtr hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out IntPtr lpBytesReturned, out IntPtr lpOverlapped); @@ -347,18 +356,42 @@ public sealed class FileCompactor : IDisposable var mountOptions = GetMountOptionsForPath(path); if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) { - _logger.LogWarning("Cannot safely decompress {file}: filesystem mounted with compression ({opts}). Remount with 'compress=no' before running decompression.", path, mountOptions); + _logger.LogWarning( + "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). Remount with 'compress=no' before running decompression.", + path, mountOptions); return; } - _logger.LogDebug("Rewriting {file} to remove btrfs compression...", path); - - var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -- \"{path}\"") + string realPath = path; + bool isWine = _dalamudUtilService?.IsWine ?? false; + if (isWine) { + if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + { + realPath = "/" + path.Substring(3).Replace('\\', '/'); + } + else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + { + // fallback for Wine's C:\ mapping + realPath = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.Personal), + path.Substring(3).Replace('\\', '/') + ).Replace('\\', '/'); + } + + _logger.LogTrace("Detected Wine environment. Converted path for decompression: {realPath}", realPath); + } + + string command = $"btrfs filesystem defragment -- \"{realPath}\""; + var psi = new ProcessStartInfo + { + FileName = isWine ? "/bin/bash" : "btrfs", + Arguments = isWine ? $"-c \"{command}\"" : $"filesystem defragment -- \"{realPath}\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, - CreateNoWindow = true + CreateNoWindow = true, + WorkingDirectory = "/" }; using var proc = Process.Start(psi); @@ -368,7 +401,7 @@ public sealed class FileCompactor : IDisposable return; } - //End stream of process to read the files + // 4️⃣ Read process output var stdout = proc.StandardOutput.ReadToEnd(); var stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); @@ -495,12 +528,36 @@ public sealed class FileCompactor : IDisposable { try { - var psi = new ProcessStartInfo("filefrag", $"-v \"{path}\"") + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = path; + + if (isWine) { + if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + { + realPath = "/" + path.Substring(3).Replace('\\', '/'); + } + else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + { + realPath = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.Personal), + path.Substring(3).Replace('\\', '/') + ).Replace('\\', '/'); + } + + _logger.LogTrace("Detected Wine environment. Converted path for filefrag: {realPath}", realPath); + } + + string command = $"filefrag -v -- \"{realPath}\""; + var psi = new ProcessStartInfo + { + FileName = isWine ? "/bin/bash" : "filefrag", + Arguments = isWine ? $"-c \"{command}\"" : $"-v -- \"{realPath}\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, - CreateNoWindow = true + CreateNoWindow = true, + WorkingDirectory = "/" }; using var proc = Process.Start(psi); @@ -511,10 +568,17 @@ public sealed class FileCompactor : IDisposable } string output = proc.StandardOutput.ReadToEnd(); + string stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); - bool compressed = output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); + if (proc.ExitCode != 0) + { + _logger.LogDebug("filefrag exited with {code} for {file}. stderr: {stderr}", + proc.ExitCode, path, stderr); + return false; + } + bool compressed = output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); _logger.LogTrace("Btrfs compression check for {file}: {compressed}", path, compressed); return compressed; } @@ -567,7 +631,20 @@ public sealed class FileCompactor : IDisposable { try { - var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -czstd -- \"{path}\"") + string realPath = path; + if (_dalamudUtilService.IsWine) + { + realPath = ConvertWinePathToLinux(path); + _logger.LogTrace("Detected Wine environment, remapped path: {realPath}", realPath); + } + + if (!File.Exists("/usr/bin/btrfs") && !File.Exists("/bin/btrfs")) + { + _logger.LogWarning("Skipping Btrfs compression — btrfs binary not found"); + return false; + } + + var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -czstd -- \"{realPath}\"") { RedirectStandardOutput = true, RedirectStandardError = true, @@ -578,7 +655,7 @@ public sealed class FileCompactor : IDisposable using var proc = Process.Start(psi); if (proc == null) { - _logger.LogWarning("Failed to start btrfs process for {file}", path); + _logger.LogWarning("Failed to start btrfs process for {file}", realPath); return false; } @@ -588,7 +665,7 @@ public sealed class FileCompactor : IDisposable if (proc.ExitCode != 0) { - _logger.LogWarning("btrfs defrag returned {code} for {file}: {err}", proc.ExitCode, path, stderr); + _logger.LogWarning("btrfs defrag returned {code} for {file}: {err}", proc.ExitCode, realPath, stderr); return false; } @@ -597,7 +674,7 @@ public sealed class FileCompactor : IDisposable } catch (Exception ex) { - _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); + _logger.LogWarning(ex, "Error running btrfs defragment for {file}", realPath); return false; } } From 5feb74c1c08594245e5a523339dc685d70c357c6 Mon Sep 17 00:00:00 2001 From: cake Date: Thu, 30 Oct 2025 03:46:55 +0100 Subject: [PATCH 10/13] Added another wine check in parralel with dalamud --- LightlessSync/FileCache/FileCompactor.cs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index a8170f4..5f05549 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -364,7 +364,7 @@ public sealed class FileCompactor : IDisposable string realPath = path; bool isWine = _dalamudUtilService?.IsWine ?? false; - if (isWine) + if (isWine && IsProbablyWine()) { if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) { @@ -372,7 +372,6 @@ public sealed class FileCompactor : IDisposable } else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) { - // fallback for Wine's C:\ mapping realPath = Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.Personal), path.Substring(3).Replace('\\', '/') @@ -401,7 +400,6 @@ public sealed class FileCompactor : IDisposable return; } - // 4️⃣ Read process output var stdout = proc.StandardOutput.ReadToEnd(); var stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); @@ -531,7 +529,7 @@ public sealed class FileCompactor : IDisposable bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = path; - if (isWine) + if (isWine && IsProbablyWine()) { if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) { @@ -539,10 +537,7 @@ public sealed class FileCompactor : IDisposable } else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) { - realPath = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.Personal), - path.Substring(3).Replace('\\', '/') - ).Replace('\\', '/'); + realPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), path.Substring(3).Replace('\\', '/')).Replace('\\', '/'); } _logger.LogTrace("Detected Wine environment. Converted path for filefrag: {realPath}", realPath); @@ -632,7 +627,7 @@ public sealed class FileCompactor : IDisposable try { string realPath = path; - if (_dalamudUtilService.IsWine) + if (_dalamudUtilService.IsWine && IsProbablyWine()) { realPath = ConvertWinePathToLinux(path); _logger.LogTrace("Detected Wine environment, remapped path: {realPath}", realPath); @@ -674,7 +669,7 @@ public sealed class FileCompactor : IDisposable } catch (Exception ex) { - _logger.LogWarning(ex, "Error running btrfs defragment for {file}", realPath); + _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); return false; } } @@ -758,4 +753,6 @@ public sealed class FileCompactor : IDisposable _logger.LogDebug("Queue has been cancelled by token"); } } + + private static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); } From 6e3c60f627c5a732dc5a15b02647b1d67989a723 Mon Sep 17 00:00:00 2001 From: cake Date: Fri, 31 Oct 2025 23:47:41 +0100 Subject: [PATCH 11/13] Changes in file compression for windows, redone linux side because wine issues. --- LightlessSync/FileCache/CacheMonitor.cs | 68 +- LightlessSync/FileCache/FileCompactor.cs | 960 ++++++++++++----------- LightlessSync/Utils/FileSystemHelper.cs | 107 ++- 3 files changed, 627 insertions(+), 508 deletions(-) diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 64910f3..85fb30e 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -137,7 +137,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase if (fsType == FileSystemHelper.FilesystemType.Btrfs) { StorageIsBtrfs = true; - Logger.LogInformation("Lightless Storage is on BTRFS drive: {isNtfs}", StorageIsBtrfs); + Logger.LogInformation("Lightless Storage is on BTRFS drive: {isBtrfs}", StorageIsBtrfs); } Logger.LogDebug("Initializing Lightless FSW on {path}", lightlessPath); @@ -661,44 +661,44 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase if (ct.IsCancellationRequested) return; - // scan new files - if (allScannedFiles.Any(c => !c.Value)) + var newFiles = allScannedFiles.Where(c => !c.Value).Select(c => c.Key).ToList(); + foreach (var cachePath in newFiles) { - Parallel.ForEach(allScannedFiles.Where(c => !c.Value).Select(c => c.Key), - new ParallelOptions() - { - MaxDegreeOfParallelism = threadCount, - CancellationToken = ct - }, (cachePath) => - { - if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null) - { - Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}", _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null); - return; - } + if (ct.IsCancellationRequested) break; + ProcessOne(cachePath); + Interlocked.Increment(ref _currentFileProgress); + } - if (ct.IsCancellationRequested) return; + Logger.LogTrace("Scanner added {count} new files to db", newFiles.Count); - if (!_ipcManager.Penumbra.APIAvailable) - { - Logger.LogWarning("Penumbra not available"); - return; - } + void ProcessOne(string? cachePath) + { + if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null) + { + Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}", + _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null); + return; + } - try - { - var entry = _fileDbManager.CreateFileEntry(cachePath); - if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath); - } - catch (Exception ex) - { - Logger.LogWarning(ex, "Failed adding {file}", cachePath); - } + if (!_ipcManager.Penumbra.APIAvailable) + { + Logger.LogWarning("Penumbra not available"); + return; + } - Interlocked.Increment(ref _currentFileProgress); - }); - - Logger.LogTrace("Scanner added {notScanned} new files to db", allScannedFiles.Count(c => !c.Value)); + try + { + var entry = _fileDbManager.CreateFileEntry(cachePath); + if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath); + } + catch (IOException ioex) + { + Logger.LogDebug(ioex, "File busy or locked: {file}", cachePath); + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Failed adding {file}", cachePath); + } } Logger.LogDebug("Scan complete"); diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 5f05549..a917b59 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,6 +1,7 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; using Microsoft.Extensions.Logging; +using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.InteropServices; @@ -14,44 +15,21 @@ public sealed class FileCompactor : IDisposable public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U; public const ulong WOF_PROVIDER_FILE = 2UL; - private readonly Dictionary _clusterSizes; private readonly ConcurrentDictionary _pendingCompactions; - private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo; private readonly ILogger _logger; - private readonly LightlessConfigService _lightlessConfigService; private readonly DalamudUtilService _dalamudUtilService; + private readonly Channel _compactionQueue; private readonly CancellationTokenSource _compactionCts = new(); private readonly Task _compactionWorker; - - public FileCompactor(ILogger logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService) + + private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo = new() { - _clusterSizes = new(StringComparer.Ordinal); - _pendingCompactions = new(StringComparer.OrdinalIgnoreCase); - _logger = logger; - _lightlessConfigService = lightlessConfigService; - _dalamudUtilService = dalamudUtilService; - _efInfo = new WOF_FILE_COMPRESSION_INFO_V1 - { - Algorithm = CompressionAlgorithm.XPRESS8K, - Flags = 0 - }; - - _compactionQueue = Channel.CreateUnbounded(new UnboundedChannelOptions - { - SingleReader = true, - SingleWriter = false - }); - _compactionWorker = Task.Factory.StartNew( - () => ProcessQueueAsync(_compactionCts.Token), - _compactionCts.Token, - TaskCreationOptions.LongRunning, - TaskScheduler.Default) - .Unwrap(); - } - - private enum CompressionAlgorithm + Algorithm = (int)CompressionAlgorithm.XPRESS8K, + Flags = 0 + }; + private enum CompressionAlgorithm { NO_COMPRESSION = -2, LZNT1 = -1, @@ -61,493 +39,500 @@ public sealed class FileCompactor : IDisposable XPRESS16K = 3 } - public bool MassCompactRunning { get; private set; } = false; + public FileCompactor(ILogger logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService) + { + _pendingCompactions = new(StringComparer.OrdinalIgnoreCase); + _logger = logger; + _lightlessConfigService = lightlessConfigService; + _dalamudUtilService = dalamudUtilService; + _compactionQueue = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = true, + SingleWriter = false + }); + + _compactionWorker = Task.Factory.StartNew(() => ProcessQueueAsync(_compactionCts.Token), _compactionCts.Token, TaskCreationOptions.LongRunning,TaskScheduler.Default).Unwrap(); + } + + public bool MassCompactRunning { get; private set; } public string Progress { get; private set; } = string.Empty; public void CompactStorage(bool compress) { MassCompactRunning = true; - - int currentFile = 1; - var allFiles = Directory.EnumerateFiles(_lightlessConfigService.Current.CacheFolder).ToList(); - int allFilesCount = allFiles.Count; - foreach (var file in allFiles) + try { - Progress = $"{currentFile}/{allFilesCount}"; - if (compress) - CompactFile(file); - else - DecompressFile(file); - currentFile++; - } + var allFiles = Directory.EnumerateFiles(_lightlessConfigService.Current.CacheFolder).ToList(); + int total = allFiles.Count; + int current = 0; - MassCompactRunning = false; + foreach (var file in allFiles) + { + current++; + Progress = $"{current}/{total}"; + + try + { + if (compress) + CompactFile(file); + else + DecompressFile(file); + } + catch (IOException ioEx) + { + _logger.LogDebug(ioEx, "File {file} locked or busy, skipping", file); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error compacting/decompressing file {file}", file); + } + } + } + finally + { + MassCompactRunning = false; + Progress = string.Empty; + } } + /// + /// Write all bytes into a directory async + /// + /// Bytes will be writen to this filepath + /// Bytes that have to be written + /// Cancellation Token for interupts + /// Writing Task + public async Task WriteAllBytesAsync(string filePath, byte[] bytes, CancellationToken token) + { + var dir = Path.GetDirectoryName(filePath); + if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) + Directory.CreateDirectory(dir); + + await File.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false); + + if (_lightlessConfigService.Current.UseCompactor) + EnqueueCompaction(filePath); + } + + /// + /// Gets the File size for an BTRFS or NTFS file system for the given FileInfo + /// + /// Amount of blocks used in the disk public long GetFileSizeOnDisk(FileInfo fileInfo) { var fsType = GetFilesystemType(fileInfo.FullName, _dalamudUtilService.IsWine); - if (fsType != FilesystemType.Btrfs && fsType != FilesystemType.NTFS) - { - return fileInfo.Length; - } - if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { - return GetFileSizeOnDisk(fileInfo, GetClusterSize); + var blockSize = GetBlockSizeForPath(fileInfo.FullName, _logger, _dalamudUtilService.IsWine); + var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); + var size = (long)hosize << 32 | losize; + return ((size + blockSize - 1) / blockSize) * blockSize; } if (fsType == FilesystemType.Btrfs) { try { - long blocks = RunStatGetBlocks(fileInfo.FullName); - //st_blocks are always calculated in 512-byte units, hence we use 512L + var realPath = fileInfo.FullName.Replace("\"", "\\\"", StringComparison.Ordinal); + var psi = new ProcessStartInfo("stat", $"-c %b \"{realPath}\"") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat"); + var outp = proc.StandardOutput.ReadToEnd(); + var err = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + + if (proc.ExitCode != 0) + throw new InvalidOperationException($"stat failed: {err}"); + + if (!long.TryParse(outp.Trim(), out var blocks)) + throw new InvalidOperationException($"invalid stat output: {outp}"); + + // st_blocks are always 512-byte on Linux enviroment. return blocks * 512L; } catch (Exception ex) { - _logger.LogDebug(ex, "Failed to get on-disk size via stat for {file}, falling back to Length", fileInfo.FullName); - return fileInfo.Length; + _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); } } return fileInfo.Length; } - public async Task WriteAllBytesAsync(string filePath, byte[] decompressedFile, CancellationToken token) - { - var dir = Path.GetDirectoryName(filePath); - if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) - Directory.CreateDirectory(dir); - - await File.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false); - - if (!_lightlessConfigService.Current.UseCompactor) - return; - - EnqueueCompaction(filePath); - } - - public void Dispose() - { - _compactionQueue.Writer.TryComplete(); - _compactionCts.Cancel(); - try - { - if (!_compactionWorker.Wait(TimeSpan.FromSeconds(5), _compactionCts.Token)) - { - _logger.LogDebug("Compaction worker did not shut down within timeout"); - } - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogDebug(ex, "Error shutting down compaction worker"); - } - finally - { - _compactionCts.Dispose(); - } - - GC.SuppressFinalize(this); - } - - [DllImport("libc", SetLastError = true)] - private static extern int statvfs(string path, out Statvfs buf); - - [StructLayout(LayoutKind.Sequential)] - private struct Statvfs - { - public ulong f_bsize; /* Filesystem block size */ - public ulong f_frsize; /* Fragment size */ - public ulong f_blocks; /* Size of fs in f_frsize units */ - public ulong f_bfree; /* Number of free blocks */ - public ulong f_bavail; /* Number of free blocks for unprivileged users */ - public ulong f_files; /* Number of inodes */ - public ulong f_ffree; /* Number of free inodes */ - public ulong f_favail; /* Number of free inodes for unprivileged users */ - public ulong f_fsid; /* Filesystem ID */ - public ulong f_flag; /* Mount flags */ - public ulong f_namemax; /* Maximum filename length */ - } - - private static int GetLinuxBlockSize(string path) - { - try - { - int result = statvfs(path, out var buf); - if (result != 0) - return -1; - - //return fragment size of Linux file system - return (int)buf.f_frsize; - } - catch - { - return -1; - } - } - - private static string ConvertWinePathToLinux(string winePath) - { - if (winePath.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - return "/" + winePath.Substring(3).Replace('\\', '/'); - if (winePath.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), - winePath.Substring(3).Replace('\\', '/')).Replace('\\', '/'); - return winePath.Replace('\\', '/'); - } - - [DllImport("kernel32.dll")] - private static extern int DeviceIoControl(IntPtr hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out IntPtr lpBytesReturned, out IntPtr lpOverlapped); - - [DllImport("kernel32.dll")] - private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, - [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh); - - [DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)] - private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, - out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters, - out uint lpTotalNumberOfClusters); - - [DllImport("WoFUtil.dll")] - private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength); - - [DllImport("WofUtil.dll")] - private static extern int WofSetFileDataLocation(IntPtr FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); - + /// + /// Compressing the given path with BTRFS or NTFS file system. + /// + /// Path of the decompressed/normal file private void CompactFile(string filePath) { var fi = new FileInfo(filePath); if (!fi.Exists) { - _logger.LogDebug("Skipping compaction for missing file {file}", filePath); + _logger.LogTrace("Skip compact: missing {file}", filePath); return; } var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); + _logger.LogTrace("Detected filesystem {fs} for {file} (isWine={wine})", fsType, filePath, _dalamudUtilService.IsWine); var oldSize = fi.Length; - int clusterSize = GetClusterSize(fi); - if (oldSize < Math.Max(clusterSize, 8 * 1024)) + int blockSize = GetBlockSizeForPath(fi.FullName, _logger, _dalamudUtilService.IsWine); + if (oldSize < Math.Max(blockSize, 8 * 1024)) { - _logger.LogDebug("File {file} is smaller than cluster size ({size}), ignoring", filePath, clusterSize); + _logger.LogTrace("Skip compact: {file} < block {block}", filePath, blockSize); return; } - // NTFS Compression. if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { if (!IsWOFCompactedFile(filePath)) { - _logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath); - var success = WOFCompressFile(filePath); - - if (success) + _logger.LogDebug("NTFS compact XPRESS8K: {file}", filePath); + if (WOFCompressFile(filePath)) { var newSize = GetFileSizeOnDisk(fi); - _logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); + _logger.LogDebug("NTFS compressed {file} {old} -> {new}", filePath, oldSize, newSize); } else { - _logger.LogWarning("NTFS compression failed or not available for {file}", filePath); + _logger.LogWarning("NTFS compression failed or unavailable for {file}", filePath); } - } else { - _logger.LogDebug("File {file} already compressed (NTFS)", filePath); - } - } - - // BTRFS Compression - if (fsType == FilesystemType.Btrfs) - { - if (!IsBtrfsCompressedFile(filePath)) - { - _logger.LogDebug("Attempting btrfs compression for {file}", filePath); - var success = BtrfsCompressFile(filePath); - - if (success) - { - var newSize = GetFileSizeOnDisk(fi); - _logger.LogDebug("Btrfs-compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); - } - else - { - _logger.LogWarning("Btrfs compression failed or not available for {file}", filePath); - } - } - else - { - _logger.LogDebug("File {file} already compressed (Btrfs)", filePath); - } - } - } - - private static long GetFileSizeOnDisk(FileInfo fileInfo, Func getClusterSize) - { - int clusterSize = getClusterSize(fileInfo); - if (clusterSize <= 0) - return fileInfo.Length; - - uint low = GetCompressedFileSizeW(fileInfo.FullName, out uint high); - long compressed = ((long)high << 32) | low; - return ((compressed + clusterSize - 1) / clusterSize) * clusterSize; - } - - private static long RunStatGetBlocks(string path) - { - var psi = new ProcessStartInfo("stat", $"-c %b \"{path}\"") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - - using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat process"); - var outp = proc.StandardOutput.ReadToEnd(); - var err = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); - if (proc.ExitCode != 0) - { - throw new InvalidOperationException($"stat failed: {err}"); - } - - if (!long.TryParse(outp.Trim(), out var blocks)) - { - throw new InvalidOperationException($"invalid stat output: {outp}"); - } - - return blocks; - } - - private void DecompressFile(string path) - { - _logger.LogDebug("Removing compression from {file}", path); - var fsType = GetFilesystemType(path, _dalamudUtilService.IsWine); - - //NTFS Decompression - if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) - { - try - { - using var fs = new FileStream(path, FileMode.Open); -#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called - var hDevice = fs.SafeFileHandle.DangerousGetHandle(); -#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called - _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Error decompressing file {path}", path); + _logger.LogTrace("Already NTFS-compressed: {file}", filePath); } return; } - //BTRFS Decompression if (fsType == FilesystemType.Btrfs) { - try + if (!IsBtrfsCompressedFile(filePath)) { - var mountOptions = GetMountOptionsForPath(path); - if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) + _logger.LogDebug("Btrfs compress zstd: {file}", filePath); + if (BtrfsCompressFile(filePath)) { - _logger.LogWarning( - "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). Remount with 'compress=no' before running decompression.", - path, mountOptions); - return; - } - - string realPath = path; - bool isWine = _dalamudUtilService?.IsWine ?? false; - if (isWine && IsProbablyWine()) - { - if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - { - realPath = "/" + path.Substring(3).Replace('\\', '/'); - } - else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - { - realPath = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.Personal), - path.Substring(3).Replace('\\', '/') - ).Replace('\\', '/'); - } - - _logger.LogTrace("Detected Wine environment. Converted path for decompression: {realPath}", realPath); - } - - string command = $"btrfs filesystem defragment -- \"{realPath}\""; - var psi = new ProcessStartInfo - { - FileName = isWine ? "/bin/bash" : "btrfs", - Arguments = isWine ? $"-c \"{command}\"" : $"filesystem defragment -- \"{realPath}\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; - - using var proc = Process.Start(psi); - if (proc == null) - { - _logger.LogWarning("Failed to start btrfs defragment for decompression of {file}", path); - return; - } - - var stdout = proc.StandardOutput.ReadToEnd(); - var stderr = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); - - if (proc.ExitCode != 0) - { - _logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr); + var newSize = GetFileSizeOnDisk(fi); + _logger.LogDebug("Btrfs compressed {file} {old} -> {new}", filePath, oldSize, newSize); } else { - if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogDebug("btrfs defragment output for {file}: {out}", path, stdout.Trim()); + _logger.LogWarning("Btrfs compression failed or unavailable for {file}", filePath); + } + } + else + { + _logger.LogTrace("Already Btrfs-compressed: {file}", filePath); + } + return; + } - _logger.LogInformation("Decompressed (rewritten uncompressed) btrfs file: {file}", path); + _logger.LogTrace("Skip compact: unsupported FS for {file}", filePath); + } + + /// + /// Decompressing the given path with BTRFS file system or NTFS file system. + /// + /// Path of the compressed file + private void DecompressFile(string path) + { + _logger.LogDebug("Decompress request: {file}", path); + var fsType = GetFilesystemType(path, _dalamudUtilService.IsWine); + + if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) + { + try + { + bool flowControl = DecompressWOFFile(path, out FileStream fs); + if (!flowControl) + { + return; } } catch (Exception ex) { - _logger.LogWarning(ex, "Error rewriting {file} for decompression", path); + _logger.LogWarning(ex, "NTFS decompress error {file}", path); + } + } + + if (fsType == FilesystemType.Btrfs) + { + try + { + bool flowControl = DecompressBtrfsFile(path); + if (!flowControl) + { + return; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Btrfs decompress error {file}", path); } } } - private int GetClusterSize(FileInfo fi) + /// + /// Decompress an BTRFS File + /// + /// Path of the compressed file + /// Decompessing state + private bool DecompressBtrfsFile(string path) { try { - if (!fi.Exists) - return -1; - - var root = fi.Directory?.Root.FullName; - if (string.IsNullOrEmpty(root)) - return -1; - - root = root.ToLowerInvariant(); - - if (_clusterSizes.TryGetValue(root, out int cached)) - return cached; - - _logger.LogDebug("Determining cluster/block size for {path} (root: {root})", fi.FullName, root); - - int clusterSize; - - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !_dalamudUtilService.IsWine) + var opts = GetMountOptionsForPath(path); + if (opts.Contains("compress", StringComparison.OrdinalIgnoreCase)) { - int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _); - - if (result == 0) - { - _logger.LogWarning("GetDiskFreeSpaceW failed for {root}", root); - return -1; - } - - clusterSize = (int)(sectorsPerCluster * bytesPerSector); - } - else - { - clusterSize = GetLinuxBlockSize(root); - if (clusterSize <= 0) - { - _logger.LogWarning("Failed to determine block size for {root}", root); - return -1; - } + _logger.LogWarning("Cannot safely decompress {file}: mount options include compression ({opts})", path, opts); + return false; } - _clusterSizes[root] = clusterSize; - _logger.LogDebug("Determined cluster/block size for {root}: {cluster} bytes", root, clusterSize); - return clusterSize; + string realPath = ToLinuxPathIfWine(path, _dalamudUtilService.IsWine); + var psi = new ProcessStartInfo + { + FileName = _dalamudUtilService.IsWine ? "/bin/bash" : "btrfs", + Arguments = _dalamudUtilService.IsWine + ? $"-c \"btrfs filesystem defragment -- '{EscapeSingle(realPath)}'\"" + : $"filesystem defragment -- \"{realPath}\"", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = "/" + }; + + using var proc = Process.Start(psi); + if (proc == null) + { + _logger.LogWarning("Failed to start btrfs defragment for {file}", path); + return false; + } + + var stdout = proc.StandardOutput.ReadToEnd(); + var stderr = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + + if (proc.ExitCode != 0) + { + _logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr); + return false; + } + + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs defragment output {file}: {out}", path, stdout.Trim()); + + _logger.LogInformation("Btrfs rewritten uncompressed: {file}", path); } catch (Exception ex) { - _logger.LogWarning(ex, "Error determining cluster size for {file}", fi.FullName); - return -1; + _logger.LogWarning(ex, "Btrfs decompress error {file}", path); } + + return true; } - public static bool UseSafeHandle(SafeHandle handle, Func action) + /// + /// Decompress an NTFS File + /// + /// Path of the compressed file + /// Decompessing state + private bool DecompressWOFFile(string path, out FileStream fs) { - bool addedRef = false; + fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + var handle = fs.SafeFileHandle; + + if (handle.IsInvalid) + { + _logger.LogWarning("Invalid handle: {file}", path); + return false; + } + + if (!DeviceIoControl(handle, FSCTL_DELETE_EXTERNAL_BACKING, + IntPtr.Zero, 0, IntPtr.Zero, 0, + out _, IntPtr.Zero)) + { + int err = Marshal.GetLastWin32Error(); + + if (err == 342) + { + _logger.LogTrace("File {file} not externally backed (already decompressed)", path); + } + else + { + _logger.LogWarning("DeviceIoControl failed for {file} with Win32 error {err}", path, err); + } + } + else + { + _logger.LogTrace("Successfully decompressed NTFS file {file}", path); + } + + return true; + } + + private static string EscapeSingle(string p) => p.Replace("'", "'\\'", StringComparison.Ordinal); + + private static string ToLinuxPathIfWine(string path, bool isWine) + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + return path; + + if (!IsProbablyWine() && !isWine) + return path; + + string realPath = path; + if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + realPath = "/" + path[3..].Replace('\\', '/'); + else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + realPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/'); + + return realPath; + } + + /// + /// Compress an WOF File + /// + /// Path of the decompressed/normal file + /// Compessing state + private bool WOFCompressFile(string path) + { + FileStream? fs = null; + int size = Marshal.SizeOf(); + IntPtr efInfoPtr = Marshal.AllocHGlobal(size); + try { - handle.DangerousAddRef(ref addedRef); - IntPtr ptr = handle.DangerousGetHandle(); - return action(ptr); + Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: false); + ulong length = (ulong)size; + + const int maxRetries = 3; + + for (int attempt = 0; attempt < maxRetries; attempt++) + { + try + { + fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + break; + } + catch (IOException) + { + if (attempt == maxRetries - 1) + { + _logger.LogWarning("File still in use after {attempts} attempts, skipping compression for {file}", maxRetries, path); + return false; + } + + int delay = 150 * (attempt + 1); + _logger.LogTrace("File in use, retrying in {delay}ms for {file}", delay, path); + Thread.Sleep(delay); + } + } + + if (fs == null) + { + _logger.LogWarning("Failed to open {file} for compression; skipping", path); + return false; + } + + var handle = fs.SafeFileHandle; + + if (handle.IsInvalid) + { + _logger.LogWarning("Invalid file handle for {file}", path); + return false; + } + + int ret = WofSetFileDataLocation(handle, WOF_PROVIDER_FILE, efInfoPtr, length); + + // 0x80070158 is WOF error whenever compression fails in an non-fatal way. + if (ret != 0 && ret != unchecked((int)0x80070158)) + { + _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); + return false; + } + + return true; + } + catch (DllNotFoundException) + { + _logger.LogTrace("WofUtil.dll not available; skipping NTFS compaction for {file}", path); + return false; + } + catch (EntryPointNotFoundException) + { + _logger.LogTrace("WOF entrypoint missing (Wine/older OS); skipping NTFS compaction for {file}", path); + return false; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error compacting file {path}", path); + return false; } finally { - if (addedRef) - handle.DangerousRelease(); + fs?.Dispose(); + + if (efInfoPtr != IntPtr.Zero) + { + Marshal.FreeHGlobal(efInfoPtr); + } } } + /// + /// Checks if an File is compacted with WOF compression + /// + /// Path of the file + /// State of the file private static bool IsWOFCompactedFile(string filePath) { try { uint buf = (uint)Marshal.SizeOf(); - int result = WofIsExternalFile(filePath, out int isExternal, out uint _, out var info, ref buf); - + int result = WofIsExternalFile(filePath, out int isExternal, out _, out var info, ref buf); if (result != 0 || isExternal == 0) return false; - return info.Algorithm == CompressionAlgorithm.XPRESS8K || info.Algorithm == CompressionAlgorithm.XPRESS4K - || info.Algorithm == CompressionAlgorithm.XPRESS16K || info.Algorithm == CompressionAlgorithm.LZX; + return info.Algorithm == (int)CompressionAlgorithm.XPRESS8K + || info.Algorithm == (int)CompressionAlgorithm.XPRESS4K + || info.Algorithm == (int)CompressionAlgorithm.XPRESS16K + || info.Algorithm == (int)CompressionAlgorithm.LZX + || info.Algorithm == (int)CompressionAlgorithm.LZNT1 + || info.Algorithm == (int)CompressionAlgorithm.NO_COMPRESSION; } - catch (DllNotFoundException) + catch { - // WofUtil.dll not available - return false; - } - catch (EntryPointNotFoundException) - { - // Running under Wine or non-NTFS systems - return false; - } - catch (Exception) - { - // Exception happened return false; } } + /// + /// Checks if an File is compacted with Btrfs compression + /// + /// Path of the file + /// State of the file private bool IsBtrfsCompressedFile(string path) { try { bool isWine = _dalamudUtilService?.IsWine ?? false; - string realPath = path; + string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; - if (isWine && IsProbablyWine()) - { - if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - { - realPath = "/" + path.Substring(3).Replace('\\', '/'); - } - else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - { - realPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), path.Substring(3).Replace('\\', '/')).Replace('\\', '/'); - } - - _logger.LogTrace("Detected Wine environment. Converted path for filefrag: {realPath}", realPath); - } - - string command = $"filefrag -v -- \"{realPath}\""; var psi = new ProcessStartInfo { FileName = isWine ? "/bin/bash" : "filefrag", - Arguments = isWine ? $"-c \"{command}\"" : $"-v -- \"{realPath}\"", + Arguments = isWine + ? $"-c \"filefrag -v '{EscapeSingle(realPath)}'\"" + : $"-v \"{realPath}\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, @@ -562,109 +547,116 @@ public sealed class FileCompactor : IDisposable return false; } - string output = proc.StandardOutput.ReadToEnd(); + string stdout = proc.StandardOutput.ReadToEnd(); string stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); - if (proc.ExitCode != 0) + if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) { - _logger.LogDebug("filefrag exited with {code} for {file}. stderr: {stderr}", - proc.ExitCode, path, stderr); - return false; + _logger.LogTrace("filefrag exited with code {code}: {stderr}", proc.ExitCode, stderr); } - bool compressed = output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); + bool compressed = stdout.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); _logger.LogTrace("Btrfs compression check for {file}: {compressed}", path, compressed); return compressed; } catch (Exception ex) { - _logger.LogDebug(ex, "Failed to detect btrfs compression for {file}", path); + _logger.LogDebug(ex, "Failed to detect Btrfs compression for {file}", path); return false; } } - private bool WOFCompressFile(string path) - { - var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo)); - Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: true); - ulong length = (ulong)Marshal.SizeOf(_efInfo); - try - { - using var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None); - var handle = fs.SafeFileHandle; - - if (handle.IsInvalid) - { - _logger.LogWarning("Invalid file handle to {file}", path); - return false; - } - - return UseSafeHandle(handle, hFile => - { - int ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length); - if (ret != 0 && ret != unchecked((int)0x80070158)) - { - _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); - return false; - } - return true; - }); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Error compacting file {path}", path); - return false; - } - finally - { - Marshal.FreeHGlobal(efInfoPtr); - } - } - + /// + /// Compress an Btrfs File + /// + /// Path of the decompressed/normal file + /// Compessing state private bool BtrfsCompressFile(string path) { try { - string realPath = path; - if (_dalamudUtilService.IsWine && IsProbablyWine()) + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; + + if (isWine && IsProbablyWine()) { - realPath = ConvertWinePathToLinux(path); - _logger.LogTrace("Detected Wine environment, remapped path: {realPath}", realPath); + if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + { + realPath = "/" + path[3..].Replace('\\', '/'); + } + else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + { + string linuxHome = Environment.GetEnvironmentVariable("HOME") ?? "/home"; + realPath = Path.Combine(linuxHome, path[3..].Replace('\\', '/')).Replace('\\', '/'); + } + + _logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", realPath); } - if (!File.Exists("/usr/bin/btrfs") && !File.Exists("/bin/btrfs")) + const int maxRetries = 3; + for (int attempt = 0; attempt < maxRetries; attempt++) { - _logger.LogWarning("Skipping Btrfs compression — btrfs binary not found"); - return false; + try + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + break; + } + catch (IOException) + { + if (attempt == maxRetries - 1) + { + _logger.LogWarning("File still in use after {attempts} attempts; skipping btrfs compression for {file}", maxRetries, path); + return false; + } + + int delay = 150 * (attempt + 1); + _logger.LogTrace("File busy, retrying in {delay}ms for {file}", delay, path); + Thread.Sleep(delay); + } } - var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -czstd -- \"{realPath}\"") + string command = $"btrfs filesystem defragment -czstd -- \"{realPath}\""; + var psi = new ProcessStartInfo { + FileName = isWine ? "/bin/bash" : "btrfs", + Arguments = isWine ? $"-c \"{command}\"" : $"filesystem defragment -czstd -- \"{realPath}\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, - CreateNoWindow = true + CreateNoWindow = true, + WorkingDirectory = "/" }; using var proc = Process.Start(psi); if (proc == null) { - _logger.LogWarning("Failed to start btrfs process for {file}", realPath); + _logger.LogWarning("Failed to start btrfs defragment for compression of {file}", path); return false; } - var stdout = proc.StandardOutput.ReadToEnd(); - var stderr = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); + string stdout = proc.StandardOutput.ReadToEnd(); + string stderr = proc.StandardError.ReadToEnd(); + + try + { + proc.WaitForExit(); + } + catch (Exception ex) + { + _logger.LogTrace(ex, "Process.WaitForExit threw under Wine for {file}", path); + } if (proc.ExitCode != 0) { - _logger.LogWarning("btrfs defrag returned {code} for {file}: {err}", proc.ExitCode, realPath, stderr); + _logger.LogWarning("btrfs defragment failed for {file}: {stderr}", path, stderr); return false; } - _logger.LogDebug("btrfs output: {out}", stdout); + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs defragment output for {file}: {stdout}", path, stdout.Trim()); + + _logger.LogInformation("Compressed btrfs file successfully: {file}", path); return true; } catch (Exception ex) @@ -674,22 +666,15 @@ public sealed class FileCompactor : IDisposable } } - private struct WOF_FILE_COMPRESSION_INFO_V1 - { - public CompressionAlgorithm Algorithm; - public ulong Flags; - } - private void EnqueueCompaction(string filePath) { if (!_pendingCompactions.TryAdd(filePath, 0)) return; - - var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); + var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) { - _logger.LogTrace("Skipping compaction enqueue for unsupported filesystem {fs} ({file})", fsType, filePath); + _logger.LogTrace("Skip enqueue (unsupported fs) {fs} {file}", fsType, filePath); _pendingCompactions.TryRemove(filePath, out _); return; } @@ -697,11 +682,7 @@ public sealed class FileCompactor : IDisposable if (!_compactionQueue.Writer.TryWrite(filePath)) { _pendingCompactions.TryRemove(filePath, out _); - _logger.LogDebug("Failed to enqueue compaction job for {file}", filePath); - } - else - { - _logger.LogTrace("Queued compaction job for {file} (fs={fs})", filePath, fsType); + _logger.LogDebug("Failed to enqueue compaction {file}", filePath); } } @@ -727,13 +708,13 @@ public sealed class FileCompactor : IDisposable if (!File.Exists(filePath)) { - _logger.LogTrace("Skipping compaction for missing file {file}", filePath); + _logger.LogTrace("Skip compact (missing) {file}", filePath); continue; } CompactFile(filePath); } - catch (OperationCanceledException) + catch (OperationCanceledException) { return; } @@ -750,9 +731,46 @@ public sealed class FileCompactor : IDisposable } catch (OperationCanceledException) { - _logger.LogDebug("Queue has been cancelled by token"); + _logger.LogDebug("Compaction queue cancelled"); } } - private static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); + [StructLayout(LayoutKind.Sequential, Pack = 1)] + private struct WOF_FILE_COMPRESSION_INFO_V1 + { + public int Algorithm; + public ulong Flags; + } + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped); + + [DllImport("kernel32.dll")] + private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh); + + [DllImport("WofUtil.dll")] + private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength); + + [DllImport("WofUtil.dll", SetLastError = true)] + private static extern int WofSetFileDataLocation(SafeFileHandle FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); + + public void Dispose() + { + _compactionQueue.Writer.TryComplete(); + _compactionCts.Cancel(); + try + { + _compactionWorker.Wait(TimeSpan.FromSeconds(5)); + } + catch + { + //ignore on catch ^^ + } + finally + { + _compactionCts.Dispose(); + } + + GC.SuppressFinalize(this); + } } diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs index 4bbfc75..af4c98b 100644 --- a/LightlessSync/Utils/FileSystemHelper.cs +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -1,4 +1,6 @@ -using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using System.Collections.Concurrent; +using System.Diagnostics; using System.Runtime.InteropServices; namespace LightlessSync.Utils @@ -20,6 +22,8 @@ namespace LightlessSync.Utils } private const string _mountPath = "/proc/mounts"; + private const int _defaultBlockSize = 4096; + private static readonly Dictionary _blockSizeCache = new(StringComparer.OrdinalIgnoreCase); private static readonly ConcurrentDictionary _filesystemTypeCache = new(StringComparer.OrdinalIgnoreCase); public static FilesystemType GetFilesystemType(string filePath, bool isWine = false) @@ -27,6 +31,7 @@ namespace LightlessSync.Utils try { string rootPath; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (!IsProbablyWine() || !isWine)) { var info = new FileInfo(filePath); @@ -49,6 +54,7 @@ namespace LightlessSync.Utils { var root = new DriveInfo(rootPath); var format = root.DriveFormat?.ToUpperInvariant() ?? string.Empty; + detected = format switch { "NTFS" => FilesystemType.NTFS, @@ -62,10 +68,28 @@ namespace LightlessSync.Utils detected = GetLinuxFilesystemType(filePath); } + if (isWine || IsProbablyWine()) + { + switch (detected) + { + case FilesystemType.NTFS: + case FilesystemType.Unknown: + { + var linuxDetected = GetLinuxFilesystemType(filePath); + if (linuxDetected != FilesystemType.Unknown) + { + detected = linuxDetected; + } + + break; + } + } + } + _filesystemTypeCache[rootPath] = detected; return detected; } - catch (Exception ex) + catch { return FilesystemType.Unknown; } @@ -175,7 +199,84 @@ namespace LightlessSync.Utils } } + public static int GetBlockSizeForPath(string path, ILogger? logger = null, bool isWine = false) + { + try + { + if (string.IsNullOrWhiteSpace(path)) + return _defaultBlockSize; + + var fi = new FileInfo(path); + if (!fi.Exists) + return _defaultBlockSize; + + var root = fi.Directory?.Root.FullName.ToLowerInvariant() ?? "/"; + if (_blockSizeCache.TryGetValue(root, out int cached)) + return cached; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !isWine) + { + int result = GetDiskFreeSpaceW(root, + out uint sectorsPerCluster, + out uint bytesPerSector, + out _, + out _); + + if (result == 0) + { + logger?.LogWarning("Failed to determine block size for {root}", root); + return _defaultBlockSize; + } + + int clusterSize = (int)(sectorsPerCluster * bytesPerSector); + _blockSizeCache[root] = clusterSize; + logger?.LogTrace("NTFS cluster size for {root}: {cluster}", root, clusterSize); + return clusterSize; + } + + string realPath = fi.FullName; + if (isWine && realPath.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + { + realPath = "/" + realPath.Substring(3).Replace('\\', '/'); + } + + var psi = new ProcessStartInfo + { + FileName = "/bin/bash", + Arguments = $"-c \"stat -f -c %s '{realPath.Replace("'", "'\\''")}'\"", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = "/" + }; + + using var proc = Process.Start(psi); + string stdout = proc?.StandardOutput.ReadToEnd().Trim() ?? ""; + proc?.WaitForExit(); + + if (int.TryParse(stdout, out int blockSize) && blockSize > 0) + { + _blockSizeCache[root] = blockSize; + logger?.LogTrace("Filesystem block size via stat for {root}: {block}", root, blockSize); + return blockSize; + } + + logger?.LogTrace("stat did not return valid block size for {file}, output: {out}", fi.FullName, stdout); + _blockSizeCache[root] = _defaultBlockSize; + return _defaultBlockSize; + } + catch (Exception ex) + { + logger?.LogTrace(ex, "Error determining block size for {path}", path); + return _defaultBlockSize; + } + } + + [DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)] + private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters, out uint lpTotalNumberOfClusters); + //Extra check on - private static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); + public static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); } } From d4dca455ba8d8abe9323ef4081f9938e4f7f8e98 Mon Sep 17 00:00:00 2001 From: cake Date: Mon, 3 Nov 2025 18:54:35 +0100 Subject: [PATCH 12/13] Clean-up, added extra checks on linux in cache monitor, documentation added --- LightlessSync/FileCache/CacheMonitor.cs | 91 ++++-- LightlessSync/FileCache/FileCompactor.cs | 372 ++++++++++++----------- 2 files changed, 260 insertions(+), 203 deletions(-) diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 85fb30e..486e11e 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -403,57 +403,94 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase public void RecalculateFileCacheSize(CancellationToken token) { - if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder)) + if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || + !Directory.Exists(_configService.Current.CacheFolder)) { FileCacheSize = 0; return; } - FileCacheSize = -1; - - var drive = DriveInfo.GetDrives().FirstOrDefault(d => _configService.Current.CacheFolder.StartsWith(d.Name, StringComparison.Ordinal)); - if (drive == null) - { - return; - } + FileCacheSize = -1; + bool isWine = _dalamudUtil?.IsWine ?? false; try { - FileCacheDriveFree = drive.AvailableFreeSpace; + var drive = DriveInfo.GetDrives() + .FirstOrDefault(d => _configService.Current.CacheFolder + .StartsWith(d.Name, StringComparison.OrdinalIgnoreCase)); + + if (drive != null) + FileCacheDriveFree = drive.AvailableFreeSpace; } catch (Exception ex) { - Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder); + Logger.LogWarning(ex, "Could not determine drive size for storage folder {folder}", _configService.Current.CacheFolder); } - var files = Directory.EnumerateFiles(_configService.Current.CacheFolder).Select(f => new FileInfo(f)) - .OrderBy(f => f.LastAccessTime).ToList(); - FileCacheSize = files - .Sum(f => - { - token.ThrowIfCancellationRequested(); + var files = Directory.EnumerateFiles(_configService.Current.CacheFolder) + .Select(f => new FileInfo(f)) + .OrderBy(f => f.LastAccessTime) + .ToList(); - try + long totalSize = 0; + + foreach (var f in files) + { + token.ThrowIfCancellationRequested(); + + try + { + long size = 0; + + if (!isWine) { - return _fileCompactor.GetFileSizeOnDisk(f); + try + { + size = _fileCompactor.GetFileSizeOnDisk(f); + } + catch (Exception ex) + { + Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName); + size = f.Length; + } } - catch + else { - return 0; + size = f.Length; } - }); + + totalSize += size; + } + catch (Exception ex) + { + Logger.LogTrace(ex, "Error getting size for {file}", f.FullName); + } + } + + FileCacheSize = totalSize; var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d); - - if (FileCacheSize < maxCacheInBytes) return; + if (FileCacheSize < maxCacheInBytes) + return; var maxCacheBuffer = maxCacheInBytes * 0.05d; - while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer) + + while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer && files.Count > 0) { var oldestFile = files[0]; - FileCacheSize -= _fileCompactor.GetFileSizeOnDisk(oldestFile); - File.Delete(oldestFile.FullName); - files.Remove(oldestFile); + + try + { + long fileSize = oldestFile.Length; + File.Delete(oldestFile.FullName); + FileCacheSize -= fileSize; + } + catch (Exception ex) + { + Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullName); + } + + files.RemoveAt(0); } } diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index a917b59..b9c2118 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Diagnostics; +using System.IO; using System.Runtime.InteropServices; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; @@ -14,6 +15,7 @@ public sealed class FileCompactor : IDisposable { public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U; public const ulong WOF_PROVIDER_FILE = 2UL; + public const int _maxRetries = 3; private readonly ConcurrentDictionary _pendingCompactions; private readonly ILogger _logger; @@ -29,6 +31,14 @@ public sealed class FileCompactor : IDisposable Algorithm = (int)CompressionAlgorithm.XPRESS8K, Flags = 0 }; + + [StructLayout(LayoutKind.Sequential, Pack = 1)] + private struct WOF_FILE_COMPRESSION_INFO_V1 + { + public int Algorithm; + public ulong Flags; + } + private enum CompressionAlgorithm { NO_COMPRESSION = -2, @@ -58,6 +68,10 @@ public sealed class FileCompactor : IDisposable public bool MassCompactRunning { get; private set; } public string Progress { get; private set; } = string.Empty; + /// + /// Compact the storage of the Cache Folder + /// + /// Used to check if files needs to be compressed public void CompactStorage(bool compress) { MassCompactRunning = true; @@ -74,6 +88,7 @@ public sealed class FileCompactor : IDisposable try { + // Compress or decompress files if (compress) CompactFile(file); else @@ -135,25 +150,19 @@ public sealed class FileCompactor : IDisposable { try { - var realPath = fileInfo.FullName.Replace("\"", "\\\"", StringComparison.Ordinal); - var psi = new ProcessStartInfo("stat", $"-c %b \"{realPath}\"") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(fileInfo.FullName, isWine) : fileInfo.FullName; - using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat"); - var outp = proc.StandardOutput.ReadToEnd(); - var err = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); + var fileName = "stat"; + var arguments = $"-c %b \"{realPath}\""; - if (proc.ExitCode != 0) - throw new InvalidOperationException($"stat failed: {err}"); + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, arguments, out Process? proc, out string stdout); - if (!long.TryParse(outp.Trim(), out var blocks)) - throw new InvalidOperationException($"invalid stat output: {outp}"); + if (!processControl && !success) + throw new InvalidOperationException($"stat failed: {proc}"); + + if (!long.TryParse(stdout.Trim(), out var blocks)) + throw new InvalidOperationException($"invalid stat output: {stdout}"); // st_blocks are always 512-byte on Linux enviroment. return blocks * 512L; @@ -287,57 +296,57 @@ public sealed class FileCompactor : IDisposable /// Decompessing state private bool DecompressBtrfsFile(string path) { + var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + try { - var opts = GetMountOptionsForPath(path); - if (opts.Contains("compress", StringComparison.OrdinalIgnoreCase)) + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; + + var mountOptions = GetMountOptionsForPath(realPath); + if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) { - _logger.LogWarning("Cannot safely decompress {file}: mount options include compression ({opts})", path, opts); + _logger.LogWarning( + "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " + + "Remount with 'compress=no' before running decompression.", + realPath, mountOptions); return false; } - string realPath = ToLinuxPathIfWine(path, _dalamudUtilService.IsWine); - var psi = new ProcessStartInfo + if (!IsBtrfsCompressedFile(realPath)) { - FileName = _dalamudUtilService.IsWine ? "/bin/bash" : "btrfs", - Arguments = _dalamudUtilService.IsWine - ? $"-c \"btrfs filesystem defragment -- '{EscapeSingle(realPath)}'\"" - : $"filesystem defragment -- \"{realPath}\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; - - using var proc = Process.Start(psi); - if (proc == null) - { - _logger.LogWarning("Failed to start btrfs defragment for {file}", path); - return false; + _logger.LogTrace("File {file} is not compressed, skipping decompression.", realPath); + return true; } - var stdout = proc.StandardOutput.ReadToEnd(); - var stderr = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); + (bool flowControl, bool value) = FileStreamOpening(realPath, ref fs); - if (proc.ExitCode != 0) + if (!flowControl) { - _logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr); - return false; + return value; + } + + string fileName = isWine ? "/bin/bash" : "btrfs"; + string command = isWine ? $"-c \"filesystem defragment -- \"{realPath}\"\"" : $"filesystem defragment -- \"{realPath}\""; + + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); + if (!processControl && !success) + { + return value; } if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogTrace("btrfs defragment output {file}: {out}", path, stdout.Trim()); + _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); - _logger.LogInformation("Btrfs rewritten uncompressed: {file}", path); + _logger.LogInformation("Decompressed btrfs file successfully: {file}", realPath); + + return true; } catch (Exception ex) { - _logger.LogWarning(ex, "Btrfs decompress error {file}", path); + _logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path); + return false; } - - return true; } /// @@ -379,23 +388,25 @@ public sealed class FileCompactor : IDisposable return true; } - private static string EscapeSingle(string p) => p.Replace("'", "'\\'", StringComparison.Ordinal); - - private static string ToLinuxPathIfWine(string path, bool isWine) + /// + /// Converts to Linux Path if its using Wine (diferent pathing system in Wine) + /// + /// Path that has to be converted + /// Extra check if using the wine enviroment + /// Converted path to be used in Linux + private string ToLinuxPathIfWine(string path, bool isWine) { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - return path; - if (!IsProbablyWine() && !isWine) return path; - string realPath = path; + string linuxPath = path; if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - realPath = "/" + path[3..].Replace('\\', '/'); + linuxPath = "/" + path[3..].Replace('\\', '/'); else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - realPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/'); + linuxPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/'); - return realPath; + _logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", linuxPath); + return linuxPath; } /// @@ -414,27 +425,11 @@ public sealed class FileCompactor : IDisposable Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: false); ulong length = (ulong)size; - const int maxRetries = 3; + (bool flowControl, bool value) = FileStreamOpening(path, ref fs); - for (int attempt = 0; attempt < maxRetries; attempt++) + if (!flowControl) { - try - { - fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); - break; - } - catch (IOException) - { - if (attempt == maxRetries - 1) - { - _logger.LogWarning("File still in use after {attempts} attempts, skipping compression for {file}", maxRetries, path); - return false; - } - - int delay = 150 * (attempt + 1); - _logger.LogTrace("File in use, retrying in {delay}ms for {file}", delay, path); - Thread.Sleep(delay); - } + return value; } if (fs == null) @@ -462,14 +457,14 @@ public sealed class FileCompactor : IDisposable return true; } - catch (DllNotFoundException) + catch (DllNotFoundException ex) { - _logger.LogTrace("WofUtil.dll not available; skipping NTFS compaction for {file}", path); + _logger.LogTrace(ex, "WofUtil.dll not available, this DLL is needed for compression; skipping NTFS compaction for {file}", path); return false; } - catch (EntryPointNotFoundException) + catch (EntryPointNotFoundException ex) { - _logger.LogTrace("WOF entrypoint missing (Wine/older OS); skipping NTFS compaction for {file}", path); + _logger.LogTrace(ex, "WOF entrypoint missing (Wine/older OS); skipping NTFS compaction for {file}", path); return false; } catch (Exception ex) @@ -527,37 +522,24 @@ public sealed class FileCompactor : IDisposable bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; - var psi = new ProcessStartInfo - { - FileName = isWine ? "/bin/bash" : "filefrag", - Arguments = isWine - ? $"-c \"filefrag -v '{EscapeSingle(realPath)}'\"" - : $"-v \"{realPath}\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; + var fi = new FileInfo(realPath); - using var proc = Process.Start(psi); - if (proc == null) + if (fi == null) { - _logger.LogWarning("Failed to start filefrag for {file}", path); + _logger.LogWarning("Failed to open {file} for checking on compression; skipping", realPath); return false; } - string stdout = proc.StandardOutput.ReadToEnd(); - string stderr = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); - - if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) + string fileName = isWine ? "/bin/bash" : "filefrag"; + string command = isWine ? $"-c \"filefrag -v '{EscapeSingle(realPath)}'\"" : $"-v \"{realPath}\""; + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); + if (!processControl && !success) { - _logger.LogTrace("filefrag exited with code {code}: {stderr}", proc.ExitCode, stderr); + return success; } bool compressed = stdout.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); - _logger.LogTrace("Btrfs compression check for {file}: {compressed}", path, compressed); + _logger.LogTrace("Btrfs compression check for {file}: {compressed}", realPath, compressed); return compressed; } catch (Exception ex) @@ -574,89 +556,56 @@ public sealed class FileCompactor : IDisposable /// Compessing state private bool BtrfsCompressFile(string path) { + FileStream? fs = null; + try { bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; - if (isWine && IsProbablyWine()) + var fi = new FileInfo(realPath); + + if (fi == null) { - if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - { - realPath = "/" + path[3..].Replace('\\', '/'); - } - else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - { - string linuxHome = Environment.GetEnvironmentVariable("HOME") ?? "/home"; - realPath = Path.Combine(linuxHome, path[3..].Replace('\\', '/')).Replace('\\', '/'); - } - - _logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", realPath); - } - - const int maxRetries = 3; - for (int attempt = 0; attempt < maxRetries; attempt++) - { - try - { - using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); - break; - } - catch (IOException) - { - if (attempt == maxRetries - 1) - { - _logger.LogWarning("File still in use after {attempts} attempts; skipping btrfs compression for {file}", maxRetries, path); - return false; - } - - int delay = 150 * (attempt + 1); - _logger.LogTrace("File busy, retrying in {delay}ms for {file}", delay, path); - Thread.Sleep(delay); - } - } - - string command = $"btrfs filesystem defragment -czstd -- \"{realPath}\""; - var psi = new ProcessStartInfo - { - FileName = isWine ? "/bin/bash" : "btrfs", - Arguments = isWine ? $"-c \"{command}\"" : $"filesystem defragment -czstd -- \"{realPath}\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; - - using var proc = Process.Start(psi); - if (proc == null) - { - _logger.LogWarning("Failed to start btrfs defragment for compression of {file}", path); + _logger.LogWarning("Failed to open {file} for compression; skipping", realPath); return false; } - string stdout = proc.StandardOutput.ReadToEnd(); - string stderr = proc.StandardError.ReadToEnd(); - - try - { - proc.WaitForExit(); - } - catch (Exception ex) + //Skipping small files to make compression a bit faster, its not that effective on small files. + int blockSize = GetBlockSizeForPath(realPath, _logger, isWine); + if (fi.Length < Math.Max(blockSize * 2, 128 * 1024)) { - _logger.LogTrace(ex, "Process.WaitForExit threw under Wine for {file}", path); + _logger.LogTrace("Skipping Btrfs compression for small file {file} ({size} bytes)", realPath, fi.Length); + return true; } - if (proc.ExitCode != 0) + if (IsBtrfsCompressedFile(realPath)) { - _logger.LogWarning("btrfs defragment failed for {file}: {stderr}", path, stderr); - return false; + _logger.LogTrace("File {file} already compressed (Btrfs), skipping file", realPath); + return true; + } + + (bool flowControl, bool value) = FileStreamOpening(realPath, ref fs); + + if (!flowControl) + { + return value; + } + + string fileName = isWine ? "/bin/bash" : "btrfs"; + string command = isWine ? $"-c \"btrfs filesystem defragment -czstd:1 -- \"{realPath}\"\"" : $"btrfs filesystem defragment -czstd:1 -- \"{realPath}\""; + + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); + if (!processControl && !success) + { + return value; } if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogTrace("btrfs defragment output for {file}: {stdout}", path, stdout.Trim()); + _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); + + _logger.LogInformation("Compressed btrfs file successfully: {file}", realPath); - _logger.LogInformation("Compressed btrfs file successfully: {file}", path); return true; } catch (Exception ex) @@ -666,6 +615,84 @@ public sealed class FileCompactor : IDisposable } } + + /// + /// Trying opening file stream in certain amount of tries. + /// + /// Path where the file is located + /// Filestream used for the function + /// State of the filestream opening + private (bool flowControl, bool value) FileStreamOpening(string path, ref FileStream? fs) + { + for (int attempt = 0; attempt < _maxRetries; attempt++) + { + try + { + fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + break; + } + catch (IOException) + { + if (attempt == _maxRetries - 1) + { + _logger.LogWarning("File still in use after {attempts} attempts, skipping compression for {file}", _maxRetries, path); + return (flowControl: false, value: false); + } + + int delay = 150 * (attempt + 1); + _logger.LogTrace("File in use, retrying in {delay}ms for {file}", delay, path); + Thread.Sleep(delay); + } + } + + return (flowControl: true, value: default); + } + + /// + /// Starts an process with given Filename and Arguments + /// + /// Path you want to use for the process (Compression is using these) + /// File of the command + /// Arguments used for the command + /// Returns process of the given command + /// Returns output of the given command + /// Returns if the process been done succesfully or not + private (bool processControl, bool success) StartProcessInfo(string path, string fileName, string arguments, out Process? proc, out string stdout) + { + var psi = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = "/" + }; + proc = Process.Start(psi); + + if (proc == null) + { + _logger.LogWarning("Failed to start {arguments} for {file}", arguments, path); + stdout = string.Empty; + return (processControl: false, success: false); + } + + stdout = proc.StandardOutput.ReadToEnd(); + string stderr = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + + if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) + { + _logger.LogTrace("{arguments} exited with code {code}: {stderr}", arguments, proc.ExitCode, stderr); + return (processControl: false, success: false); + } + + return (processControl: true, success: default); + } + + private static string EscapeSingle(string p) => p.Replace("'", "'\\'", StringComparison.Ordinal); + private void EnqueueCompaction(string filePath) { if (!_pendingCompactions.TryAdd(filePath, 0)) @@ -735,13 +762,6 @@ public sealed class FileCompactor : IDisposable } } - [StructLayout(LayoutKind.Sequential, Pack = 1)] - private struct WOF_FILE_COMPRESSION_INFO_V1 - { - public int Algorithm; - public ulong Flags; - } - [DllImport("kernel32.dll", SetLastError = true)] private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped); From cfc9f60176ff29bf237427572a7000154fafc72b Mon Sep 17 00:00:00 2001 From: cake Date: Mon, 3 Nov 2025 19:27:47 +0100 Subject: [PATCH 13/13] Added safe checks on enqueue. --- LightlessSync/FileCache/FileCompactor.cs | 152 ++++++++++++++++------- 1 file changed, 107 insertions(+), 45 deletions(-) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index b9c2118..4722b1f 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -4,7 +4,6 @@ using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Diagnostics; -using System.IO; using System.Runtime.InteropServices; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; @@ -140,42 +139,82 @@ public sealed class FileCompactor : IDisposable if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { - var blockSize = GetBlockSizeForPath(fileInfo.FullName, _logger, _dalamudUtilService.IsWine); - var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); - var size = (long)hosize << 32 | losize; - return ((size + blockSize - 1) / blockSize) * blockSize; + (bool flowControl, long value) = GetFileSizeNTFS(fileInfo); + if (!flowControl) + { + return value; + } } if (fsType == FilesystemType.Btrfs) { - try + (bool flowControl, long value) = GetFileSizeBtrfs(fileInfo); + if (!flowControl) { - bool isWine = _dalamudUtilService?.IsWine ?? false; - string realPath = isWine ? ToLinuxPathIfWine(fileInfo.FullName, isWine) : fileInfo.FullName; - - var fileName = "stat"; - var arguments = $"-c %b \"{realPath}\""; - - (bool processControl, bool success) = StartProcessInfo(realPath, fileName, arguments, out Process? proc, out string stdout); - - if (!processControl && !success) - throw new InvalidOperationException($"stat failed: {proc}"); - - if (!long.TryParse(stdout.Trim(), out var blocks)) - throw new InvalidOperationException($"invalid stat output: {stdout}"); - - // st_blocks are always 512-byte on Linux enviroment. - return blocks * 512L; - } - catch (Exception ex) - { - _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); + return value; } } return fileInfo.Length; } + /// + /// Get File Size in an Btrfs file system (Linux/Wine). + /// + /// File that you want the size from. + /// Succesful check and value of the filesize. + /// Fails on the Process in StartProcessInfo + private (bool flowControl, long value) GetFileSizeBtrfs(FileInfo fileInfo) + { + try + { + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(fileInfo.FullName, isWine) : fileInfo.FullName; + + var fileName = "stat"; + var arguments = $"-c %b \"{realPath}\""; + + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, arguments, out Process? proc, out string stdout); + + if (!processControl && !success) + throw new InvalidOperationException($"stat failed: {proc}"); + + if (!long.TryParse(stdout.Trim(), out var blocks)) + throw new InvalidOperationException($"invalid stat output: {stdout}"); + + // st_blocks are always 512-byte on Linux enviroment. + return (flowControl: false, value: blocks * 512L); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); + } + + return (flowControl: true, value: default); + } + + /// + /// Get File Size in an NTFS file system (Windows). + /// + /// File that you want the size from. + /// Succesful check and value of the filesize. + private (bool flowControl, long value) GetFileSizeNTFS(FileInfo fileInfo) + { + try + { + var blockSize = GetBlockSizeForPath(fileInfo.FullName, _logger, _dalamudUtilService.IsWine); + var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); + var size = (long)hosize << 32 | losize; + return (flowControl: false, value: ((size + blockSize - 1) / blockSize) * blockSize); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); + } + + return (flowControl: true, value: default); + } + /// /// Compressing the given path with BTRFS or NTFS file system. /// @@ -293,7 +332,7 @@ public sealed class FileCompactor : IDisposable /// Decompress an BTRFS File /// /// Path of the compressed file - /// Decompessing state + /// Decompressing state private bool DecompressBtrfsFile(string path) { var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); @@ -353,7 +392,7 @@ public sealed class FileCompactor : IDisposable /// Decompress an NTFS File /// /// Path of the compressed file - /// Decompessing state + /// Decompressing state private bool DecompressWOFFile(string path, out FileStream fs) { fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); @@ -571,14 +610,6 @@ public sealed class FileCompactor : IDisposable return false; } - //Skipping small files to make compression a bit faster, its not that effective on small files. - int blockSize = GetBlockSizeForPath(realPath, _logger, isWine); - if (fi.Length < Math.Max(blockSize * 2, 128 * 1024)) - { - _logger.LogTrace("Skipping Btrfs compression for small file {file} ({size} bytes)", realPath, fi.Length); - return true; - } - if (IsBtrfsCompressedFile(realPath)) { _logger.LogTrace("File {file} already compressed (Btrfs), skipping file", realPath); @@ -695,21 +726,52 @@ public sealed class FileCompactor : IDisposable private void EnqueueCompaction(string filePath) { + // Safe-checks + if (string.IsNullOrWhiteSpace(filePath)) + return; + + if (!_lightlessConfigService.Current.UseCompactor) + return; + + if (!File.Exists(filePath)) + return; + if (!_pendingCompactions.TryAdd(filePath, 0)) return; - var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); - if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) + bool enqueued = false; + try { - _logger.LogTrace("Skip enqueue (unsupported fs) {fs} {file}", fsType, filePath); - _pendingCompactions.TryRemove(filePath, out _); - return; - } + bool isWine = _dalamudUtilService?.IsWine ?? false; + var fsType = GetFilesystemType(filePath, isWine); - if (!_compactionQueue.Writer.TryWrite(filePath)) + // If under Wine, we should skip NTFS because its not Windows but might return NTFS. + if (fsType == FilesystemType.NTFS && isWine) + { + _logger.LogTrace("Skip enqueue (NTFS under Wine) {file}", filePath); + return; + } + + // Unknown file system should be skipped. + if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) + { + _logger.LogTrace("Skip enqueue (unsupported fs) {fs} {file}", fsType, filePath); + return; + } + + if (!_compactionQueue.Writer.TryWrite(filePath)) + { + _logger.LogTrace("Skip enqueue: compaction channel is closed {file}", filePath); + return; + } + + enqueued = true; + _logger.LogTrace("Queued compaction for {file} (fs={fs})", filePath, fsType); + } + finally { - _pendingCompactions.TryRemove(filePath, out _); - _logger.LogDebug("Failed to enqueue compaction {file}", filePath); + if (!enqueued) + _pendingCompactions.TryRemove(filePath, out _); } }