From 1862689b1b37e355078b5420167b6338194f112e Mon Sep 17 00:00:00 2001 From: cake Date: Tue, 11 Nov 2025 17:09:50 +0100 Subject: [PATCH] Changed some commands in file getting, redone compression check commands and turned off btrfs compactor for 1.12.4 --- LightlessSync/FileCache/FileCompactor.cs | 144 +++++++++--------- .../BatchFileFragService.cs | 91 ++++------- LightlessSync/UI/SettingsUi.cs | 10 +- 3 files changed, 106 insertions(+), 139 deletions(-) rename LightlessSync/Services/{Compression => Compactor}/BatchFileFragService.cs (71%) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 5e0b4a9..3edf96a 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,12 +1,11 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; -using LightlessSync.Services.Compression; +using LightlessSync.Services.Compactor; using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.InteropServices; -using System.Threading; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; @@ -28,6 +27,8 @@ public sealed class FileCompactor : IDisposable private readonly List _workers = []; private readonly SemaphoreSlim _globalGate; + + //Limit btrfs gate on half of threads given to compactor. private static readonly SemaphoreSlim _btrfsGate = new(4, 4); private readonly BatchFilefragService _fragBatch; @@ -83,8 +84,11 @@ public sealed class FileCompactor : IDisposable _fragBatch = new BatchFilefragService( useShell: _dalamudUtilService.IsWine, log: _logger, - batchSize: 128, - flushMs: 25); + batchSize: 64, + flushMs: 25, + runDirect: RunProcessDirect, + runShell: RunProcessShell + ); _logger.LogInformation("FileCompactor started with {workers} workers", workerCount); } @@ -196,21 +200,15 @@ public sealed class FileCompactor : IDisposable bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); var (_, linuxPath) = ResolvePathsForBtrfs(fileInfo.FullName); - var (ok1, out1, err1, code1) = + var (ok, output, err, code) = isWindowsProc - ? RunProcessShell($"stat -c %b -- {QuoteSingle(linuxPath)}", null, 10000) - : RunProcessDirect("stat", ["-c", "%b", "--", linuxPath], null, 10000); + ? RunProcessShell($"stat -c='%b' {QuoteSingle(linuxPath)}", workingDir: null, 10000) + : RunProcessDirect("stat", ["-c='%b'", linuxPath], workingDir: null, 10000); - if (ok1 && long.TryParse(out1.Trim(), out long blocks)) + if (ok && long.TryParse(output.Trim(), out long blocks)) return (false, blocks * 512L); // st_blocks are always 512B units - var (ok2, out2, err2, code2) = RunProcessShell($"du -B1 -- {QuoteSingle(linuxPath)} | cut -f1", workingDir: null, 10000); // use shell for the pipe - - if (ok2 && long.TryParse(out2.Trim(), out long bytes)) - return (false, bytes); - - _logger.LogDebug("Btrfs size probe failed for {linux} (stat {code1}, du {code2}). Falling back to Length.", - linuxPath, code1, code2); + _logger.LogDebug("Btrfs size probe failed for {linux} (stat {code}, err {err}). Falling back to Length.", linuxPath, code, err); return (false, fileInfo.Length); } catch (Exception ex) @@ -360,7 +358,7 @@ public sealed class FileCompactor : IDisposable } /// - /// Decompress an BTRFS File + /// Decompress an BTRFS File on Wine/Linux /// /// Path of the compressed file /// Decompressing state @@ -370,44 +368,55 @@ public sealed class FileCompactor : IDisposable { try { - var (winPath, linuxPath) = ResolvePathsForBtrfs(path); - bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + bool isWine = _dalamudUtilService?.IsWine ?? false; + string linuxPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; var opts = GetMountOptionsForPath(linuxPath); - if (opts.Contains("compress", StringComparison.OrdinalIgnoreCase)) + bool hasCompress = opts.Contains("compress", StringComparison.OrdinalIgnoreCase); + bool hasCompressForce = opts.Contains("compress-force", StringComparison.OrdinalIgnoreCase); + + if (hasCompressForce) { - _logger.LogWarning( - "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). Remount with 'compress=no'.", - linuxPath, opts); + _logger.LogWarning("Cannot safely decompress {file}: mount options contains compress-force ({opts}).", linuxPath, opts); return false; } + if (hasCompress) + { + var setCmd = $"btrfs property set -- {QuoteDouble(linuxPath)} compression none"; + var (okSet, _, errSet, codeSet) = isWine + ? RunProcessShell(setCmd) + : RunProcessDirect("btrfs", ["property", "set", "--", linuxPath, "compression", "none"]); + + if (!okSet) + { + _logger.LogWarning("Failed to set 'compression none' on {file}, please check drive options (exit code is: {code}): {err}", linuxPath, codeSet, errSet); + return false; + } + _logger.LogTrace("Set per-file 'compression none' on {file}", linuxPath); + } + if (!IsBtrfsCompressedFile(linuxPath)) { - _logger.LogTrace("Btrfs: not compressed, skip {file}", linuxPath); + _logger.LogTrace("{file} is not compressed, skipping decompression completely", linuxPath); return true; } - if (!ProbeFileReadableForBtrfs(winPath, linuxPath)) - return false; - - // Rewrite file uncompressed - (bool ok, string stdout, string stderr, int code) = - isWindowsProc - ? RunProcessShell($"btrfs filesystem defragment -- {QuoteSingle(linuxPath)}") - : RunProcessDirect("btrfs", ["filesystem", "defragment", "--", linuxPath]); + var (ok, stdout, stderr, code) = isWine + ? RunProcessShell($"btrfs filesystem defragment -- {QuoteDouble(linuxPath)}") + : RunProcessDirect("btrfs", ["filesystem", "defragment", "--", linuxPath]); if (!ok) { - _logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit {code}): {stderr}", + _logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit code is: {code}): {stderr}", linuxPath, code, stderr); return false; } if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogTrace("btrfs (decompress) output {file}: {out}", linuxPath, stdout.Trim()); + _logger.LogTrace("btrfs defragment output for {file}: {out}", linuxPath, stdout.Trim()); - _logger.LogInformation("Decompressed (rewritten) Btrfs file: {file}", linuxPath); + _logger.LogInformation("Decompressed (rewritten uncompressed) Btrfs file: {file}", linuxPath); return true; } catch (Exception ex) @@ -467,19 +476,19 @@ public sealed class FileCompactor : IDisposable /// Path that has to be converted /// Extra check if using the wine enviroment /// Converted path to be used in Linux - private static string ToLinuxPathIfWine(string path, bool isWine) + private string ToLinuxPathIfWine(string path, bool isWine, bool preferShell = true) { if (!isWine || !IsProbablyWine()) return path; if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - return "/" + path[3..].Replace('\\', '/'); + return ("/" + path[3..].Replace('\\', '/')).Replace("//", "/", StringComparison.Ordinal); if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) { + const string usersPrefix = "C:\\Users\\"; var p = path.Replace('/', '\\'); - const string usersPrefix = "C:\\Users\\"; if (p.StartsWith(usersPrefix, StringComparison.OrdinalIgnoreCase)) { int afterUsers = usersPrefix.Length; @@ -493,48 +502,38 @@ public sealed class FileCompactor : IDisposable var linuxUser = Environment.GetEnvironmentVariable("USER") ?? Environment.UserName; home = "/home/" + linuxUser; } - // Join as Unix path - return (home.TrimEnd('/') + "/" + rel).Replace("//", "/", StringComparison.Ordinal); + return (home!.TrimEnd('/') + "/" + rel).Replace("//", "/", StringComparison.Ordinal); } } try { - var inner = "winepath -u " + "'" + path.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; - var psi = new ProcessStartInfo + (bool ok, string stdout, string stderr, int code) = preferShell + ? RunProcessShell($"winepath -u {QuoteSingle(path)}", timeoutMs: 5000, workingDir: "/") + : RunProcessDirect("winepath", ["-u", path], workingDir: "/", timeoutMs: 5000); + + if (ok) { - FileName = "/bin/bash", - Arguments = "-c " + "\"" + inner.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal) + "\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; - using var proc = Process.Start(psi); - var outp = proc?.StandardOutput.ReadToEnd().Trim(); - try - { - proc?.WaitForExit(); - } - catch - { - /* Wine can throw here; ignore */ + var outp = (stdout ?? "").Trim(); + if (!string.IsNullOrEmpty(outp) && outp.StartsWith('/')) + return outp.Replace("//", "/", StringComparison.Ordinal); + } + else + { + _logger.LogTrace("winepath failed for {path} (exit {code}): {err}", path, code, stderr); } - if (!string.IsNullOrEmpty(outp) && outp.StartsWith("/", StringComparison.Ordinal)) - return outp; } - catch + catch (Exception ex) { - /* ignore and fall through the floor! */ + _logger.LogTrace(ex, "winepath invocation failed for {path}", path); } } - - return path.Replace('\\', '/'); + + return path.Replace('\\', '/').Replace("//", "/", StringComparison.Ordinal); } /// - /// Compress an WOF File + /// Compress an File using the WOF methods (NTFS) /// /// Path of the decompressed/normal file /// Compessing state @@ -585,7 +584,7 @@ public sealed class FileCompactor : IDisposable } /// - /// Checks if an File is compacted with WOF compression + /// Checks if an File is compacted with WOF compression (NTFS) /// /// Path of the file /// State of the file @@ -612,7 +611,7 @@ public sealed class FileCompactor : IDisposable } /// - /// Checks if an File is compacted any WOF compression with an WOF backing + /// Checks if an File is compacted any WOF compression with an WOF backing (NTFS) /// /// Path of the file /// State of the file, if its an external (no backing) and which algorithm if detected @@ -821,7 +820,8 @@ public sealed class FileCompactor : IDisposable CreateNoWindow = true }; if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir; - // Use a Login shell so PATH includes /usr/sbin etc. AKA -lc + + // Use a Login shell so PATH includes /usr/sbin etc. AKA -lc for login shell psi.ArgumentList.Add("-lc"); psi.ArgumentList.Add(QuoteDouble(command)); EnsureUnixPathEnv(psi); @@ -1011,7 +1011,7 @@ public sealed class FileCompactor : IDisposable } /// - /// Ensures the Unix pathing to be included into the process + /// Ensures the Unix pathing to be included into the process start /// /// Process private static void EnsureUnixPathEnv(ProcessStartInfo psi) @@ -1034,10 +1034,8 @@ public sealed class FileCompactor : IDisposable if (!isWindowsProc) return (path, path); - // Prefer winepath -u; fall back to your existing mapper - var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(path)}", null, 5000); - var linux = (ok && !string.IsNullOrWhiteSpace(outp)) ? outp.Trim() - : ToLinuxPathIfWine(path, isWine: true); + var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(path)}", workingDir: null, 5000); + var linux = (ok && !string.IsNullOrWhiteSpace(outp)) ? outp.Trim() : ToLinuxPathIfWine(path, isWine: true); return (path, linux); } diff --git a/LightlessSync/Services/Compression/BatchFileFragService.cs b/LightlessSync/Services/Compactor/BatchFileFragService.cs similarity index 71% rename from LightlessSync/Services/Compression/BatchFileFragService.cs rename to LightlessSync/Services/Compactor/BatchFileFragService.cs index 16c05e1..b31919e 100644 --- a/LightlessSync/Services/Compression/BatchFileFragService.cs +++ b/LightlessSync/Services/Compactor/BatchFileFragService.cs @@ -1,9 +1,8 @@ using Microsoft.Extensions.Logging; -using System.Diagnostics; using System.Text.RegularExpressions; using System.Threading.Channels; -namespace LightlessSync.Services.Compression +namespace LightlessSync.Services.Compactor { /// /// This batch service is made for the File Frag command, because of each file needing to use this command. @@ -19,13 +18,26 @@ namespace LightlessSync.Services.Compression private readonly TimeSpan _flushDelay; private readonly CancellationTokenSource _cts = new(); - public BatchFilefragService(bool useShell, ILogger log, int batchSize = 128, int flushMs = 25) + public delegate (bool ok, string stdout, string stderr, int exitCode) RunDirect(string fileName, IEnumerable args, string? workingDir, int timeoutMs); + private readonly RunDirect _runDirect; + + public delegate (bool ok, string stdout, string stderr, int exitCode) RunShell(string command, string? workingDir, int timeoutMs); + private readonly RunShell _runShell; + + public BatchFilefragService(bool useShell, ILogger log, int batchSize = 128, int flushMs = 25, RunDirect? runDirect = null, RunShell? runShell = null) { _useShell = useShell; _log = log; _batchSize = Math.Max(8, batchSize); _flushDelay = TimeSpan.FromMilliseconds(Math.Max(5, flushMs)); _ch = Channel.CreateUnbounded<(string, TaskCompletionSource)>(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); + + // require runners to be setup, wouldnt start otherwise + if (runDirect is null || runShell is null) + throw new ArgumentNullException(nameof(runDirect), "Provide process runners from FileCompactor"); + _runDirect = runDirect; + _runShell = runShell; + _worker = Task.Run(ProcessAsync, _cts.Token); } @@ -92,7 +104,7 @@ namespace LightlessSync.Services.Compression try { - var map = await RunBatchAsync(pending.Select(p => p.path)).ConfigureAwait(false); + var map = RunBatch(pending.Select(p => p.path)); foreach (var (path, tcs) in pending) { tcs.TrySetResult(map.TryGetValue(path, out var c) && c); @@ -124,75 +136,33 @@ namespace LightlessSync.Services.Compression /// Paths that are needed for the command building for the batch return /// Path of the file and if it went correctly /// Failing to start filefrag on the system if this exception is found - private async Task> RunBatchAsync(IEnumerable paths) + private Dictionary RunBatch(IEnumerable paths) { var list = paths.Distinct(StringComparer.Ordinal).ToList(); var result = list.ToDictionary(p => p, _ => false, StringComparer.Ordinal); - ProcessStartInfo psi; + (bool ok, string stdout, string stderr, int code) res; + if (_useShell) { - var inner = "filefrag -v -- " + string.Join(' ', list.Select(QuoteSingle)); - psi = new ProcessStartInfo - { - FileName = "/bin/bash", - Arguments = "-lc " + QuoteDouble(inner), - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; - - if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p)) - psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin"; - else - psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p; + var inner = "filefrag -v " + string.Join(' ', list.Select(QuoteSingle)); + res = _runShell(inner, timeoutMs: 15000, workingDir: "/"); } else { - psi = new ProcessStartInfo - { - FileName = "filefrag", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - - if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p)) - psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin"; - else - psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p; - - psi.ArgumentList.Add("-v"); - psi.ArgumentList.Add("--"); + var args = new List { "-v" }; foreach (var path in list) - psi.ArgumentList.Add(path); + { + args.Add(' ' + path); + } + + res = _runDirect("filefrag", args, workingDir: "/", timeoutMs: 15000); } - using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Failed to start filefrag"); + if (!string.IsNullOrWhiteSpace(res.stderr)) + _log.LogTrace("filefrag stderr (batch): {err}", res.stderr.Trim()); - var outTask = proc.StandardOutput.ReadToEndAsync(_cts.Token); - var errTask = proc.StandardError.ReadToEndAsync(_cts.Token); - - var timeout = TimeSpan.FromSeconds(15); - var combined = Task.WhenAll(outTask, errTask); - var finished = await Task.WhenAny(combined, Task.Delay(timeout, _cts.Token)).ConfigureAwait(false); - - if (finished != combined) - { - try { proc.Kill(entireProcessTree: true); } catch { /* ignore */ } - try { await combined.ConfigureAwait(false); } catch { /* ignore */ } - } - - var stdout = outTask.IsCompletedSuccessfully ? await outTask.ConfigureAwait(false) : ""; - var stderr = errTask.IsCompletedSuccessfully ? await errTask.ConfigureAwait(false) : ""; - - if (!string.IsNullOrWhiteSpace(stderr)) - _log.LogTrace("filefrag stderr (batch): {err}", stderr.Trim()); - - ParseFilefrag(stdout, result); + ParseFilefrag(res.stdout, result); return result; } @@ -225,7 +195,6 @@ namespace LightlessSync.Services.Compression } private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; - private static string QuoteDouble(string s) => "\"" + s.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal).Replace("$", "\\$", StringComparison.Ordinal).Replace("`", "\\`", StringComparison.Ordinal) + "\""; /// /// Regex of the File Size return on the Linux/Wine systems, giving back the amount diff --git a/LightlessSync/UI/SettingsUi.cs b/LightlessSync/UI/SettingsUi.cs index d686a75..0645118 100644 --- a/LightlessSync/UI/SettingsUi.cs +++ b/LightlessSync/UI/SettingsUi.cs @@ -1236,7 +1236,7 @@ public class SettingsUi : WindowMediatorSubscriberBase UIColors.Get("LightlessYellow")); } - if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); + if (!_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); if (ImGui.Checkbox("Use file compactor", ref useFileCompactor)) { _configService.Current.UseCompactor = useFileCompactor; @@ -1281,20 +1281,20 @@ public class SettingsUi : WindowMediatorSubscriberBase UIColors.Get("LightlessYellow")); } - if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) + if (!_cacheMonitor.StorageisNTFS) { ImGui.EndDisabled(); - ImGui.TextUnformatted("The file compactor is only available on BTRFS and NTFS drives."); + ImGui.TextUnformatted("The file compactor is only available NTFS drives, soon for btrfs."); } if (_cacheMonitor.StorageisNTFS) { - ImGui.TextUnformatted("The file compactor is running on NTFS Drive."); + ImGui.TextUnformatted("The file compactor detected an NTFS Drive."); } if (_cacheMonitor.StorageIsBtrfs) { - ImGui.TextUnformatted("The file compactor is running on Btrfs Drive."); + ImGui.TextUnformatted("The file compactor detected an Btrfs Drive."); } ImGuiHelpers.ScaledDummy(new Vector2(10, 10));