From d7182e9d5760bbc53f1d9546fc264589529355f7 Mon Sep 17 00:00:00 2001 From: cake Date: Mon, 10 Nov 2025 03:52:37 +0100 Subject: [PATCH 1/3] Hopefully fixes all issues with linux based path finding --- LightlessSync/FileCache/FileCompactor.cs | 491 +++++++++++------- .../Compression/BatchFileFragService.cs | 44 +- LightlessSync/Utils/FileSystemHelper.cs | 16 +- 3 files changed, 347 insertions(+), 204 deletions(-) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index be89c1f..e6d67ca 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -82,7 +82,7 @@ public sealed class FileCompactor : IDisposable _fragBatch = new BatchFilefragService( useShell: _dalamudUtilService.IsWine, log: _logger, - batchSize: 128, + batchSize: 256, flushMs: 25); _logger.LogInformation("FileCompactor started with {workers} workers", workerCount); @@ -192,23 +192,32 @@ public sealed class FileCompactor : IDisposable { try { - bool isWine = _dalamudUtilService?.IsWine ?? false; - string realPath = isWine ? ToLinuxPathIfWine(fileInfo.FullName, isWine) : fileInfo.FullName; + bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + var (_, linuxPath) = ResolvePathsForBtrfs(fileInfo.FullName); - (bool ok, string stdout, string stderr, int code) = - RunProcessDirect("stat", ["-c", "%b", realPath]); + var (ok1, out1, err1, code1) = + isWindowsProc + ? RunProcessShell($"stat -c %b -- {QuoteSingle(linuxPath)}", null, 10000) + : RunProcessDirect("stat", new[] { "-c", "%b", "--", linuxPath }, null, 10000); - if (!ok || !long.TryParse(stdout.Trim(), out var blocks)) - throw new InvalidOperationException($"stat failed (exit {code}): {stderr}"); + if (ok1 && long.TryParse(out1.Trim(), out long blocks)) + return (false, blocks * 512L); // st_blocks are 512B units - return (flowControl: false, value: blocks * 512L); + // Fallback: du -B1 (true on-disk bytes) + var (ok2, out2, err2, code2) = RunProcessShell($"du -B1 -- {QuoteSingle(linuxPath)} | cut -f1", null, 10000); // use shell for the pipe + + if (ok2 && long.TryParse(out2.Trim(), out long bytes)) + return (false, bytes); + + _logger.LogDebug("Btrfs size probe failed for {linux} (stat {code1}, du {code2}). Falling back to Length.", + linuxPath, code1, code2); + return (false, fileInfo.Length); } catch (Exception ex) { - _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); + _logger.LogDebug(ex, "Failed Btrfs size probe for {file}, using Length", fileInfo.FullName); + return (false, fileInfo.Length); } - - return (flowControl: true, value: default); } /// @@ -357,59 +366,56 @@ public sealed class FileCompactor : IDisposable /// Decompressing state private bool DecompressBtrfsFile(string path) { - try + return RunWithBtrfsGate(() => { - _btrfsGate.Wait(_compactionCts.Token); - bool isWine = _dalamudUtilService?.IsWine ?? false; - string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; - - var mountOptions = GetMountOptionsForPath(realPath); - if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) + try { - _logger.LogWarning( - "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " + - "Remount with 'compress=no' before running decompression.", - realPath, mountOptions); - return false; - } + var (winPath, linuxPath) = ResolvePathsForBtrfs(path); + bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); - if (!IsBtrfsCompressedFile(realPath)) - { - _logger.LogTrace("File {file} is not compressed, skipping decompression.", realPath); + var opts = GetMountOptionsForPath(linuxPath); + if (opts.Contains("compress", StringComparison.OrdinalIgnoreCase)) + { + _logger.LogWarning( + "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). Remount with 'compress=no'.", + linuxPath, opts); + return false; + } + + if (!IsBtrfsCompressedFile(linuxPath)) + { + _logger.LogTrace("Btrfs: not compressed, skip {file}", linuxPath); + return true; + } + + if (!ProbeFileReadableForBtrfs(winPath, linuxPath)) + return false; + + // Rewrite file uncompressed + (bool ok, string stdout, string stderr, int code) = + isWindowsProc + ? RunProcessShell($"btrfs filesystem defragment -- {QuoteSingle(linuxPath)}") + : RunProcessDirect("btrfs", ["filesystem", "defragment", "--", linuxPath]); + + if (!ok) + { + _logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit {code}): {stderr}", + linuxPath, code, stderr); + return false; + } + + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs (decompress) output {file}: {out}", linuxPath, stdout.Trim()); + + _logger.LogInformation("Decompressed (rewritten) Btrfs file: {file}", linuxPath); return true; } - - if (!ProbeFileReadable(realPath)) - return false; - - (bool ok, string stdout, string stderr, int code) = - isWine - ? RunProcessShell($"btrfs filesystem defragment -- {QuoteSingle(realPath)}") - : RunProcessDirect("btrfs", ["filesystem", "defragment", "--", realPath]); - - if (!ok) + catch (Exception ex) { - _logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit {code}): {stderr}", - realPath, code, stderr); + _logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path); return false; } - - if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); - - _logger.LogInformation("Decompressed (rewritten) Btrfs file: {file}", realPath); - return true; - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path); - return false; - } - finally - { - if (_btrfsGate.CurrentCount < 4) - _btrfsGate.Release(); - } + }); } /// @@ -459,19 +465,70 @@ public sealed class FileCompactor : IDisposable /// Path that has to be converted /// Extra check if using the wine enviroment /// Converted path to be used in Linux - private string ToLinuxPathIfWine(string path, bool isWine) + private static string ToLinuxPathIfWine(string path, bool isWine) { - if (!IsProbablyWine() && !isWine) + if (!isWine || !IsProbablyWine()) return path; - string linuxPath = path; - if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - linuxPath = "/" + path[3..].Replace('\\', '/'); - else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - linuxPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/'); + if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + return "/" + path[3..].Replace('\\', '/'); - _logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", linuxPath); - return linuxPath; + if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + { + var p = path.Replace('/', '\\'); + + const string usersPrefix = "C:\\Users\\"; + if (p.StartsWith(usersPrefix, StringComparison.OrdinalIgnoreCase)) + { + int afterUsers = usersPrefix.Length; + int slash = p.IndexOf('\\', afterUsers); + if (slash > 0 && slash + 1 < p.Length) + { + var rel = p[(slash + 1)..].Replace('\\', '/'); + var home = Environment.GetEnvironmentVariable("HOME"); + if (string.IsNullOrEmpty(home)) + { + var linuxUser = Environment.GetEnvironmentVariable("USER") ?? Environment.UserName; + home = "/home/" + linuxUser; + } + // Join as Unix path + return (home.TrimEnd('/') + "/" + rel).Replace("//", "/", StringComparison.Ordinal); + } + } + + try + { + var inner = "winepath -u " + "'" + path.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; + var psi = new ProcessStartInfo + { + FileName = "/bin/bash", + Arguments = "-c " + "\"" + inner.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal) + "\"", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = "/" + }; + using var proc = Process.Start(psi); + var outp = proc?.StandardOutput.ReadToEnd().Trim(); + try + { + proc?.WaitForExit(); + } + catch + { + /* Wine can throw here; ignore */ + } + if (!string.IsNullOrEmpty(outp) && outp.StartsWith("/", StringComparison.Ordinal)) + return outp; + } + catch + { + /* ignore and fall through */ + } + } + + return path.Replace('\\', '/'); } /// @@ -589,25 +646,31 @@ public sealed class FileCompactor : IDisposable /// State of the file private bool IsBtrfsCompressedFile(string path) { - try + return RunWithBtrfsGate(() => { - _btrfsGate.Wait(_compactionCts.Token); + try + { + bool windowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + string linuxPath = windowsProc ? ResolveLinuxPathForWine(path) : path; - bool isWine = _dalamudUtilService?.IsWine ?? false; - string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; + var task = _fragBatch.IsCompressedAsync(linuxPath, _compactionCts.Token); - return _fragBatch.IsCompressedAsync(realPath, _compactionCts.Token).GetAwaiter().GetResult(); - } - catch (Exception ex) - { - _logger.LogDebug(ex, "Failed to detect Btrfs compression for {file}", path); - return false; - } - finally - { - if (_btrfsGate.CurrentCount < 4) - _btrfsGate.Release(); - } + if (task.Wait(TimeSpan.FromSeconds(5), _compactionCts.Token) && task.IsCompletedSuccessfully) + return task.Result; + + _logger.LogTrace("filefrag batch timed out for {file}", linuxPath); + return false; + } + catch (OperationCanceledException) + { + return false; + } + catch (Exception ex) + { + _logger.LogDebug(ex, "filefrag batch check failed for {file}", path); + return false; + } + }); } /// @@ -617,85 +680,48 @@ public sealed class FileCompactor : IDisposable /// Compessing state private bool BtrfsCompressFile(string path) { - try - { - bool isWine = _dalamudUtilService?.IsWine ?? false; - string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; - - var fi = new FileInfo(realPath); - - if (fi == null) - { - _logger.LogWarning("Failed to open {file} for compression; skipping", realPath); - return false; - } - - if (IsBtrfsCompressedFile(realPath)) - { - _logger.LogTrace("File {file} already compressed (Btrfs), skipping file", realPath); - return true; - } - - if (!ProbeFileReadable(realPath)) - return false; - - (bool ok, string stdout, string stderr, int code) = - isWine - ? RunProcessShell($"btrfs filesystem defragment -clzo -- {QuoteSingle(realPath)}") - : RunProcessDirect("btrfs", ["filesystem", "defragment", "-clzo", "--", realPath]); - - if (!ok) - { - _logger.LogWarning("btrfs defragment failed for {file} (exit {code}): {stderr}", realPath, code, stderr); - return false; - } - - if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogTrace("btrfs output for {file}: {stdout}", realPath, stdout.Trim()); - - if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); - - _logger.LogInformation("Compressed btrfs file successfully: {file}", realPath); - - return true; - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); - return false; - } - } - - - /// - /// Probe file if its readable for certain amount of tries. - /// - /// Path where the file is located - /// Filestream used for the function - /// State of the filestream opening - private bool ProbeFileReadable(string path) - { - for (int attempt = 0; attempt < _maxRetries; attempt++) - { + return RunWithBtrfsGate(() => { try { - using var _ = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); - return true; - } - catch (IOException ex) - { - if (attempt == _maxRetries - 1) + var (winPath, linuxPath) = ResolvePathsForBtrfs(path); + bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + + if (IsBtrfsCompressedFile(linuxPath)) { - _logger.LogWarning(ex, "File still in use after {attempts} attempts, skipping {file}", _maxRetries, path); + _logger.LogTrace("Already Btrfs compressed: {file} (linux={linux})", winPath, linuxPath); + return true; + } + + if (!ProbeFileReadableForBtrfs(winPath, linuxPath)) + { + _logger.LogTrace("Probe failed; cannot open file for compress: {file} (linux={linux})", winPath, linuxPath); return false; } - int delay = 150 * (attempt + 1); - _logger.LogTrace(ex, "File busy, retrying in {delay}ms for {file}", delay, path); - Thread.Sleep(delay); + + (bool ok, string stdout, string stderr, int code) = + isWindowsProc + ? RunProcessShell($"btrfs filesystem defragment -clzo -- {QuoteSingle(linuxPath)}") + : RunProcessDirect("btrfs", ["filesystem", "defragment", "-clzo", "--", linuxPath]); + + if (!ok) + { + _logger.LogWarning("btrfs defragment failed for {file} (linux={linux}) exit {code}: {stderr}", + winPath, linuxPath, code, stderr); + return false; + } + + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs output for {file}: {out}", winPath, stdout.Trim()); + + _logger.LogInformation("Compressed btrfs file successfully: {file} (linux={linux})", winPath, linuxPath); + return true; } - } - return false; + catch (Exception ex) + { + _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); + return false; + } + }); } /// @@ -742,7 +768,7 @@ public sealed class FileCompactor : IDisposable } /// - /// Runs an nonshell process meant for Linux/Wine enviroments + /// Runs an nonshell process meant for Linux enviroments /// /// File that has to be excuted /// Arguments meant for the file/command @@ -759,32 +785,47 @@ public sealed class FileCompactor : IDisposable CreateNoWindow = true }; if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir; - foreach (var a in args) psi.ArgumentList.Add(a); + EnsureUnixPathEnv(psi); using var proc = Process.Start(psi); if (proc is null) return (false, "", "failed to start process", -1); var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token); var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token); + var both = Task.WhenAll(outTask, errTask); + + if (_dalamudUtilService.IsWine) + { + var finished = Task.WhenAny(both, Task.Delay(timeoutMs, _compactionCts.Token)).GetAwaiter().GetResult(); + if (finished != both) + { + try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ } + try { Task.WaitAll(new[] { outTask, errTask }, 1000, _compactionCts.Token); } catch { /* ignore this */ } + var so = outTask.IsCompleted ? outTask.Result : ""; + var se = errTask.IsCompleted ? errTask.Result : "timeout"; + return (false, so, se, -1); + } + + var stdout = outTask.Result; + var stderr = errTask.Result; + var ok = string.IsNullOrWhiteSpace(stderr); + return (ok, stdout, stderr, ok ? 0 : -1); + } if (!proc.WaitForExit(timeoutMs)) { - try - { - proc.Kill(entireProcessTree: true); - } - catch - { - // Ignore this catch on the dispose - } - - Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); - return (false, outTask.Result, "timeout", -1); + try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ } + try { Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); } catch { /* ignore this */ } + return (false, outTask.IsCompleted ? outTask.Result : "", "timeout", -1); } Task.WaitAll(outTask, errTask); - return (proc.ExitCode == 0, outTask.Result, errTask.Result, proc.ExitCode); + var so2 = outTask.Result; + var se2 = errTask.Result; + int code; + try { code = proc.ExitCode; } catch { code = -1; } + return (code == 0, so2, se2, code); } /// @@ -793,8 +834,9 @@ public sealed class FileCompactor : IDisposable /// Command that has to be excuted /// Timeout timer for the process /// State of the process, output of the process and error with exit code - private (bool ok, string stdout, string stderr, int exitCode) RunProcessShell(string command, int timeoutMs = 60000) + private (bool ok, string stdout, string stderr, int exitCode) RunProcessShell(string command, string? workingDir = null, int timeoutMs = 60000) { + // Use a LOGIN shell so PATH includes /usr/sbin etc. var psi = new ProcessStartInfo("/bin/bash") { RedirectStandardOutput = true, @@ -802,32 +844,50 @@ public sealed class FileCompactor : IDisposable UseShellExecute = false, CreateNoWindow = true }; - psi.ArgumentList.Add("-c"); - psi.ArgumentList.Add(command); + if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir; + + psi.ArgumentList.Add("-lc"); + psi.ArgumentList.Add(QuoteDouble(command)); + EnsureUnixPathEnv(psi); using var proc = Process.Start(psi); if (proc is null) return (false, "", "failed to start /bin/bash", -1); var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token); var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token); + var both = Task.WhenAll(outTask, errTask); + + if (_dalamudUtilService.IsWine) + { + var finished = Task.WhenAny(both, Task.Delay(timeoutMs, _compactionCts.Token)).GetAwaiter().GetResult(); + if (finished != both) + { + try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ } + try { Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); } catch { /* ignore this */ } + var so = outTask.IsCompleted ? outTask.Result : ""; + var se = errTask.IsCompleted ? errTask.Result : "timeout"; + return (false, so, se, -1); + } + + var stdout = outTask.Result; + var stderr = errTask.Result; + var ok = string.IsNullOrWhiteSpace(stderr); + return (ok, stdout, stderr, ok ? 0 : -1); + } if (!proc.WaitForExit(timeoutMs)) { - try - { - proc.Kill(entireProcessTree: true); - } - catch - { - // Ignore this catch on the dispose - } - - Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); - return (false, outTask.Result, "timeout", -1); + try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ } + try { Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); } catch { /* ignore this */ } + return (false, outTask.IsCompleted ? outTask.Result : "", "timeout", -1); } Task.WaitAll(outTask, errTask); - return (proc.ExitCode == 0, outTask.Result, errTask.Result, proc.ExitCode); + var so2 = outTask.Result; + var se2 = errTask.Result; + int code; + try { code = proc.ExitCode; } catch { code = -1; } + return (code == 0, so2, se2, code); } /// @@ -931,6 +991,69 @@ public sealed class FileCompactor : IDisposable } } + private string ResolveLinuxPathForWine(string windowsPath) + { + var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(windowsPath)}", null, 5000); + if (ok && !string.IsNullOrWhiteSpace(outp)) return outp.Trim(); + return ToLinuxPathIfWine(windowsPath, isWine: true); + } + + private static void EnsureUnixPathEnv(ProcessStartInfo psi) + { + if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p)) + psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin"; + else if (!p.Contains("/usr/sbin", StringComparison.Ordinal)) + psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p; + } + + private (string windowsPath, string linuxPath) ResolvePathsForBtrfs(string path) + { + bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + + if (!isWindowsProc) + return (path, path); + + // Prefer winepath -u; fall back to your existing mapper + var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(path)}", null, 5000); + var linux = (ok && !string.IsNullOrWhiteSpace(outp)) ? outp.Trim() + : ToLinuxPathIfWine(path, isWine: true); + + return (path, linux); + } + + private bool ProbeFileReadableForBtrfs(string windowsPath, string linuxPath) + { + try + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + using var _ = new FileStream(windowsPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + } + else + { + using var _ = new FileStream(linuxPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + } + return true; + } + catch { return false; } + } + + private T RunWithBtrfsGate(Func body) + { + bool acquired = false; + try + { + _btrfsGate.Wait(_compactionCts.Token); + acquired = true; + return body(); + } + finally + { + if (acquired) _btrfsGate.Release(); + } + } + + [DllImport("kernel32.dll", SetLastError = true)] private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped); @@ -945,6 +1068,8 @@ public sealed class FileCompactor : IDisposable private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; + private static string QuoteDouble(string s) => "\"" + s.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal).Replace("$", "\\$", StringComparison.Ordinal).Replace("`", "\\`", StringComparison.Ordinal) + "\""; + public void Dispose() { _fragBatch?.Dispose(); diff --git a/LightlessSync/Services/Compression/BatchFileFragService.cs b/LightlessSync/Services/Compression/BatchFileFragService.cs index ae5bb71..16c05e1 100644 --- a/LightlessSync/Services/Compression/BatchFileFragService.cs +++ b/LightlessSync/Services/Compression/BatchFileFragService.cs @@ -136,13 +136,18 @@ namespace LightlessSync.Services.Compression psi = new ProcessStartInfo { FileName = "/bin/bash", - Arguments = "-c " + QuoteDouble(inner), + Arguments = "-lc " + QuoteDouble(inner), RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true, WorkingDirectory = "/" }; + + if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p)) + psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin"; + else + psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p; } else { @@ -154,29 +159,38 @@ namespace LightlessSync.Services.Compression UseShellExecute = false, CreateNoWindow = true }; + + if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p)) + psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin"; + else + psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p; + psi.ArgumentList.Add("-v"); psi.ArgumentList.Add("--"); - foreach (var p in list) psi.ArgumentList.Add(p); + foreach (var path in list) + psi.ArgumentList.Add(path); } using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Failed to start filefrag"); - var stdoutTask = proc.StandardOutput.ReadToEndAsync(_cts.Token); - var stderrTask = proc.StandardError.ReadToEndAsync(_cts.Token); - await Task.WhenAll(stdoutTask, stderrTask).ConfigureAwait(false); - try - { - await proc.WaitForExitAsync(_cts.Token).ConfigureAwait(false); - } - catch (Exception ex) + + var outTask = proc.StandardOutput.ReadToEndAsync(_cts.Token); + var errTask = proc.StandardError.ReadToEndAsync(_cts.Token); + + var timeout = TimeSpan.FromSeconds(15); + var combined = Task.WhenAll(outTask, errTask); + var finished = await Task.WhenAny(combined, Task.Delay(timeout, _cts.Token)).ConfigureAwait(false); + + if (finished != combined) { - _log.LogWarning(ex, "Error in the batch frag service. proc = {proc}", proc); + try { proc.Kill(entireProcessTree: true); } catch { /* ignore */ } + try { await combined.ConfigureAwait(false); } catch { /* ignore */ } } - var stdout = await stdoutTask.ConfigureAwait(false); - var stderr = await stderrTask.ConfigureAwait(false); + var stdout = outTask.IsCompletedSuccessfully ? await outTask.ConfigureAwait(false) : ""; + var stderr = errTask.IsCompletedSuccessfully ? await errTask.ConfigureAwait(false) : ""; - if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) - _log.LogTrace("filefrag exited {code}: {err}", proc.ExitCode, stderr.Trim()); + if (!string.IsNullOrWhiteSpace(stderr)) + _log.LogTrace("filefrag stderr (batch): {err}", stderr.Trim()); ParseFilefrag(stdout, result); return result; diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs index af4c98b..d63b3b9 100644 --- a/LightlessSync/Utils/FileSystemHelper.cs +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -252,14 +252,18 @@ namespace LightlessSync.Utils }; using var proc = Process.Start(psi); - string stdout = proc?.StandardOutput.ReadToEnd().Trim() ?? ""; - proc?.WaitForExit(); - if (int.TryParse(stdout, out int blockSize) && blockSize > 0) + string stdout = proc?.StandardOutput.ReadToEnd().Trim() ?? ""; + string _stderr = proc?.StandardError.ReadToEnd() ?? ""; + + try { proc?.WaitForExit(); } + catch (Exception ex) { logger?.LogTrace(ex, "stat WaitForExit failed under Wine; ignoring"); } + + if (!(!int.TryParse(stdout, out int block) || block <= 0)) { - _blockSizeCache[root] = blockSize; - logger?.LogTrace("Filesystem block size via stat for {root}: {block}", root, blockSize); - return blockSize; + _blockSizeCache[root] = block; + logger?.LogTrace("Filesystem block size via stat for {root}: {block}", root, block); + return block; } logger?.LogTrace("stat did not return valid block size for {file}, output: {out}", fi.FullName, stdout); From 7de72471bb96f091aba2db5c81440d53a5f72976 Mon Sep 17 00:00:00 2001 From: cake Date: Mon, 10 Nov 2025 06:25:35 +0100 Subject: [PATCH 2/3] Refactored --- LightlessSync/FileCache/FileCompactor.cs | 171 ++++++++++++++--------- 1 file changed, 102 insertions(+), 69 deletions(-) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index e6d67ca..5e0b4a9 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -6,6 +6,7 @@ using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.InteropServices; +using System.Threading; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; @@ -82,7 +83,7 @@ public sealed class FileCompactor : IDisposable _fragBatch = new BatchFilefragService( useShell: _dalamudUtilService.IsWine, log: _logger, - batchSize: 256, + batchSize: 128, flushMs: 25); _logger.LogInformation("FileCompactor started with {workers} workers", workerCount); @@ -198,13 +199,12 @@ public sealed class FileCompactor : IDisposable var (ok1, out1, err1, code1) = isWindowsProc ? RunProcessShell($"stat -c %b -- {QuoteSingle(linuxPath)}", null, 10000) - : RunProcessDirect("stat", new[] { "-c", "%b", "--", linuxPath }, null, 10000); + : RunProcessDirect("stat", ["-c", "%b", "--", linuxPath], null, 10000); if (ok1 && long.TryParse(out1.Trim(), out long blocks)) - return (false, blocks * 512L); // st_blocks are 512B units + return (false, blocks * 512L); // st_blocks are always 512B units - // Fallback: du -B1 (true on-disk bytes) - var (ok2, out2, err2, code2) = RunProcessShell($"du -B1 -- {QuoteSingle(linuxPath)} | cut -f1", null, 10000); // use shell for the pipe + var (ok2, out2, err2, code2) = RunProcessShell($"du -B1 -- {QuoteSingle(linuxPath)} | cut -f1", workingDir: null, 10000); // use shell for the pipe if (ok2 && long.TryParse(out2.Trim(), out long bytes)) return (false, bytes); @@ -425,6 +425,7 @@ public sealed class FileCompactor : IDisposable /// Decompressing state private bool DecompressWOFFile(string path) { + //Check if its already been compressed if (TryIsWofExternal(path, out bool isExternal, out int algo)) { if (!isExternal) @@ -436,6 +437,7 @@ public sealed class FileCompactor : IDisposable _logger.LogTrace("WOF compression (algo={algo}) detected for {file}", compressString, path); } + //This will attempt to start WOF thread. return WithFileHandleForWOF(path, FileAccess.ReadWrite, h => { if (!DeviceIoControl(h, FSCTL_DELETE_EXTERNAL_BACKING, @@ -524,7 +526,7 @@ public sealed class FileCompactor : IDisposable } catch { - /* ignore and fall through */ + /* ignore and fall through the floor! */ } } @@ -550,7 +552,7 @@ public sealed class FileCompactor : IDisposable { int ret = WofSetFileDataLocation(h, WOF_PROVIDER_FILE, efInfoPtr, length); - // 0x80070158 is the benign "already compressed/unsupported" style return + // 0x80070158 is the being "already compressed/unsupported" style return if (ret != 0 && ret != unchecked((int)0x80070158)) { _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); @@ -791,38 +793,12 @@ public sealed class FileCompactor : IDisposable using var proc = Process.Start(psi); if (proc is null) return (false, "", "failed to start process", -1); - var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token); - var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token); - var both = Task.WhenAll(outTask, errTask); - - if (_dalamudUtilService.IsWine) + var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token); + if (!success) { - var finished = Task.WhenAny(both, Task.Delay(timeoutMs, _compactionCts.Token)).GetAwaiter().GetResult(); - if (finished != both) - { - try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ } - try { Task.WaitAll(new[] { outTask, errTask }, 1000, _compactionCts.Token); } catch { /* ignore this */ } - var so = outTask.IsCompleted ? outTask.Result : ""; - var se = errTask.IsCompleted ? errTask.Result : "timeout"; - return (false, so, se, -1); - } - - var stdout = outTask.Result; - var stderr = errTask.Result; - var ok = string.IsNullOrWhiteSpace(stderr); - return (ok, stdout, stderr, ok ? 0 : -1); + return (false, so2, se2, -1); } - if (!proc.WaitForExit(timeoutMs)) - { - try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ } - try { Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); } catch { /* ignore this */ } - return (false, outTask.IsCompleted ? outTask.Result : "", "timeout", -1); - } - - Task.WaitAll(outTask, errTask); - var so2 = outTask.Result; - var se2 = errTask.Result; int code; try { code = proc.ExitCode; } catch { code = -1; } return (code == 0, so2, se2, code); @@ -836,7 +812,7 @@ public sealed class FileCompactor : IDisposable /// State of the process, output of the process and error with exit code private (bool ok, string stdout, string stderr, int exitCode) RunProcessShell(string command, string? workingDir = null, int timeoutMs = 60000) { - // Use a LOGIN shell so PATH includes /usr/sbin etc. + var psi = new ProcessStartInfo("/bin/bash") { RedirectStandardOutput = true, @@ -845,7 +821,7 @@ public sealed class FileCompactor : IDisposable CreateNoWindow = true }; if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir; - + // Use a Login shell so PATH includes /usr/sbin etc. AKA -lc psi.ArgumentList.Add("-lc"); psi.ArgumentList.Add(QuoteDouble(command)); EnsureUnixPathEnv(psi); @@ -853,43 +829,74 @@ public sealed class FileCompactor : IDisposable using var proc = Process.Start(psi); if (proc is null) return (false, "", "failed to start /bin/bash", -1); - var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token); - var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token); - var both = Task.WhenAll(outTask, errTask); - - if (_dalamudUtilService.IsWine) + var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token); + if (!success) { - var finished = Task.WhenAny(both, Task.Delay(timeoutMs, _compactionCts.Token)).GetAwaiter().GetResult(); - if (finished != both) - { - try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ } - try { Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); } catch { /* ignore this */ } - var so = outTask.IsCompleted ? outTask.Result : ""; - var se = errTask.IsCompleted ? errTask.Result : "timeout"; - return (false, so, se, -1); - } - - var stdout = outTask.Result; - var stderr = errTask.Result; - var ok = string.IsNullOrWhiteSpace(stderr); - return (ok, stdout, stderr, ok ? 0 : -1); + return (false, so2, se2, -1); } - if (!proc.WaitForExit(timeoutMs)) - { - try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ } - try { Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); } catch { /* ignore this */ } - return (false, outTask.IsCompleted ? outTask.Result : "", "timeout", -1); - } - - Task.WaitAll(outTask, errTask); - var so2 = outTask.Result; - var se2 = errTask.Result; int code; try { code = proc.ExitCode; } catch { code = -1; } return (code == 0, so2, se2, code); } + /// + /// Checking the process result for shell or direct processes + /// + /// Process + /// How long when timeout is gotten + /// Cancellation Token + /// Multiple variables + private (bool success, string testy, string testi) CheckProcessResult(Process proc, int timeoutMs, CancellationToken token) + { + var outTask = proc.StandardOutput.ReadToEndAsync(token); + var errTask = proc.StandardError.ReadToEndAsync(token); + var bothTasks = Task.WhenAll(outTask, errTask); + + //On wine, we dont wanna use waitforexit as it will be always broken and giving an error. + if (_dalamudUtilService.IsWine) + { + var finished = Task.WhenAny(bothTasks, Task.Delay(timeoutMs, token)).GetAwaiter().GetResult(); + if (finished != bothTasks) + { + try + { + proc.Kill(entireProcessTree: true); + Task.WaitAll([outTask, errTask], 1000, token); + } + catch + { + // ignore this + } + var so = outTask.IsCompleted ? outTask.Result : ""; + var se = errTask.IsCompleted ? errTask.Result : "timeout"; + return (false, so, se); + } + + var stderr = errTask.Result; + var ok = string.IsNullOrWhiteSpace(stderr); + return (ok, outTask.Result, stderr); + } + + // On linux, we can use it as we please + if (!proc.WaitForExit(timeoutMs)) + { + try + { + proc.Kill(entireProcessTree: true); + Task.WaitAll([outTask, errTask], 1000, token); + } + catch + { + // ignore this + } + return (false, outTask.IsCompleted ? outTask.Result : "", "timeout"); + } + + Task.WaitAll(outTask, errTask); + return (true, outTask.Result, errTask.Result); + } + /// /// Enqueues the compaction/decompation of an filepath. /// @@ -991,6 +998,11 @@ public sealed class FileCompactor : IDisposable } } + /// + /// Resolves linux path from wine pathing + /// + /// Windows path given from Wine + /// Linux path to be used in Linux private string ResolveLinuxPathForWine(string windowsPath) { var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(windowsPath)}", null, 5000); @@ -998,6 +1010,10 @@ public sealed class FileCompactor : IDisposable return ToLinuxPathIfWine(windowsPath, isWine: true); } + /// + /// Ensures the Unix pathing to be included into the process + /// + /// Process private static void EnsureUnixPathEnv(ProcessStartInfo psi) { if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p)) @@ -1006,6 +1022,11 @@ public sealed class FileCompactor : IDisposable psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p; } + /// + /// Resolves paths for Btrfs to be used on wine or linux and windows in case + /// + /// Path given t + /// private (string windowsPath, string linuxPath) ResolvePathsForBtrfs(string path) { bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); @@ -1021,13 +1042,19 @@ public sealed class FileCompactor : IDisposable return (path, linux); } - private bool ProbeFileReadableForBtrfs(string windowsPath, string linuxPath) + /// + /// Probes file if its readable to be used + /// + /// Windows path + /// Linux path + /// Succesfully probed or not + private bool ProbeFileReadableForBtrfs(string winePath, string linuxPath) { try { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - using var _ = new FileStream(windowsPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var _ = new FileStream(winePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); } else { @@ -1038,6 +1065,12 @@ public sealed class FileCompactor : IDisposable catch { return false; } } + /// + /// Running functions into the Btrfs Gate/Threading. + /// + /// Type of the function that wants to be run inside Btrfs Gate + /// Body of the function + /// Task private T RunWithBtrfsGate(Func body) { bool acquired = false; From 1862689b1b37e355078b5420167b6338194f112e Mon Sep 17 00:00:00 2001 From: cake Date: Tue, 11 Nov 2025 17:09:50 +0100 Subject: [PATCH 3/3] Changed some commands in file getting, redone compression check commands and turned off btrfs compactor for 1.12.4 --- LightlessSync/FileCache/FileCompactor.cs | 144 +++++++++--------- .../BatchFileFragService.cs | 91 ++++------- LightlessSync/UI/SettingsUi.cs | 10 +- 3 files changed, 106 insertions(+), 139 deletions(-) rename LightlessSync/Services/{Compression => Compactor}/BatchFileFragService.cs (71%) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 5e0b4a9..3edf96a 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,12 +1,11 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; -using LightlessSync.Services.Compression; +using LightlessSync.Services.Compactor; using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.InteropServices; -using System.Threading; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; @@ -28,6 +27,8 @@ public sealed class FileCompactor : IDisposable private readonly List _workers = []; private readonly SemaphoreSlim _globalGate; + + //Limit btrfs gate on half of threads given to compactor. private static readonly SemaphoreSlim _btrfsGate = new(4, 4); private readonly BatchFilefragService _fragBatch; @@ -83,8 +84,11 @@ public sealed class FileCompactor : IDisposable _fragBatch = new BatchFilefragService( useShell: _dalamudUtilService.IsWine, log: _logger, - batchSize: 128, - flushMs: 25); + batchSize: 64, + flushMs: 25, + runDirect: RunProcessDirect, + runShell: RunProcessShell + ); _logger.LogInformation("FileCompactor started with {workers} workers", workerCount); } @@ -196,21 +200,15 @@ public sealed class FileCompactor : IDisposable bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); var (_, linuxPath) = ResolvePathsForBtrfs(fileInfo.FullName); - var (ok1, out1, err1, code1) = + var (ok, output, err, code) = isWindowsProc - ? RunProcessShell($"stat -c %b -- {QuoteSingle(linuxPath)}", null, 10000) - : RunProcessDirect("stat", ["-c", "%b", "--", linuxPath], null, 10000); + ? RunProcessShell($"stat -c='%b' {QuoteSingle(linuxPath)}", workingDir: null, 10000) + : RunProcessDirect("stat", ["-c='%b'", linuxPath], workingDir: null, 10000); - if (ok1 && long.TryParse(out1.Trim(), out long blocks)) + if (ok && long.TryParse(output.Trim(), out long blocks)) return (false, blocks * 512L); // st_blocks are always 512B units - var (ok2, out2, err2, code2) = RunProcessShell($"du -B1 -- {QuoteSingle(linuxPath)} | cut -f1", workingDir: null, 10000); // use shell for the pipe - - if (ok2 && long.TryParse(out2.Trim(), out long bytes)) - return (false, bytes); - - _logger.LogDebug("Btrfs size probe failed for {linux} (stat {code1}, du {code2}). Falling back to Length.", - linuxPath, code1, code2); + _logger.LogDebug("Btrfs size probe failed for {linux} (stat {code}, err {err}). Falling back to Length.", linuxPath, code, err); return (false, fileInfo.Length); } catch (Exception ex) @@ -360,7 +358,7 @@ public sealed class FileCompactor : IDisposable } /// - /// Decompress an BTRFS File + /// Decompress an BTRFS File on Wine/Linux /// /// Path of the compressed file /// Decompressing state @@ -370,44 +368,55 @@ public sealed class FileCompactor : IDisposable { try { - var (winPath, linuxPath) = ResolvePathsForBtrfs(path); - bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + bool isWine = _dalamudUtilService?.IsWine ?? false; + string linuxPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; var opts = GetMountOptionsForPath(linuxPath); - if (opts.Contains("compress", StringComparison.OrdinalIgnoreCase)) + bool hasCompress = opts.Contains("compress", StringComparison.OrdinalIgnoreCase); + bool hasCompressForce = opts.Contains("compress-force", StringComparison.OrdinalIgnoreCase); + + if (hasCompressForce) { - _logger.LogWarning( - "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). Remount with 'compress=no'.", - linuxPath, opts); + _logger.LogWarning("Cannot safely decompress {file}: mount options contains compress-force ({opts}).", linuxPath, opts); return false; } + if (hasCompress) + { + var setCmd = $"btrfs property set -- {QuoteDouble(linuxPath)} compression none"; + var (okSet, _, errSet, codeSet) = isWine + ? RunProcessShell(setCmd) + : RunProcessDirect("btrfs", ["property", "set", "--", linuxPath, "compression", "none"]); + + if (!okSet) + { + _logger.LogWarning("Failed to set 'compression none' on {file}, please check drive options (exit code is: {code}): {err}", linuxPath, codeSet, errSet); + return false; + } + _logger.LogTrace("Set per-file 'compression none' on {file}", linuxPath); + } + if (!IsBtrfsCompressedFile(linuxPath)) { - _logger.LogTrace("Btrfs: not compressed, skip {file}", linuxPath); + _logger.LogTrace("{file} is not compressed, skipping decompression completely", linuxPath); return true; } - if (!ProbeFileReadableForBtrfs(winPath, linuxPath)) - return false; - - // Rewrite file uncompressed - (bool ok, string stdout, string stderr, int code) = - isWindowsProc - ? RunProcessShell($"btrfs filesystem defragment -- {QuoteSingle(linuxPath)}") - : RunProcessDirect("btrfs", ["filesystem", "defragment", "--", linuxPath]); + var (ok, stdout, stderr, code) = isWine + ? RunProcessShell($"btrfs filesystem defragment -- {QuoteDouble(linuxPath)}") + : RunProcessDirect("btrfs", ["filesystem", "defragment", "--", linuxPath]); if (!ok) { - _logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit {code}): {stderr}", + _logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit code is: {code}): {stderr}", linuxPath, code, stderr); return false; } if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogTrace("btrfs (decompress) output {file}: {out}", linuxPath, stdout.Trim()); + _logger.LogTrace("btrfs defragment output for {file}: {out}", linuxPath, stdout.Trim()); - _logger.LogInformation("Decompressed (rewritten) Btrfs file: {file}", linuxPath); + _logger.LogInformation("Decompressed (rewritten uncompressed) Btrfs file: {file}", linuxPath); return true; } catch (Exception ex) @@ -467,19 +476,19 @@ public sealed class FileCompactor : IDisposable /// Path that has to be converted /// Extra check if using the wine enviroment /// Converted path to be used in Linux - private static string ToLinuxPathIfWine(string path, bool isWine) + private string ToLinuxPathIfWine(string path, bool isWine, bool preferShell = true) { if (!isWine || !IsProbablyWine()) return path; if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - return "/" + path[3..].Replace('\\', '/'); + return ("/" + path[3..].Replace('\\', '/')).Replace("//", "/", StringComparison.Ordinal); if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) { + const string usersPrefix = "C:\\Users\\"; var p = path.Replace('/', '\\'); - const string usersPrefix = "C:\\Users\\"; if (p.StartsWith(usersPrefix, StringComparison.OrdinalIgnoreCase)) { int afterUsers = usersPrefix.Length; @@ -493,48 +502,38 @@ public sealed class FileCompactor : IDisposable var linuxUser = Environment.GetEnvironmentVariable("USER") ?? Environment.UserName; home = "/home/" + linuxUser; } - // Join as Unix path - return (home.TrimEnd('/') + "/" + rel).Replace("//", "/", StringComparison.Ordinal); + return (home!.TrimEnd('/') + "/" + rel).Replace("//", "/", StringComparison.Ordinal); } } try { - var inner = "winepath -u " + "'" + path.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; - var psi = new ProcessStartInfo + (bool ok, string stdout, string stderr, int code) = preferShell + ? RunProcessShell($"winepath -u {QuoteSingle(path)}", timeoutMs: 5000, workingDir: "/") + : RunProcessDirect("winepath", ["-u", path], workingDir: "/", timeoutMs: 5000); + + if (ok) { - FileName = "/bin/bash", - Arguments = "-c " + "\"" + inner.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal) + "\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; - using var proc = Process.Start(psi); - var outp = proc?.StandardOutput.ReadToEnd().Trim(); - try - { - proc?.WaitForExit(); - } - catch - { - /* Wine can throw here; ignore */ + var outp = (stdout ?? "").Trim(); + if (!string.IsNullOrEmpty(outp) && outp.StartsWith('/')) + return outp.Replace("//", "/", StringComparison.Ordinal); + } + else + { + _logger.LogTrace("winepath failed for {path} (exit {code}): {err}", path, code, stderr); } - if (!string.IsNullOrEmpty(outp) && outp.StartsWith("/", StringComparison.Ordinal)) - return outp; } - catch + catch (Exception ex) { - /* ignore and fall through the floor! */ + _logger.LogTrace(ex, "winepath invocation failed for {path}", path); } } - - return path.Replace('\\', '/'); + + return path.Replace('\\', '/').Replace("//", "/", StringComparison.Ordinal); } /// - /// Compress an WOF File + /// Compress an File using the WOF methods (NTFS) /// /// Path of the decompressed/normal file /// Compessing state @@ -585,7 +584,7 @@ public sealed class FileCompactor : IDisposable } /// - /// Checks if an File is compacted with WOF compression + /// Checks if an File is compacted with WOF compression (NTFS) /// /// Path of the file /// State of the file @@ -612,7 +611,7 @@ public sealed class FileCompactor : IDisposable } /// - /// Checks if an File is compacted any WOF compression with an WOF backing + /// Checks if an File is compacted any WOF compression with an WOF backing (NTFS) /// /// Path of the file /// State of the file, if its an external (no backing) and which algorithm if detected @@ -821,7 +820,8 @@ public sealed class FileCompactor : IDisposable CreateNoWindow = true }; if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir; - // Use a Login shell so PATH includes /usr/sbin etc. AKA -lc + + // Use a Login shell so PATH includes /usr/sbin etc. AKA -lc for login shell psi.ArgumentList.Add("-lc"); psi.ArgumentList.Add(QuoteDouble(command)); EnsureUnixPathEnv(psi); @@ -1011,7 +1011,7 @@ public sealed class FileCompactor : IDisposable } /// - /// Ensures the Unix pathing to be included into the process + /// Ensures the Unix pathing to be included into the process start /// /// Process private static void EnsureUnixPathEnv(ProcessStartInfo psi) @@ -1034,10 +1034,8 @@ public sealed class FileCompactor : IDisposable if (!isWindowsProc) return (path, path); - // Prefer winepath -u; fall back to your existing mapper - var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(path)}", null, 5000); - var linux = (ok && !string.IsNullOrWhiteSpace(outp)) ? outp.Trim() - : ToLinuxPathIfWine(path, isWine: true); + var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(path)}", workingDir: null, 5000); + var linux = (ok && !string.IsNullOrWhiteSpace(outp)) ? outp.Trim() : ToLinuxPathIfWine(path, isWine: true); return (path, linux); } diff --git a/LightlessSync/Services/Compression/BatchFileFragService.cs b/LightlessSync/Services/Compactor/BatchFileFragService.cs similarity index 71% rename from LightlessSync/Services/Compression/BatchFileFragService.cs rename to LightlessSync/Services/Compactor/BatchFileFragService.cs index 16c05e1..b31919e 100644 --- a/LightlessSync/Services/Compression/BatchFileFragService.cs +++ b/LightlessSync/Services/Compactor/BatchFileFragService.cs @@ -1,9 +1,8 @@ using Microsoft.Extensions.Logging; -using System.Diagnostics; using System.Text.RegularExpressions; using System.Threading.Channels; -namespace LightlessSync.Services.Compression +namespace LightlessSync.Services.Compactor { /// /// This batch service is made for the File Frag command, because of each file needing to use this command. @@ -19,13 +18,26 @@ namespace LightlessSync.Services.Compression private readonly TimeSpan _flushDelay; private readonly CancellationTokenSource _cts = new(); - public BatchFilefragService(bool useShell, ILogger log, int batchSize = 128, int flushMs = 25) + public delegate (bool ok, string stdout, string stderr, int exitCode) RunDirect(string fileName, IEnumerable args, string? workingDir, int timeoutMs); + private readonly RunDirect _runDirect; + + public delegate (bool ok, string stdout, string stderr, int exitCode) RunShell(string command, string? workingDir, int timeoutMs); + private readonly RunShell _runShell; + + public BatchFilefragService(bool useShell, ILogger log, int batchSize = 128, int flushMs = 25, RunDirect? runDirect = null, RunShell? runShell = null) { _useShell = useShell; _log = log; _batchSize = Math.Max(8, batchSize); _flushDelay = TimeSpan.FromMilliseconds(Math.Max(5, flushMs)); _ch = Channel.CreateUnbounded<(string, TaskCompletionSource)>(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); + + // require runners to be setup, wouldnt start otherwise + if (runDirect is null || runShell is null) + throw new ArgumentNullException(nameof(runDirect), "Provide process runners from FileCompactor"); + _runDirect = runDirect; + _runShell = runShell; + _worker = Task.Run(ProcessAsync, _cts.Token); } @@ -92,7 +104,7 @@ namespace LightlessSync.Services.Compression try { - var map = await RunBatchAsync(pending.Select(p => p.path)).ConfigureAwait(false); + var map = RunBatch(pending.Select(p => p.path)); foreach (var (path, tcs) in pending) { tcs.TrySetResult(map.TryGetValue(path, out var c) && c); @@ -124,75 +136,33 @@ namespace LightlessSync.Services.Compression /// Paths that are needed for the command building for the batch return /// Path of the file and if it went correctly /// Failing to start filefrag on the system if this exception is found - private async Task> RunBatchAsync(IEnumerable paths) + private Dictionary RunBatch(IEnumerable paths) { var list = paths.Distinct(StringComparer.Ordinal).ToList(); var result = list.ToDictionary(p => p, _ => false, StringComparer.Ordinal); - ProcessStartInfo psi; + (bool ok, string stdout, string stderr, int code) res; + if (_useShell) { - var inner = "filefrag -v -- " + string.Join(' ', list.Select(QuoteSingle)); - psi = new ProcessStartInfo - { - FileName = "/bin/bash", - Arguments = "-lc " + QuoteDouble(inner), - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; - - if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p)) - psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin"; - else - psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p; + var inner = "filefrag -v " + string.Join(' ', list.Select(QuoteSingle)); + res = _runShell(inner, timeoutMs: 15000, workingDir: "/"); } else { - psi = new ProcessStartInfo - { - FileName = "filefrag", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - - if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p)) - psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin"; - else - psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p; - - psi.ArgumentList.Add("-v"); - psi.ArgumentList.Add("--"); + var args = new List { "-v" }; foreach (var path in list) - psi.ArgumentList.Add(path); + { + args.Add(' ' + path); + } + + res = _runDirect("filefrag", args, workingDir: "/", timeoutMs: 15000); } - using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Failed to start filefrag"); + if (!string.IsNullOrWhiteSpace(res.stderr)) + _log.LogTrace("filefrag stderr (batch): {err}", res.stderr.Trim()); - var outTask = proc.StandardOutput.ReadToEndAsync(_cts.Token); - var errTask = proc.StandardError.ReadToEndAsync(_cts.Token); - - var timeout = TimeSpan.FromSeconds(15); - var combined = Task.WhenAll(outTask, errTask); - var finished = await Task.WhenAny(combined, Task.Delay(timeout, _cts.Token)).ConfigureAwait(false); - - if (finished != combined) - { - try { proc.Kill(entireProcessTree: true); } catch { /* ignore */ } - try { await combined.ConfigureAwait(false); } catch { /* ignore */ } - } - - var stdout = outTask.IsCompletedSuccessfully ? await outTask.ConfigureAwait(false) : ""; - var stderr = errTask.IsCompletedSuccessfully ? await errTask.ConfigureAwait(false) : ""; - - if (!string.IsNullOrWhiteSpace(stderr)) - _log.LogTrace("filefrag stderr (batch): {err}", stderr.Trim()); - - ParseFilefrag(stdout, result); + ParseFilefrag(res.stdout, result); return result; } @@ -225,7 +195,6 @@ namespace LightlessSync.Services.Compression } private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; - private static string QuoteDouble(string s) => "\"" + s.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal).Replace("$", "\\$", StringComparison.Ordinal).Replace("`", "\\`", StringComparison.Ordinal) + "\""; /// /// Regex of the File Size return on the Linux/Wine systems, giving back the amount diff --git a/LightlessSync/UI/SettingsUi.cs b/LightlessSync/UI/SettingsUi.cs index d686a75..0645118 100644 --- a/LightlessSync/UI/SettingsUi.cs +++ b/LightlessSync/UI/SettingsUi.cs @@ -1236,7 +1236,7 @@ public class SettingsUi : WindowMediatorSubscriberBase UIColors.Get("LightlessYellow")); } - if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); + if (!_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); if (ImGui.Checkbox("Use file compactor", ref useFileCompactor)) { _configService.Current.UseCompactor = useFileCompactor; @@ -1281,20 +1281,20 @@ public class SettingsUi : WindowMediatorSubscriberBase UIColors.Get("LightlessYellow")); } - if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) + if (!_cacheMonitor.StorageisNTFS) { ImGui.EndDisabled(); - ImGui.TextUnformatted("The file compactor is only available on BTRFS and NTFS drives."); + ImGui.TextUnformatted("The file compactor is only available NTFS drives, soon for btrfs."); } if (_cacheMonitor.StorageisNTFS) { - ImGui.TextUnformatted("The file compactor is running on NTFS Drive."); + ImGui.TextUnformatted("The file compactor detected an NTFS Drive."); } if (_cacheMonitor.StorageIsBtrfs) { - ImGui.TextUnformatted("The file compactor is running on Btrfs Drive."); + ImGui.TextUnformatted("The file compactor detected an Btrfs Drive."); } ImGuiHelpers.ScaledDummy(new Vector2(10, 10));