diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index be89c1f..3edf96a 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,6 +1,6 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; -using LightlessSync.Services.Compression; +using LightlessSync.Services.Compactor; using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; @@ -27,6 +27,8 @@ public sealed class FileCompactor : IDisposable private readonly List _workers = []; private readonly SemaphoreSlim _globalGate; + + //Limit btrfs gate on half of threads given to compactor. private static readonly SemaphoreSlim _btrfsGate = new(4, 4); private readonly BatchFilefragService _fragBatch; @@ -82,8 +84,11 @@ public sealed class FileCompactor : IDisposable _fragBatch = new BatchFilefragService( useShell: _dalamudUtilService.IsWine, log: _logger, - batchSize: 128, - flushMs: 25); + batchSize: 64, + flushMs: 25, + runDirect: RunProcessDirect, + runShell: RunProcessShell + ); _logger.LogInformation("FileCompactor started with {workers} workers", workerCount); } @@ -192,23 +197,25 @@ public sealed class FileCompactor : IDisposable { try { - bool isWine = _dalamudUtilService?.IsWine ?? false; - string realPath = isWine ? ToLinuxPathIfWine(fileInfo.FullName, isWine) : fileInfo.FullName; + bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + var (_, linuxPath) = ResolvePathsForBtrfs(fileInfo.FullName); - (bool ok, string stdout, string stderr, int code) = - RunProcessDirect("stat", ["-c", "%b", realPath]); + var (ok, output, err, code) = + isWindowsProc + ? RunProcessShell($"stat -c='%b' {QuoteSingle(linuxPath)}", workingDir: null, 10000) + : RunProcessDirect("stat", ["-c='%b'", linuxPath], workingDir: null, 10000); - if (!ok || !long.TryParse(stdout.Trim(), out var blocks)) - throw new InvalidOperationException($"stat failed (exit {code}): {stderr}"); + if (ok && long.TryParse(output.Trim(), out long blocks)) + return (false, blocks * 512L); // st_blocks are always 512B units - return (flowControl: false, value: blocks * 512L); + _logger.LogDebug("Btrfs size probe failed for {linux} (stat {code}, err {err}). Falling back to Length.", linuxPath, code, err); + return (false, fileInfo.Length); } catch (Exception ex) { - _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); + _logger.LogDebug(ex, "Failed Btrfs size probe for {file}, using Length", fileInfo.FullName); + return (false, fileInfo.Length); } - - return (flowControl: true, value: default); } /// @@ -351,65 +358,73 @@ public sealed class FileCompactor : IDisposable } /// - /// Decompress an BTRFS File + /// Decompress an BTRFS File on Wine/Linux /// /// Path of the compressed file /// Decompressing state private bool DecompressBtrfsFile(string path) { - try + return RunWithBtrfsGate(() => { - _btrfsGate.Wait(_compactionCts.Token); - bool isWine = _dalamudUtilService?.IsWine ?? false; - string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; - - var mountOptions = GetMountOptionsForPath(realPath); - if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) + try { - _logger.LogWarning( - "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " + - "Remount with 'compress=no' before running decompression.", - realPath, mountOptions); - return false; - } + bool isWine = _dalamudUtilService?.IsWine ?? false; + string linuxPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; - if (!IsBtrfsCompressedFile(realPath)) - { - _logger.LogTrace("File {file} is not compressed, skipping decompression.", realPath); + var opts = GetMountOptionsForPath(linuxPath); + bool hasCompress = opts.Contains("compress", StringComparison.OrdinalIgnoreCase); + bool hasCompressForce = opts.Contains("compress-force", StringComparison.OrdinalIgnoreCase); + + if (hasCompressForce) + { + _logger.LogWarning("Cannot safely decompress {file}: mount options contains compress-force ({opts}).", linuxPath, opts); + return false; + } + + if (hasCompress) + { + var setCmd = $"btrfs property set -- {QuoteDouble(linuxPath)} compression none"; + var (okSet, _, errSet, codeSet) = isWine + ? RunProcessShell(setCmd) + : RunProcessDirect("btrfs", ["property", "set", "--", linuxPath, "compression", "none"]); + + if (!okSet) + { + _logger.LogWarning("Failed to set 'compression none' on {file}, please check drive options (exit code is: {code}): {err}", linuxPath, codeSet, errSet); + return false; + } + _logger.LogTrace("Set per-file 'compression none' on {file}", linuxPath); + } + + if (!IsBtrfsCompressedFile(linuxPath)) + { + _logger.LogTrace("{file} is not compressed, skipping decompression completely", linuxPath); + return true; + } + + var (ok, stdout, stderr, code) = isWine + ? RunProcessShell($"btrfs filesystem defragment -- {QuoteDouble(linuxPath)}") + : RunProcessDirect("btrfs", ["filesystem", "defragment", "--", linuxPath]); + + if (!ok) + { + _logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit code is: {code}): {stderr}", + linuxPath, code, stderr); + return false; + } + + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs defragment output for {file}: {out}", linuxPath, stdout.Trim()); + + _logger.LogInformation("Decompressed (rewritten uncompressed) Btrfs file: {file}", linuxPath); return true; } - - if (!ProbeFileReadable(realPath)) - return false; - - (bool ok, string stdout, string stderr, int code) = - isWine - ? RunProcessShell($"btrfs filesystem defragment -- {QuoteSingle(realPath)}") - : RunProcessDirect("btrfs", ["filesystem", "defragment", "--", realPath]); - - if (!ok) + catch (Exception ex) { - _logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit {code}): {stderr}", - realPath, code, stderr); + _logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path); return false; } - - if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); - - _logger.LogInformation("Decompressed (rewritten) Btrfs file: {file}", realPath); - return true; - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path); - return false; - } - finally - { - if (_btrfsGate.CurrentCount < 4) - _btrfsGate.Release(); - } + }); } /// @@ -419,6 +434,7 @@ public sealed class FileCompactor : IDisposable /// Decompressing state private bool DecompressWOFFile(string path) { + //Check if its already been compressed if (TryIsWofExternal(path, out bool isExternal, out int algo)) { if (!isExternal) @@ -430,6 +446,7 @@ public sealed class FileCompactor : IDisposable _logger.LogTrace("WOF compression (algo={algo}) detected for {file}", compressString, path); } + //This will attempt to start WOF thread. return WithFileHandleForWOF(path, FileAccess.ReadWrite, h => { if (!DeviceIoControl(h, FSCTL_DELETE_EXTERNAL_BACKING, @@ -459,23 +476,64 @@ public sealed class FileCompactor : IDisposable /// Path that has to be converted /// Extra check if using the wine enviroment /// Converted path to be used in Linux - private string ToLinuxPathIfWine(string path, bool isWine) + private string ToLinuxPathIfWine(string path, bool isWine, bool preferShell = true) { - if (!IsProbablyWine() && !isWine) + if (!isWine || !IsProbablyWine()) return path; - string linuxPath = path; - if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - linuxPath = "/" + path[3..].Replace('\\', '/'); - else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - linuxPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/'); + if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + return ("/" + path[3..].Replace('\\', '/')).Replace("//", "/", StringComparison.Ordinal); - _logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", linuxPath); - return linuxPath; + if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + { + const string usersPrefix = "C:\\Users\\"; + var p = path.Replace('/', '\\'); + + if (p.StartsWith(usersPrefix, StringComparison.OrdinalIgnoreCase)) + { + int afterUsers = usersPrefix.Length; + int slash = p.IndexOf('\\', afterUsers); + if (slash > 0 && slash + 1 < p.Length) + { + var rel = p[(slash + 1)..].Replace('\\', '/'); + var home = Environment.GetEnvironmentVariable("HOME"); + if (string.IsNullOrEmpty(home)) + { + var linuxUser = Environment.GetEnvironmentVariable("USER") ?? Environment.UserName; + home = "/home/" + linuxUser; + } + return (home!.TrimEnd('/') + "/" + rel).Replace("//", "/", StringComparison.Ordinal); + } + } + + try + { + (bool ok, string stdout, string stderr, int code) = preferShell + ? RunProcessShell($"winepath -u {QuoteSingle(path)}", timeoutMs: 5000, workingDir: "/") + : RunProcessDirect("winepath", ["-u", path], workingDir: "/", timeoutMs: 5000); + + if (ok) + { + var outp = (stdout ?? "").Trim(); + if (!string.IsNullOrEmpty(outp) && outp.StartsWith('/')) + return outp.Replace("//", "/", StringComparison.Ordinal); + } + else + { + _logger.LogTrace("winepath failed for {path} (exit {code}): {err}", path, code, stderr); + } + } + catch (Exception ex) + { + _logger.LogTrace(ex, "winepath invocation failed for {path}", path); + } + } + + return path.Replace('\\', '/').Replace("//", "/", StringComparison.Ordinal); } /// - /// Compress an WOF File + /// Compress an File using the WOF methods (NTFS) /// /// Path of the decompressed/normal file /// Compessing state @@ -493,7 +551,7 @@ public sealed class FileCompactor : IDisposable { int ret = WofSetFileDataLocation(h, WOF_PROVIDER_FILE, efInfoPtr, length); - // 0x80070158 is the benign "already compressed/unsupported" style return + // 0x80070158 is the being "already compressed/unsupported" style return if (ret != 0 && ret != unchecked((int)0x80070158)) { _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); @@ -526,7 +584,7 @@ public sealed class FileCompactor : IDisposable } /// - /// Checks if an File is compacted with WOF compression + /// Checks if an File is compacted with WOF compression (NTFS) /// /// Path of the file /// State of the file @@ -553,7 +611,7 @@ public sealed class FileCompactor : IDisposable } /// - /// Checks if an File is compacted any WOF compression with an WOF backing + /// Checks if an File is compacted any WOF compression with an WOF backing (NTFS) /// /// Path of the file /// State of the file, if its an external (no backing) and which algorithm if detected @@ -589,25 +647,31 @@ public sealed class FileCompactor : IDisposable /// State of the file private bool IsBtrfsCompressedFile(string path) { - try + return RunWithBtrfsGate(() => { - _btrfsGate.Wait(_compactionCts.Token); + try + { + bool windowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + string linuxPath = windowsProc ? ResolveLinuxPathForWine(path) : path; - bool isWine = _dalamudUtilService?.IsWine ?? false; - string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; + var task = _fragBatch.IsCompressedAsync(linuxPath, _compactionCts.Token); - return _fragBatch.IsCompressedAsync(realPath, _compactionCts.Token).GetAwaiter().GetResult(); - } - catch (Exception ex) - { - _logger.LogDebug(ex, "Failed to detect Btrfs compression for {file}", path); - return false; - } - finally - { - if (_btrfsGate.CurrentCount < 4) - _btrfsGate.Release(); - } + if (task.Wait(TimeSpan.FromSeconds(5), _compactionCts.Token) && task.IsCompletedSuccessfully) + return task.Result; + + _logger.LogTrace("filefrag batch timed out for {file}", linuxPath); + return false; + } + catch (OperationCanceledException) + { + return false; + } + catch (Exception ex) + { + _logger.LogDebug(ex, "filefrag batch check failed for {file}", path); + return false; + } + }); } /// @@ -617,85 +681,48 @@ public sealed class FileCompactor : IDisposable /// Compessing state private bool BtrfsCompressFile(string path) { - try - { - bool isWine = _dalamudUtilService?.IsWine ?? false; - string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; - - var fi = new FileInfo(realPath); - - if (fi == null) - { - _logger.LogWarning("Failed to open {file} for compression; skipping", realPath); - return false; - } - - if (IsBtrfsCompressedFile(realPath)) - { - _logger.LogTrace("File {file} already compressed (Btrfs), skipping file", realPath); - return true; - } - - if (!ProbeFileReadable(realPath)) - return false; - - (bool ok, string stdout, string stderr, int code) = - isWine - ? RunProcessShell($"btrfs filesystem defragment -clzo -- {QuoteSingle(realPath)}") - : RunProcessDirect("btrfs", ["filesystem", "defragment", "-clzo", "--", realPath]); - - if (!ok) - { - _logger.LogWarning("btrfs defragment failed for {file} (exit {code}): {stderr}", realPath, code, stderr); - return false; - } - - if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogTrace("btrfs output for {file}: {stdout}", realPath, stdout.Trim()); - - if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); - - _logger.LogInformation("Compressed btrfs file successfully: {file}", realPath); - - return true; - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); - return false; - } - } - - - /// - /// Probe file if its readable for certain amount of tries. - /// - /// Path where the file is located - /// Filestream used for the function - /// State of the filestream opening - private bool ProbeFileReadable(string path) - { - for (int attempt = 0; attempt < _maxRetries; attempt++) - { + return RunWithBtrfsGate(() => { try { - using var _ = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); - return true; - } - catch (IOException ex) - { - if (attempt == _maxRetries - 1) + var (winPath, linuxPath) = ResolvePathsForBtrfs(path); + bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + + if (IsBtrfsCompressedFile(linuxPath)) { - _logger.LogWarning(ex, "File still in use after {attempts} attempts, skipping {file}", _maxRetries, path); + _logger.LogTrace("Already Btrfs compressed: {file} (linux={linux})", winPath, linuxPath); + return true; + } + + if (!ProbeFileReadableForBtrfs(winPath, linuxPath)) + { + _logger.LogTrace("Probe failed; cannot open file for compress: {file} (linux={linux})", winPath, linuxPath); return false; } - int delay = 150 * (attempt + 1); - _logger.LogTrace(ex, "File busy, retrying in {delay}ms for {file}", delay, path); - Thread.Sleep(delay); + + (bool ok, string stdout, string stderr, int code) = + isWindowsProc + ? RunProcessShell($"btrfs filesystem defragment -clzo -- {QuoteSingle(linuxPath)}") + : RunProcessDirect("btrfs", ["filesystem", "defragment", "-clzo", "--", linuxPath]); + + if (!ok) + { + _logger.LogWarning("btrfs defragment failed for {file} (linux={linux}) exit {code}: {stderr}", + winPath, linuxPath, code, stderr); + return false; + } + + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs output for {file}: {out}", winPath, stdout.Trim()); + + _logger.LogInformation("Compressed btrfs file successfully: {file} (linux={linux})", winPath, linuxPath); + return true; } - } - return false; + catch (Exception ex) + { + _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); + return false; + } + }); } /// @@ -742,7 +769,7 @@ public sealed class FileCompactor : IDisposable } /// - /// Runs an nonshell process meant for Linux/Wine enviroments + /// Runs an nonshell process meant for Linux enviroments /// /// File that has to be excuted /// Arguments meant for the file/command @@ -759,32 +786,21 @@ public sealed class FileCompactor : IDisposable CreateNoWindow = true }; if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir; - foreach (var a in args) psi.ArgumentList.Add(a); + EnsureUnixPathEnv(psi); using var proc = Process.Start(psi); if (proc is null) return (false, "", "failed to start process", -1); - var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token); - var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token); - - if (!proc.WaitForExit(timeoutMs)) + var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token); + if (!success) { - try - { - proc.Kill(entireProcessTree: true); - } - catch - { - // Ignore this catch on the dispose - } - - Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); - return (false, outTask.Result, "timeout", -1); + return (false, so2, se2, -1); } - Task.WaitAll(outTask, errTask); - return (proc.ExitCode == 0, outTask.Result, errTask.Result, proc.ExitCode); + int code; + try { code = proc.ExitCode; } catch { code = -1; } + return (code == 0, so2, se2, code); } /// @@ -793,8 +809,9 @@ public sealed class FileCompactor : IDisposable /// Command that has to be excuted /// Timeout timer for the process /// State of the process, output of the process and error with exit code - private (bool ok, string stdout, string stderr, int exitCode) RunProcessShell(string command, int timeoutMs = 60000) + private (bool ok, string stdout, string stderr, int exitCode) RunProcessShell(string command, string? workingDir = null, int timeoutMs = 60000) { + var psi = new ProcessStartInfo("/bin/bash") { RedirectStandardOutput = true, @@ -802,32 +819,82 @@ public sealed class FileCompactor : IDisposable UseShellExecute = false, CreateNoWindow = true }; - psi.ArgumentList.Add("-c"); - psi.ArgumentList.Add(command); + if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir; + + // Use a Login shell so PATH includes /usr/sbin etc. AKA -lc for login shell + psi.ArgumentList.Add("-lc"); + psi.ArgumentList.Add(QuoteDouble(command)); + EnsureUnixPathEnv(psi); using var proc = Process.Start(psi); if (proc is null) return (false, "", "failed to start /bin/bash", -1); - var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token); - var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token); - - if (!proc.WaitForExit(timeoutMs)) + var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token); + if (!success) { - try - { - proc.Kill(entireProcessTree: true); - } - catch + return (false, so2, se2, -1); + } + + int code; + try { code = proc.ExitCode; } catch { code = -1; } + return (code == 0, so2, se2, code); + } + + /// + /// Checking the process result for shell or direct processes + /// + /// Process + /// How long when timeout is gotten + /// Cancellation Token + /// Multiple variables + private (bool success, string testy, string testi) CheckProcessResult(Process proc, int timeoutMs, CancellationToken token) + { + var outTask = proc.StandardOutput.ReadToEndAsync(token); + var errTask = proc.StandardError.ReadToEndAsync(token); + var bothTasks = Task.WhenAll(outTask, errTask); + + //On wine, we dont wanna use waitforexit as it will be always broken and giving an error. + if (_dalamudUtilService.IsWine) + { + var finished = Task.WhenAny(bothTasks, Task.Delay(timeoutMs, token)).GetAwaiter().GetResult(); + if (finished != bothTasks) { - // Ignore this catch on the dispose + try + { + proc.Kill(entireProcessTree: true); + Task.WaitAll([outTask, errTask], 1000, token); + } + catch + { + // ignore this + } + var so = outTask.IsCompleted ? outTask.Result : ""; + var se = errTask.IsCompleted ? errTask.Result : "timeout"; + return (false, so, se); } - Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); - return (false, outTask.Result, "timeout", -1); + var stderr = errTask.Result; + var ok = string.IsNullOrWhiteSpace(stderr); + return (ok, outTask.Result, stderr); + } + + // On linux, we can use it as we please + if (!proc.WaitForExit(timeoutMs)) + { + try + { + proc.Kill(entireProcessTree: true); + Task.WaitAll([outTask, errTask], 1000, token); + } + catch + { + // ignore this + } + return (false, outTask.IsCompleted ? outTask.Result : "", "timeout"); } Task.WaitAll(outTask, errTask); - return (proc.ExitCode == 0, outTask.Result, errTask.Result, proc.ExitCode); + return (true, outTask.Result, errTask.Result); } /// @@ -931,6 +998,93 @@ public sealed class FileCompactor : IDisposable } } + /// + /// Resolves linux path from wine pathing + /// + /// Windows path given from Wine + /// Linux path to be used in Linux + private string ResolveLinuxPathForWine(string windowsPath) + { + var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(windowsPath)}", null, 5000); + if (ok && !string.IsNullOrWhiteSpace(outp)) return outp.Trim(); + return ToLinuxPathIfWine(windowsPath, isWine: true); + } + + /// + /// Ensures the Unix pathing to be included into the process start + /// + /// Process + private static void EnsureUnixPathEnv(ProcessStartInfo psi) + { + if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p)) + psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin"; + else if (!p.Contains("/usr/sbin", StringComparison.Ordinal)) + psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p; + } + + /// + /// Resolves paths for Btrfs to be used on wine or linux and windows in case + /// + /// Path given t + /// + private (string windowsPath, string linuxPath) ResolvePathsForBtrfs(string path) + { + bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + + if (!isWindowsProc) + return (path, path); + + var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(path)}", workingDir: null, 5000); + var linux = (ok && !string.IsNullOrWhiteSpace(outp)) ? outp.Trim() : ToLinuxPathIfWine(path, isWine: true); + + return (path, linux); + } + + /// + /// Probes file if its readable to be used + /// + /// Windows path + /// Linux path + /// Succesfully probed or not + private bool ProbeFileReadableForBtrfs(string winePath, string linuxPath) + { + try + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + using var _ = new FileStream(winePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + } + else + { + using var _ = new FileStream(linuxPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + } + return true; + } + catch { return false; } + } + + /// + /// Running functions into the Btrfs Gate/Threading. + /// + /// Type of the function that wants to be run inside Btrfs Gate + /// Body of the function + /// Task + private T RunWithBtrfsGate(Func body) + { + bool acquired = false; + try + { + _btrfsGate.Wait(_compactionCts.Token); + acquired = true; + return body(); + } + finally + { + if (acquired) _btrfsGate.Release(); + } + } + + [DllImport("kernel32.dll", SetLastError = true)] private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped); @@ -945,6 +1099,8 @@ public sealed class FileCompactor : IDisposable private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; + private static string QuoteDouble(string s) => "\"" + s.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal).Replace("$", "\\$", StringComparison.Ordinal).Replace("`", "\\`", StringComparison.Ordinal) + "\""; + public void Dispose() { _fragBatch?.Dispose(); diff --git a/LightlessSync/Services/Compression/BatchFileFragService.cs b/LightlessSync/Services/Compactor/BatchFileFragService.cs similarity index 75% rename from LightlessSync/Services/Compression/BatchFileFragService.cs rename to LightlessSync/Services/Compactor/BatchFileFragService.cs index ae5bb71..b31919e 100644 --- a/LightlessSync/Services/Compression/BatchFileFragService.cs +++ b/LightlessSync/Services/Compactor/BatchFileFragService.cs @@ -1,9 +1,8 @@ using Microsoft.Extensions.Logging; -using System.Diagnostics; using System.Text.RegularExpressions; using System.Threading.Channels; -namespace LightlessSync.Services.Compression +namespace LightlessSync.Services.Compactor { /// /// This batch service is made for the File Frag command, because of each file needing to use this command. @@ -19,13 +18,26 @@ namespace LightlessSync.Services.Compression private readonly TimeSpan _flushDelay; private readonly CancellationTokenSource _cts = new(); - public BatchFilefragService(bool useShell, ILogger log, int batchSize = 128, int flushMs = 25) + public delegate (bool ok, string stdout, string stderr, int exitCode) RunDirect(string fileName, IEnumerable args, string? workingDir, int timeoutMs); + private readonly RunDirect _runDirect; + + public delegate (bool ok, string stdout, string stderr, int exitCode) RunShell(string command, string? workingDir, int timeoutMs); + private readonly RunShell _runShell; + + public BatchFilefragService(bool useShell, ILogger log, int batchSize = 128, int flushMs = 25, RunDirect? runDirect = null, RunShell? runShell = null) { _useShell = useShell; _log = log; _batchSize = Math.Max(8, batchSize); _flushDelay = TimeSpan.FromMilliseconds(Math.Max(5, flushMs)); _ch = Channel.CreateUnbounded<(string, TaskCompletionSource)>(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); + + // require runners to be setup, wouldnt start otherwise + if (runDirect is null || runShell is null) + throw new ArgumentNullException(nameof(runDirect), "Provide process runners from FileCompactor"); + _runDirect = runDirect; + _runShell = runShell; + _worker = Task.Run(ProcessAsync, _cts.Token); } @@ -92,7 +104,7 @@ namespace LightlessSync.Services.Compression try { - var map = await RunBatchAsync(pending.Select(p => p.path)).ConfigureAwait(false); + var map = RunBatch(pending.Select(p => p.path)); foreach (var (path, tcs) in pending) { tcs.TrySetResult(map.TryGetValue(path, out var c) && c); @@ -124,61 +136,33 @@ namespace LightlessSync.Services.Compression /// Paths that are needed for the command building for the batch return /// Path of the file and if it went correctly /// Failing to start filefrag on the system if this exception is found - private async Task> RunBatchAsync(IEnumerable paths) + private Dictionary RunBatch(IEnumerable paths) { var list = paths.Distinct(StringComparer.Ordinal).ToList(); var result = list.ToDictionary(p => p, _ => false, StringComparer.Ordinal); - ProcessStartInfo psi; + (bool ok, string stdout, string stderr, int code) res; + if (_useShell) { - var inner = "filefrag -v -- " + string.Join(' ', list.Select(QuoteSingle)); - psi = new ProcessStartInfo - { - FileName = "/bin/bash", - Arguments = "-c " + QuoteDouble(inner), - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; + var inner = "filefrag -v " + string.Join(' ', list.Select(QuoteSingle)); + res = _runShell(inner, timeoutMs: 15000, workingDir: "/"); } else { - psi = new ProcessStartInfo + var args = new List { "-v" }; + foreach (var path in list) { - FileName = "filefrag", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - psi.ArgumentList.Add("-v"); - psi.ArgumentList.Add("--"); - foreach (var p in list) psi.ArgumentList.Add(p); + args.Add(' ' + path); + } + + res = _runDirect("filefrag", args, workingDir: "/", timeoutMs: 15000); } - using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Failed to start filefrag"); - var stdoutTask = proc.StandardOutput.ReadToEndAsync(_cts.Token); - var stderrTask = proc.StandardError.ReadToEndAsync(_cts.Token); - await Task.WhenAll(stdoutTask, stderrTask).ConfigureAwait(false); - try - { - await proc.WaitForExitAsync(_cts.Token).ConfigureAwait(false); - } - catch (Exception ex) - { - _log.LogWarning(ex, "Error in the batch frag service. proc = {proc}", proc); - } + if (!string.IsNullOrWhiteSpace(res.stderr)) + _log.LogTrace("filefrag stderr (batch): {err}", res.stderr.Trim()); - var stdout = await stdoutTask.ConfigureAwait(false); - var stderr = await stderrTask.ConfigureAwait(false); - - if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) - _log.LogTrace("filefrag exited {code}: {err}", proc.ExitCode, stderr.Trim()); - - ParseFilefrag(stdout, result); + ParseFilefrag(res.stdout, result); return result; } @@ -211,7 +195,6 @@ namespace LightlessSync.Services.Compression } private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; - private static string QuoteDouble(string s) => "\"" + s.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal).Replace("$", "\\$", StringComparison.Ordinal).Replace("`", "\\`", StringComparison.Ordinal) + "\""; /// /// Regex of the File Size return on the Linux/Wine systems, giving back the amount diff --git a/LightlessSync/UI/SettingsUi.cs b/LightlessSync/UI/SettingsUi.cs index 1b8aa12..7cdeb38 100644 --- a/LightlessSync/UI/SettingsUi.cs +++ b/LightlessSync/UI/SettingsUi.cs @@ -1236,7 +1236,7 @@ public class SettingsUi : WindowMediatorSubscriberBase UIColors.Get("LightlessYellow")); } - if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); + if (!_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); if (ImGui.Checkbox("Use file compactor", ref useFileCompactor)) { _configService.Current.UseCompactor = useFileCompactor; @@ -1281,20 +1281,20 @@ public class SettingsUi : WindowMediatorSubscriberBase UIColors.Get("LightlessYellow")); } - if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) + if (!_cacheMonitor.StorageisNTFS) { ImGui.EndDisabled(); - ImGui.TextUnformatted("The file compactor is only available on BTRFS and NTFS drives."); + ImGui.TextUnformatted("The file compactor is only available NTFS drives, soon for btrfs."); } if (_cacheMonitor.StorageisNTFS) { - ImGui.TextUnformatted("The file compactor is running on NTFS Drive."); + ImGui.TextUnformatted("The file compactor detected an NTFS Drive."); } if (_cacheMonitor.StorageIsBtrfs) { - ImGui.TextUnformatted("The file compactor is running on Btrfs Drive."); + ImGui.TextUnformatted("The file compactor detected an Btrfs Drive."); } ImGuiHelpers.ScaledDummy(new Vector2(10, 10)); diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs index af4c98b..d63b3b9 100644 --- a/LightlessSync/Utils/FileSystemHelper.cs +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -252,14 +252,18 @@ namespace LightlessSync.Utils }; using var proc = Process.Start(psi); - string stdout = proc?.StandardOutput.ReadToEnd().Trim() ?? ""; - proc?.WaitForExit(); - if (int.TryParse(stdout, out int blockSize) && blockSize > 0) + string stdout = proc?.StandardOutput.ReadToEnd().Trim() ?? ""; + string _stderr = proc?.StandardError.ReadToEnd() ?? ""; + + try { proc?.WaitForExit(); } + catch (Exception ex) { logger?.LogTrace(ex, "stat WaitForExit failed under Wine; ignoring"); } + + if (!(!int.TryParse(stdout, out int block) || block <= 0)) { - _blockSizeCache[root] = blockSize; - logger?.LogTrace("Filesystem block size via stat for {root}: {block}", root, blockSize); - return blockSize; + _blockSizeCache[root] = block; + logger?.LogTrace("Filesystem block size via stat for {root}: {block}", root, block); + return block; } logger?.LogTrace("stat did not return valid block size for {file}, output: {out}", fi.FullName, stdout);