diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index e6d67ca..5e0b4a9 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -6,6 +6,7 @@ using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.InteropServices; +using System.Threading; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; @@ -82,7 +83,7 @@ public sealed class FileCompactor : IDisposable _fragBatch = new BatchFilefragService( useShell: _dalamudUtilService.IsWine, log: _logger, - batchSize: 256, + batchSize: 128, flushMs: 25); _logger.LogInformation("FileCompactor started with {workers} workers", workerCount); @@ -198,13 +199,12 @@ public sealed class FileCompactor : IDisposable var (ok1, out1, err1, code1) = isWindowsProc ? RunProcessShell($"stat -c %b -- {QuoteSingle(linuxPath)}", null, 10000) - : RunProcessDirect("stat", new[] { "-c", "%b", "--", linuxPath }, null, 10000); + : RunProcessDirect("stat", ["-c", "%b", "--", linuxPath], null, 10000); if (ok1 && long.TryParse(out1.Trim(), out long blocks)) - return (false, blocks * 512L); // st_blocks are 512B units + return (false, blocks * 512L); // st_blocks are always 512B units - // Fallback: du -B1 (true on-disk bytes) - var (ok2, out2, err2, code2) = RunProcessShell($"du -B1 -- {QuoteSingle(linuxPath)} | cut -f1", null, 10000); // use shell for the pipe + var (ok2, out2, err2, code2) = RunProcessShell($"du -B1 -- {QuoteSingle(linuxPath)} | cut -f1", workingDir: null, 10000); // use shell for the pipe if (ok2 && long.TryParse(out2.Trim(), out long bytes)) return (false, bytes); @@ -425,6 +425,7 @@ public sealed class FileCompactor : IDisposable /// Decompressing state private bool DecompressWOFFile(string path) { + //Check if its already been compressed if (TryIsWofExternal(path, out bool isExternal, out int algo)) { if (!isExternal) @@ -436,6 +437,7 @@ public sealed class FileCompactor : IDisposable _logger.LogTrace("WOF compression (algo={algo}) detected for {file}", compressString, path); } + //This will attempt to start WOF thread. return WithFileHandleForWOF(path, FileAccess.ReadWrite, h => { if (!DeviceIoControl(h, FSCTL_DELETE_EXTERNAL_BACKING, @@ -524,7 +526,7 @@ public sealed class FileCompactor : IDisposable } catch { - /* ignore and fall through */ + /* ignore and fall through the floor! */ } } @@ -550,7 +552,7 @@ public sealed class FileCompactor : IDisposable { int ret = WofSetFileDataLocation(h, WOF_PROVIDER_FILE, efInfoPtr, length); - // 0x80070158 is the benign "already compressed/unsupported" style return + // 0x80070158 is the being "already compressed/unsupported" style return if (ret != 0 && ret != unchecked((int)0x80070158)) { _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); @@ -791,38 +793,12 @@ public sealed class FileCompactor : IDisposable using var proc = Process.Start(psi); if (proc is null) return (false, "", "failed to start process", -1); - var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token); - var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token); - var both = Task.WhenAll(outTask, errTask); - - if (_dalamudUtilService.IsWine) + var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token); + if (!success) { - var finished = Task.WhenAny(both, Task.Delay(timeoutMs, _compactionCts.Token)).GetAwaiter().GetResult(); - if (finished != both) - { - try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ } - try { Task.WaitAll(new[] { outTask, errTask }, 1000, _compactionCts.Token); } catch { /* ignore this */ } - var so = outTask.IsCompleted ? outTask.Result : ""; - var se = errTask.IsCompleted ? errTask.Result : "timeout"; - return (false, so, se, -1); - } - - var stdout = outTask.Result; - var stderr = errTask.Result; - var ok = string.IsNullOrWhiteSpace(stderr); - return (ok, stdout, stderr, ok ? 0 : -1); + return (false, so2, se2, -1); } - if (!proc.WaitForExit(timeoutMs)) - { - try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ } - try { Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); } catch { /* ignore this */ } - return (false, outTask.IsCompleted ? outTask.Result : "", "timeout", -1); - } - - Task.WaitAll(outTask, errTask); - var so2 = outTask.Result; - var se2 = errTask.Result; int code; try { code = proc.ExitCode; } catch { code = -1; } return (code == 0, so2, se2, code); @@ -836,7 +812,7 @@ public sealed class FileCompactor : IDisposable /// State of the process, output of the process and error with exit code private (bool ok, string stdout, string stderr, int exitCode) RunProcessShell(string command, string? workingDir = null, int timeoutMs = 60000) { - // Use a LOGIN shell so PATH includes /usr/sbin etc. + var psi = new ProcessStartInfo("/bin/bash") { RedirectStandardOutput = true, @@ -845,7 +821,7 @@ public sealed class FileCompactor : IDisposable CreateNoWindow = true }; if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir; - + // Use a Login shell so PATH includes /usr/sbin etc. AKA -lc psi.ArgumentList.Add("-lc"); psi.ArgumentList.Add(QuoteDouble(command)); EnsureUnixPathEnv(psi); @@ -853,43 +829,74 @@ public sealed class FileCompactor : IDisposable using var proc = Process.Start(psi); if (proc is null) return (false, "", "failed to start /bin/bash", -1); - var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token); - var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token); - var both = Task.WhenAll(outTask, errTask); - - if (_dalamudUtilService.IsWine) + var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token); + if (!success) { - var finished = Task.WhenAny(both, Task.Delay(timeoutMs, _compactionCts.Token)).GetAwaiter().GetResult(); - if (finished != both) - { - try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ } - try { Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); } catch { /* ignore this */ } - var so = outTask.IsCompleted ? outTask.Result : ""; - var se = errTask.IsCompleted ? errTask.Result : "timeout"; - return (false, so, se, -1); - } - - var stdout = outTask.Result; - var stderr = errTask.Result; - var ok = string.IsNullOrWhiteSpace(stderr); - return (ok, stdout, stderr, ok ? 0 : -1); + return (false, so2, se2, -1); } - if (!proc.WaitForExit(timeoutMs)) - { - try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ } - try { Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); } catch { /* ignore this */ } - return (false, outTask.IsCompleted ? outTask.Result : "", "timeout", -1); - } - - Task.WaitAll(outTask, errTask); - var so2 = outTask.Result; - var se2 = errTask.Result; int code; try { code = proc.ExitCode; } catch { code = -1; } return (code == 0, so2, se2, code); } + /// + /// Checking the process result for shell or direct processes + /// + /// Process + /// How long when timeout is gotten + /// Cancellation Token + /// Multiple variables + private (bool success, string testy, string testi) CheckProcessResult(Process proc, int timeoutMs, CancellationToken token) + { + var outTask = proc.StandardOutput.ReadToEndAsync(token); + var errTask = proc.StandardError.ReadToEndAsync(token); + var bothTasks = Task.WhenAll(outTask, errTask); + + //On wine, we dont wanna use waitforexit as it will be always broken and giving an error. + if (_dalamudUtilService.IsWine) + { + var finished = Task.WhenAny(bothTasks, Task.Delay(timeoutMs, token)).GetAwaiter().GetResult(); + if (finished != bothTasks) + { + try + { + proc.Kill(entireProcessTree: true); + Task.WaitAll([outTask, errTask], 1000, token); + } + catch + { + // ignore this + } + var so = outTask.IsCompleted ? outTask.Result : ""; + var se = errTask.IsCompleted ? errTask.Result : "timeout"; + return (false, so, se); + } + + var stderr = errTask.Result; + var ok = string.IsNullOrWhiteSpace(stderr); + return (ok, outTask.Result, stderr); + } + + // On linux, we can use it as we please + if (!proc.WaitForExit(timeoutMs)) + { + try + { + proc.Kill(entireProcessTree: true); + Task.WaitAll([outTask, errTask], 1000, token); + } + catch + { + // ignore this + } + return (false, outTask.IsCompleted ? outTask.Result : "", "timeout"); + } + + Task.WaitAll(outTask, errTask); + return (true, outTask.Result, errTask.Result); + } + /// /// Enqueues the compaction/decompation of an filepath. /// @@ -991,6 +998,11 @@ public sealed class FileCompactor : IDisposable } } + /// + /// Resolves linux path from wine pathing + /// + /// Windows path given from Wine + /// Linux path to be used in Linux private string ResolveLinuxPathForWine(string windowsPath) { var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(windowsPath)}", null, 5000); @@ -998,6 +1010,10 @@ public sealed class FileCompactor : IDisposable return ToLinuxPathIfWine(windowsPath, isWine: true); } + /// + /// Ensures the Unix pathing to be included into the process + /// + /// Process private static void EnsureUnixPathEnv(ProcessStartInfo psi) { if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p)) @@ -1006,6 +1022,11 @@ public sealed class FileCompactor : IDisposable psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p; } + /// + /// Resolves paths for Btrfs to be used on wine or linux and windows in case + /// + /// Path given t + /// private (string windowsPath, string linuxPath) ResolvePathsForBtrfs(string path) { bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); @@ -1021,13 +1042,19 @@ public sealed class FileCompactor : IDisposable return (path, linux); } - private bool ProbeFileReadableForBtrfs(string windowsPath, string linuxPath) + /// + /// Probes file if its readable to be used + /// + /// Windows path + /// Linux path + /// Succesfully probed or not + private bool ProbeFileReadableForBtrfs(string winePath, string linuxPath) { try { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - using var _ = new FileStream(windowsPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var _ = new FileStream(winePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); } else { @@ -1038,6 +1065,12 @@ public sealed class FileCompactor : IDisposable catch { return false; } } + /// + /// Running functions into the Btrfs Gate/Threading. + /// + /// Type of the function that wants to be run inside Btrfs Gate + /// Body of the function + /// Task private T RunWithBtrfsGate(Func body) { bool acquired = false;