Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 51 Wiki Activity

Merge pull request 'Turned off btrfs compression for now as not completed' (#85) from linux-improvements into 1.12.4

Browse Source 
Reviewed-on: #85
Reviewed-by: defnotken <defnotken@noreply.git.lightless-sync.org>
This commit was merged in pull request #85.
This commit is contained in:
cake
2025-11-11 18:41:05 +01:00
parent 325dc8947d e0d1f98c70
commit d4ba1cf437
4 changed files with 397 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

548
LightlessSync/FileCache/FileCompactor.cs
View File

@@ -1,6 +1,6 @@
using LightlessSync.LightlessConfiguration;
using LightlessSync.Services;
using LightlessSync.Services.Compression;
using LightlessSync.Services.Compactor;
using Microsoft.Extensions.Logging;
using Microsoft.Win32.SafeHandles;
using System.Collections.Concurrent;
@@ -27,6 +27,8 @@ public sealed class FileCompactor : IDisposable
private readonly List<Task> _workers = [];
private readonly SemaphoreSlim _globalGate;
//Limit btrfs gate on half of threads given to compactor.
private static readonly SemaphoreSlim _btrfsGate = new(4, 4);
private readonly BatchFilefragService _fragBatch;
@@ -82,8 +84,11 @@ public sealed class FileCompactor : IDisposable
_fragBatch = new BatchFilefragService(
useShell: _dalamudUtilService.IsWine,
log: _logger,
batchSize: 128,
flushMs: 25);
batchSize: 64,
flushMs: 25,
runDirect: RunProcessDirect,
runShell: RunProcessShell
);
_logger.LogInformation("FileCompactor started with {workers} workers", workerCount);
}
@@ -192,23 +197,25 @@ public sealed class FileCompactor : IDisposable
{
try
{
bool isWine = _dalamudUtilService?.IsWine ?? false;
string realPath = isWine ? ToLinuxPathIfWine(fileInfo.FullName, isWine) : fileInfo.FullName;
bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
var (_, linuxPath) = ResolvePathsForBtrfs(fileInfo.FullName);
(bool ok, string stdout, string stderr, int code) =
RunProcessDirect("stat", ["-c", "%b", realPath]);
var (ok, output, err, code) =
isWindowsProc
? RunProcessShell($"stat -c='%b' {QuoteSingle(linuxPath)}", workingDir: null, 10000)
: RunProcessDirect("stat", ["-c='%b'", linuxPath], workingDir: null, 10000);
if (!ok || !long.TryParse(stdout.Trim(), out var blocks))
throw new InvalidOperationException($"stat failed (exit {code}): {stderr}");
if (ok && long.TryParse(output.Trim(), out long blocks))
return (false, blocks * 512L); // st_blocks are always 512B units
return (flowControl: false, value: blocks * 512L);
_logger.LogDebug("Btrfs size probe failed for {linux} (stat {code}, err {err}). Falling back to Length.", linuxPath, code, err);
return (false, fileInfo.Length);
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName);
_logger.LogDebug(ex, "Failed Btrfs size probe for {file}, using Length", fileInfo.FullName);
return (false, fileInfo.Length);
}
return (flowControl: true, value: default);
}
/// <summary>
@@ -351,65 +358,73 @@ public sealed class FileCompactor : IDisposable
}
/// <summary>
/// Decompress an BTRFS File
/// Decompress an BTRFS File on Wine/Linux
/// </summary>
/// <param name="path">Path of the compressed file</param>
/// <returns>Decompressing state</returns>
private bool DecompressBtrfsFile(string path)
{
try
return RunWithBtrfsGate(() =>
{
_btrfsGate.Wait(_compactionCts.Token);
bool isWine = _dalamudUtilService?.IsWine ?? false;
string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path;
var mountOptions = GetMountOptionsForPath(realPath);
if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase))
try
{
_logger.LogWarning(
"Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " +
"Remount with 'compress=no' before running decompression.",
realPath, mountOptions);
return false;
}
bool isWine = _dalamudUtilService?.IsWine ?? false;
string linuxPath = isWine ? ToLinuxPathIfWine(path, isWine) : path;
if (!IsBtrfsCompressedFile(realPath))
{
_logger.LogTrace("File {file} is not compressed, skipping decompression.", realPath);
var opts = GetMountOptionsForPath(linuxPath);
bool hasCompress = opts.Contains("compress", StringComparison.OrdinalIgnoreCase);
bool hasCompressForce = opts.Contains("compress-force", StringComparison.OrdinalIgnoreCase);
if (hasCompressForce)
{
_logger.LogWarning("Cannot safely decompress {file}: mount options contains compress-force ({opts}).", linuxPath, opts);
return false;
}
if (hasCompress)
{
var setCmd = $"btrfs property set -- {QuoteDouble(linuxPath)} compression none";
var (okSet, _, errSet, codeSet) = isWine
? RunProcessShell(setCmd)
: RunProcessDirect("btrfs", ["property", "set", "--", linuxPath, "compression", "none"]);
if (!okSet)
{
_logger.LogWarning("Failed to set 'compression none' on {file}, please check drive options (exit code is: {code}): {err}", linuxPath, codeSet, errSet);
return false;
}
_logger.LogTrace("Set per-file 'compression none' on {file}", linuxPath);
}
if (!IsBtrfsCompressedFile(linuxPath))
{
_logger.LogTrace("{file} is not compressed, skipping decompression completely", linuxPath);
return true;
}
var (ok, stdout, stderr, code) = isWine
? RunProcessShell($"btrfs filesystem defragment -- {QuoteDouble(linuxPath)}")
: RunProcessDirect("btrfs", ["filesystem", "defragment", "--", linuxPath]);
if (!ok)
{
_logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit code is: {code}): {stderr}",
linuxPath, code, stderr);
return false;
}
if (!string.IsNullOrWhiteSpace(stdout))
_logger.LogTrace("btrfs defragment output for {file}: {out}", linuxPath, stdout.Trim());
_logger.LogInformation("Decompressed (rewritten uncompressed) Btrfs file: {file}", linuxPath);
return true;
}
if (!ProbeFileReadable(realPath))
return false;
(bool ok, string stdout, string stderr, int code) =
isWine
? RunProcessShell($"btrfs filesystem defragment -- {QuoteSingle(realPath)}")
: RunProcessDirect("btrfs", ["filesystem", "defragment", "--", realPath]);
if (!ok)
catch (Exception ex)
{
_logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit {code}): {stderr}",
realPath, code, stderr);
_logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path);
return false;
}
if (!string.IsNullOrWhiteSpace(stdout))
_logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim());
_logger.LogInformation("Decompressed (rewritten) Btrfs file: {file}", realPath);
return true;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path);
return false;
}
finally
{
if (_btrfsGate.CurrentCount < 4)
_btrfsGate.Release();
}
});
}
/// <summary>
@@ -419,6 +434,7 @@ public sealed class FileCompactor : IDisposable
/// <returns>Decompressing state</returns>
private bool DecompressWOFFile(string path)
{
//Check if its already been compressed
if (TryIsWofExternal(path, out bool isExternal, out int algo))
{
if (!isExternal)
@@ -430,6 +446,7 @@ public sealed class FileCompactor : IDisposable
_logger.LogTrace("WOF compression (algo={algo}) detected for {file}", compressString, path);
}
//This will attempt to start WOF thread.
return WithFileHandleForWOF(path, FileAccess.ReadWrite, h =>
{
if (!DeviceIoControl(h, FSCTL_DELETE_EXTERNAL_BACKING,
@@ -459,23 +476,64 @@ public sealed class FileCompactor : IDisposable
/// <param name="path">Path that has to be converted</param>
/// <param name="isWine">Extra check if using the wine enviroment</param>
/// <returns>Converted path to be used in Linux</returns>
private string ToLinuxPathIfWine(string path, bool isWine)
private string ToLinuxPathIfWine(string path, bool isWine, bool preferShell = true)
{
if (!IsProbablyWine() && !isWine)
if (!isWine || !IsProbablyWine())
return path;
string linuxPath = path;
if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase))
linuxPath = "/" + path[3..].Replace('\\', '/');
else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase))
linuxPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/');
if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase))
return ("/" + path[3..].Replace('\\', '/')).Replace("//", "/", StringComparison.Ordinal);
_logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", linuxPath);
return linuxPath;
if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase))
{
const string usersPrefix = "C:\\Users\\";
var p = path.Replace('/', '\\');
if (p.StartsWith(usersPrefix, StringComparison.OrdinalIgnoreCase))
{
int afterUsers = usersPrefix.Length;
int slash = p.IndexOf('\\', afterUsers);
if (slash > 0 && slash + 1 < p.Length)
{
var rel = p[(slash + 1)..].Replace('\\', '/');
var home = Environment.GetEnvironmentVariable("HOME");
if (string.IsNullOrEmpty(home))
{
var linuxUser = Environment.GetEnvironmentVariable("USER") ?? Environment.UserName;
home = "/home/" + linuxUser;
}
return (home!.TrimEnd('/') + "/" + rel).Replace("//", "/", StringComparison.Ordinal);
}
}
try
{
(bool ok, string stdout, string stderr, int code) = preferShell
? RunProcessShell($"winepath -u {QuoteSingle(path)}", timeoutMs: 5000, workingDir: "/")
: RunProcessDirect("winepath", ["-u", path], workingDir: "/", timeoutMs: 5000);
if (ok)
{
var outp = (stdout ?? "").Trim();
if (!string.IsNullOrEmpty(outp) && outp.StartsWith('/'))
return outp.Replace("//", "/", StringComparison.Ordinal);
}
else
{
_logger.LogTrace("winepath failed for {path} (exit {code}): {err}", path, code, stderr);
}
}
catch (Exception ex)
{
_logger.LogTrace(ex, "winepath invocation failed for {path}", path);
}
}
return path.Replace('\\', '/').Replace("//", "/", StringComparison.Ordinal);
}
/// <summary>
/// Compress an WOF File
/// Compress an File using the WOF methods (NTFS)
/// </summary>
/// <param name="path">Path of the decompressed/normal file</param>
/// <returns>Compessing state</returns>
@@ -493,7 +551,7 @@ public sealed class FileCompactor : IDisposable
{
int ret = WofSetFileDataLocation(h, WOF_PROVIDER_FILE, efInfoPtr, length);
// 0x80070158 is the benign "already compressed/unsupported" style return
// 0x80070158 is the being "already compressed/unsupported" style return
if (ret != 0 && ret != unchecked((int)0x80070158))
{
_logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X"));
@@ -526,7 +584,7 @@ public sealed class FileCompactor : IDisposable
}
/// <summary>
/// Checks if an File is compacted with WOF compression
/// Checks if an File is compacted with WOF compression (NTFS)
/// </summary>
/// <param name="path">Path of the file</param>
/// <returns>State of the file</returns>
@@ -553,7 +611,7 @@ public sealed class FileCompactor : IDisposable
}
/// <summary>
/// Checks if an File is compacted any WOF compression with an WOF backing
/// Checks if an File is compacted any WOF compression with an WOF backing (NTFS)
/// </summary>
/// <param name="path">Path of the file</param>
/// <returns>State of the file, if its an external (no backing) and which algorithm if detected</returns>
@@ -589,25 +647,31 @@ public sealed class FileCompactor : IDisposable
/// <returns>State of the file</returns>
private bool IsBtrfsCompressedFile(string path)
{
try
return RunWithBtrfsGate(() =>
{
_btrfsGate.Wait(_compactionCts.Token);
try
{
bool windowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
string linuxPath = windowsProc ? ResolveLinuxPathForWine(path) : path;
bool isWine = _dalamudUtilService?.IsWine ?? false;
string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path;
var task = _fragBatch.IsCompressedAsync(linuxPath, _compactionCts.Token);
return _fragBatch.IsCompressedAsync(realPath, _compactionCts.Token).GetAwaiter().GetResult();
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Failed to detect Btrfs compression for {file}", path);
return false;
}
finally
{
if (_btrfsGate.CurrentCount < 4)
_btrfsGate.Release();
}
if (task.Wait(TimeSpan.FromSeconds(5), _compactionCts.Token) && task.IsCompletedSuccessfully)
return task.Result;
_logger.LogTrace("filefrag batch timed out for {file}", linuxPath);
return false;
}
catch (OperationCanceledException)
{
return false;
}
catch (Exception ex)
{
_logger.LogDebug(ex, "filefrag batch check failed for {file}", path);
return false;
}
});
}
/// <summary>
@@ -617,85 +681,48 @@ public sealed class FileCompactor : IDisposable
/// <returns>Compessing state</returns>
private bool BtrfsCompressFile(string path)
{
try
{
bool isWine = _dalamudUtilService?.IsWine ?? false;
string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path;
var fi = new FileInfo(realPath);
if (fi == null)
{
_logger.LogWarning("Failed to open {file} for compression; skipping", realPath);
return false;
}
if (IsBtrfsCompressedFile(realPath))
{
_logger.LogTrace("File {file} already compressed (Btrfs), skipping file", realPath);
return true;
}
if (!ProbeFileReadable(realPath))
return false;
(bool ok, string stdout, string stderr, int code) =
isWine
? RunProcessShell($"btrfs filesystem defragment -clzo -- {QuoteSingle(realPath)}")
: RunProcessDirect("btrfs", ["filesystem", "defragment", "-clzo", "--", realPath]);
if (!ok)
{
_logger.LogWarning("btrfs defragment failed for {file} (exit {code}): {stderr}", realPath, code, stderr);
return false;
}
if (!string.IsNullOrWhiteSpace(stdout))
_logger.LogTrace("btrfs output for {file}: {stdout}", realPath, stdout.Trim());
if (!string.IsNullOrWhiteSpace(stdout))
_logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim());
_logger.LogInformation("Compressed btrfs file successfully: {file}", realPath);
return true;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error running btrfs defragment for {file}", path);
return false;
}
}
/// <summary>
/// Probe file if its readable for certain amount of tries.
/// </summary>
/// <param name="path">Path where the file is located</param>
/// <param name="fs">Filestream used for the function</param>
/// <returns>State of the filestream opening</returns>
private bool ProbeFileReadable(string path)
{
for (int attempt = 0; attempt < _maxRetries; attempt++)
{
return RunWithBtrfsGate(() => {
try
{
using var _ = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
return true;
}
catch (IOException ex)
{
if (attempt == _maxRetries - 1)
var (winPath, linuxPath) = ResolvePathsForBtrfs(path);
bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
if (IsBtrfsCompressedFile(linuxPath))
{
_logger.LogWarning(ex, "File still in use after {attempts} attempts, skipping {file}", _maxRetries, path);
_logger.LogTrace("Already Btrfs compressed: {file} (linux={linux})", winPath, linuxPath);
return true;
}
if (!ProbeFileReadableForBtrfs(winPath, linuxPath))
{
_logger.LogTrace("Probe failed; cannot open file for compress: {file} (linux={linux})", winPath, linuxPath);
return false;
}
int delay = 150 * (attempt + 1);
_logger.LogTrace(ex, "File busy, retrying in {delay}ms for {file}", delay, path);
Thread.Sleep(delay);
(bool ok, string stdout, string stderr, int code) =
isWindowsProc
? RunProcessShell($"btrfs filesystem defragment -clzo -- {QuoteSingle(linuxPath)}")
: RunProcessDirect("btrfs", ["filesystem", "defragment", "-clzo", "--", linuxPath]);
if (!ok)
{
_logger.LogWarning("btrfs defragment failed for {file} (linux={linux}) exit {code}: {stderr}",
winPath, linuxPath, code, stderr);
return false;
}
if (!string.IsNullOrWhiteSpace(stdout))
_logger.LogTrace("btrfs output for {file}: {out}", winPath, stdout.Trim());
_logger.LogInformation("Compressed btrfs file successfully: {file} (linux={linux})", winPath, linuxPath);
return true;
}
}
return false;
catch (Exception ex)
{
_logger.LogWarning(ex, "Error running btrfs defragment for {file}", path);
return false;
}
});
}
/// <summary>
@@ -742,7 +769,7 @@ public sealed class FileCompactor : IDisposable
}
/// <summary>
/// Runs an nonshell process meant for Linux/Wine enviroments
/// Runs an nonshell process meant for Linux enviroments
/// </summary>
/// <param name="fileName">File that has to be excuted</param>
/// <param name="args">Arguments meant for the file/command</param>
@@ -759,32 +786,21 @@ public sealed class FileCompactor : IDisposable
CreateNoWindow = true
};
if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir;
foreach (var a in args) psi.ArgumentList.Add(a);
EnsureUnixPathEnv(psi);
using var proc = Process.Start(psi);
if (proc is null) return (false, "", "failed to start process", -1);
var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token);
var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token);
if (!proc.WaitForExit(timeoutMs))
var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token);
if (!success)
{
try
{
proc.Kill(entireProcessTree: true);
}
catch
{
// Ignore this catch on the dispose
}
Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token);
return (false, outTask.Result, "timeout", -1);
return (false, so2, se2, -1);
}
Task.WaitAll(outTask, errTask);
return (proc.ExitCode == 0, outTask.Result, errTask.Result, proc.ExitCode);
int code;
try { code = proc.ExitCode; } catch { code = -1; }
return (code == 0, so2, se2, code);
}
/// <summary>
@@ -793,8 +809,9 @@ public sealed class FileCompactor : IDisposable
/// <param name="command">Command that has to be excuted</param>
/// <param name="timeoutMs">Timeout timer for the process</param>
/// <returns>State of the process, output of the process and error with exit code</returns>
private (bool ok, string stdout, string stderr, int exitCode) RunProcessShell(string command, int timeoutMs = 60000)
private (bool ok, string stdout, string stderr, int exitCode) RunProcessShell(string command, string? workingDir = null, int timeoutMs = 60000)
{
var psi = new ProcessStartInfo("/bin/bash")
{
RedirectStandardOutput = true,
@@ -802,32 +819,82 @@ public sealed class FileCompactor : IDisposable
UseShellExecute = false,
CreateNoWindow = true
};
psi.ArgumentList.Add("-c");
psi.ArgumentList.Add(command);
if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir;
// Use a Login shell so PATH includes /usr/sbin etc. AKA -lc for login shell
psi.ArgumentList.Add("-lc");
psi.ArgumentList.Add(QuoteDouble(command));
EnsureUnixPathEnv(psi);
using var proc = Process.Start(psi);
if (proc is null) return (false, "", "failed to start /bin/bash", -1);
var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token);
var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token);
if (!proc.WaitForExit(timeoutMs))
var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token);
if (!success)
{
try
{
proc.Kill(entireProcessTree: true);
}
catch
return (false, so2, se2, -1);
}
int code;
try { code = proc.ExitCode; } catch { code = -1; }
return (code == 0, so2, se2, code);
}
/// <summary>
/// Checking the process result for shell or direct processes
/// </summary>
/// <param name="proc">Process</param>
/// <param name="timeoutMs">How long when timeout is gotten</param>
/// <param name="token">Cancellation Token</param>
/// <returns>Multiple variables</returns>
private (bool success, string testy, string testi) CheckProcessResult(Process proc, int timeoutMs, CancellationToken token)
{
var outTask = proc.StandardOutput.ReadToEndAsync(token);
var errTask = proc.StandardError.ReadToEndAsync(token);
var bothTasks = Task.WhenAll(outTask, errTask);
//On wine, we dont wanna use waitforexit as it will be always broken and giving an error.
if (_dalamudUtilService.IsWine)
{
var finished = Task.WhenAny(bothTasks, Task.Delay(timeoutMs, token)).GetAwaiter().GetResult();
if (finished != bothTasks)
{
// Ignore this catch on the dispose
try
{
proc.Kill(entireProcessTree: true);
Task.WaitAll([outTask, errTask], 1000, token);
}
catch
{
// ignore this
}
var so = outTask.IsCompleted ? outTask.Result : "";
var se = errTask.IsCompleted ? errTask.Result : "timeout";
return (false, so, se);
}
Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token);
return (false, outTask.Result, "timeout", -1);
var stderr = errTask.Result;
var ok = string.IsNullOrWhiteSpace(stderr);
return (ok, outTask.Result, stderr);
}
// On linux, we can use it as we please
if (!proc.WaitForExit(timeoutMs))
{
try
{
proc.Kill(entireProcessTree: true);
Task.WaitAll([outTask, errTask], 1000, token);
}
catch
{
// ignore this
}
return (false, outTask.IsCompleted ? outTask.Result : "", "timeout");
}
Task.WaitAll(outTask, errTask);
return (proc.ExitCode == 0, outTask.Result, errTask.Result, proc.ExitCode);
return (true, outTask.Result, errTask.Result);
}
/// <summary>
@@ -931,6 +998,93 @@ public sealed class FileCompactor : IDisposable
}
}
/// <summary>
/// Resolves linux path from wine pathing
/// </summary>
/// <param name="windowsPath">Windows path given from Wine</param>
/// <returns>Linux path to be used in Linux</returns>
private string ResolveLinuxPathForWine(string windowsPath)
{
var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(windowsPath)}", null, 5000);
if (ok && !string.IsNullOrWhiteSpace(outp)) return outp.Trim();
return ToLinuxPathIfWine(windowsPath, isWine: true);
}
/// <summary>
/// Ensures the Unix pathing to be included into the process start
/// </summary>
/// <param name="psi">Process</param>
private static void EnsureUnixPathEnv(ProcessStartInfo psi)
{
if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p))
psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin";
else if (!p.Contains("/usr/sbin", StringComparison.Ordinal))
psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p;
}
/// <summary>
/// Resolves paths for Btrfs to be used on wine or linux and windows in case
/// </summary>
/// <param name="path">Path given t</param>
/// <returns></returns>
private (string windowsPath, string linuxPath) ResolvePathsForBtrfs(string path)
{
bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
if (!isWindowsProc)
return (path, path);
var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(path)}", workingDir: null, 5000);
var linux = (ok && !string.IsNullOrWhiteSpace(outp)) ? outp.Trim() : ToLinuxPathIfWine(path, isWine: true);
return (path, linux);
}
/// <summary>
/// Probes file if its readable to be used
/// </summary>
/// <param name="winePath">Windows path</param>
/// <param name="linuxPath">Linux path</param>
/// <returns>Succesfully probed or not</returns>
private bool ProbeFileReadableForBtrfs(string winePath, string linuxPath)
{
try
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
using var _ = new FileStream(winePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
}
else
{
using var _ = new FileStream(linuxPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
}
return true;
}
catch { return false; }
}
/// <summary>
/// Running functions into the Btrfs Gate/Threading.
/// </summary>
/// <typeparam name="T">Type of the function that wants to be run inside Btrfs Gate</typeparam>
/// <param name="body">Body of the function</param>
/// <returns>Task</returns>
private T RunWithBtrfsGate<T>(Func<T> body)
{
bool acquired = false;
try
{
_btrfsGate.Wait(_compactionCts.Token);
acquired = true;
return body();
}
finally
{
if (acquired) _btrfsGate.Release();
}
}
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped);
@@ -945,6 +1099,8 @@ public sealed class FileCompactor : IDisposable
private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'";
private static string QuoteDouble(string s) => "\"" + s.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal).Replace("$", "\\$", StringComparison.Ordinal).Replace("`", "\\`", StringComparison.Ordinal) + "\"";
public void Dispose()
{
_fragBatch?.Dispose();

77
LightlessSync/Services/Compression/BatchFileFragService.cs → LightlessSync/Services/Compactor/BatchFileFragService.cs
View File

@@ -1,9 +1,8 @@
using Microsoft.Extensions.Logging;
using System.Diagnostics;
using System.Text.RegularExpressions;
using System.Threading.Channels;
namespace LightlessSync.Services.Compression
namespace LightlessSync.Services.Compactor
{
/// <summary>
/// This batch service is made for the File Frag command, because of each file needing to use this command.
@@ -19,13 +18,26 @@ namespace LightlessSync.Services.Compression
private readonly TimeSpan _flushDelay;
private readonly CancellationTokenSource _cts = new();
public BatchFilefragService(bool useShell, ILogger log, int batchSize = 128, int flushMs = 25)
public delegate (bool ok, string stdout, string stderr, int exitCode) RunDirect(string fileName, IEnumerable<string> args, string? workingDir, int timeoutMs);
private readonly RunDirect _runDirect;
public delegate (bool ok, string stdout, string stderr, int exitCode) RunShell(string command, string? workingDir, int timeoutMs);
private readonly RunShell _runShell;
public BatchFilefragService(bool useShell, ILogger log, int batchSize = 128, int flushMs = 25, RunDirect? runDirect = null, RunShell? runShell = null)
{
_useShell = useShell;
_log = log;
_batchSize = Math.Max(8, batchSize);
_flushDelay = TimeSpan.FromMilliseconds(Math.Max(5, flushMs));
_ch = Channel.CreateUnbounded<(string, TaskCompletionSource<bool>)>(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false });
// require runners to be setup, wouldnt start otherwise
if (runDirect is null || runShell is null)
throw new ArgumentNullException(nameof(runDirect), "Provide process runners from FileCompactor");
_runDirect = runDirect;
_runShell = runShell;
_worker = Task.Run(ProcessAsync, _cts.Token);
}
@@ -92,7 +104,7 @@ namespace LightlessSync.Services.Compression
try
{
var map = await RunBatchAsync(pending.Select(p => p.path)).ConfigureAwait(false);
var map = RunBatch(pending.Select(p => p.path));
foreach (var (path, tcs) in pending)
{
tcs.TrySetResult(map.TryGetValue(path, out var c) && c);
@@ -124,61 +136,33 @@ namespace LightlessSync.Services.Compression
/// <param name="paths">Paths that are needed for the command building for the batch return</param>
/// <returns>Path of the file and if it went correctly</returns>
/// <exception cref="InvalidOperationException">Failing to start filefrag on the system if this exception is found</exception>
private async Task<Dictionary<string, bool>> RunBatchAsync(IEnumerable<string> paths)
private Dictionary<string, bool> RunBatch(IEnumerable<string> paths)
{
var list = paths.Distinct(StringComparer.Ordinal).ToList();
var result = list.ToDictionary(p => p, _ => false, StringComparer.Ordinal);
ProcessStartInfo psi;
(bool ok, string stdout, string stderr, int code) res;
if (_useShell)
{
var inner = "filefrag -v -- " + string.Join(' ', list.Select(QuoteSingle));
psi = new ProcessStartInfo
{
FileName = "/bin/bash",
Arguments = "-c " + QuoteDouble(inner),
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true,
WorkingDirectory = "/"
};
var inner = "filefrag -v " + string.Join(' ', list.Select(QuoteSingle));
res = _runShell(inner, timeoutMs: 15000, workingDir: "/");
}
else
{
psi = new ProcessStartInfo
var args = new List<string> { "-v" };
foreach (var path in list)
{
FileName = "filefrag",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
psi.ArgumentList.Add("-v");
psi.ArgumentList.Add("--");
foreach (var p in list) psi.ArgumentList.Add(p);
args.Add(' ' + path);
}
res = _runDirect("filefrag", args, workingDir: "/", timeoutMs: 15000);
}
using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Failed to start filefrag");
var stdoutTask = proc.StandardOutput.ReadToEndAsync(_cts.Token);
var stderrTask = proc.StandardError.ReadToEndAsync(_cts.Token);
await Task.WhenAll(stdoutTask, stderrTask).ConfigureAwait(false);
try
{
await proc.WaitForExitAsync(_cts.Token).ConfigureAwait(false);
}
catch (Exception ex)
{
_log.LogWarning(ex, "Error in the batch frag service. proc = {proc}", proc);
}
if (!string.IsNullOrWhiteSpace(res.stderr))
_log.LogTrace("filefrag stderr (batch): {err}", res.stderr.Trim());
var stdout = await stdoutTask.ConfigureAwait(false);
var stderr = await stderrTask.ConfigureAwait(false);
if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr))
_log.LogTrace("filefrag exited {code}: {err}", proc.ExitCode, stderr.Trim());
ParseFilefrag(stdout, result);
ParseFilefrag(res.stdout, result);
return result;
}
@@ -211,7 +195,6 @@ namespace LightlessSync.Services.Compression
}
private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'";
private static string QuoteDouble(string s) => "\"" + s.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal).Replace("$", "\\$", StringComparison.Ordinal).Replace("`", "\\`", StringComparison.Ordinal) + "\"";
/// <summary>
/// Regex of the File Size return on the Linux/Wine systems, giving back the amount

10
LightlessSync/UI/SettingsUi.cs
View File

@@ -1236,7 +1236,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
UIColors.Get("LightlessYellow"));
}
if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled();
if (!_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled();
if (ImGui.Checkbox("Use file compactor", ref useFileCompactor))
{
_configService.Current.UseCompactor = useFileCompactor;
@@ -1281,20 +1281,20 @@ public class SettingsUi : WindowMediatorSubscriberBase
UIColors.Get("LightlessYellow"));
}
if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS)
if (!_cacheMonitor.StorageisNTFS)
{
ImGui.EndDisabled();
ImGui.TextUnformatted("The file compactor is only available on BTRFS and NTFS drives.");
ImGui.TextUnformatted("The file compactor is only available NTFS drives, soon for btrfs.");
}
if (_cacheMonitor.StorageisNTFS)
{
ImGui.TextUnformatted("The file compactor is running on NTFS Drive.");
ImGui.TextUnformatted("The file compactor detected an NTFS Drive.");
}
if (_cacheMonitor.StorageIsBtrfs)
{
ImGui.TextUnformatted("The file compactor is running on Btrfs Drive.");
ImGui.TextUnformatted("The file compactor detected an Btrfs Drive.");
}
ImGuiHelpers.ScaledDummy(new Vector2(10, 10));

16
LightlessSync/Utils/FileSystemHelper.cs
View File

@@ -252,14 +252,18 @@ namespace LightlessSync.Utils
};
using var proc = Process.Start(psi);
string stdout = proc?.StandardOutput.ReadToEnd().Trim() ?? "";
proc?.WaitForExit();
if (int.TryParse(stdout, out int blockSize) && blockSize > 0)
string stdout = proc?.StandardOutput.ReadToEnd().Trim() ?? "";
string _stderr = proc?.StandardError.ReadToEnd() ?? "";
try { proc?.WaitForExit(); }
catch (Exception ex) { logger?.LogTrace(ex, "stat WaitForExit failed under Wine; ignoring"); }
if (!(!int.TryParse(stdout, out int block) || block <= 0))
{
_blockSizeCache[root] = blockSize;
logger?.LogTrace("Filesystem block size via stat for {root}: {block}", root, blockSize);
return blockSize;
_blockSizeCache[root] = block;
logger?.LogTrace("Filesystem block size via stat for {root}: {block}", root, block);
return block;
}
logger?.LogTrace("stat did not return valid block size for {file}, output: {out}", fi.FullName, stdout);