Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 51 Wiki Activity

2.0.0 #92

Merged
defnotken merged 171 commits from 2.0.0 into master 2025-12-21 17:19:36 +00:00
Conversation 0 Commits 171 Files Changed 191 +106 -139
3 changed files with 106 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 1862689b1b - Show all commits

144
LightlessSync/FileCache/FileCompactor.cs
View File

@@ -1,12 +1,11 @@
using LightlessSync.LightlessConfiguration;
using LightlessSync.Services;
using LightlessSync.Services.Compression;
using LightlessSync.Services.Compactor;
using Microsoft.Extensions.Logging;
using Microsoft.Win32.SafeHandles;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Channels;
using static LightlessSync.Utils.FileSystemHelper;
@@ -28,6 +27,8 @@ public sealed class FileCompactor : IDisposable
private readonly List<Task> _workers = [];
private readonly SemaphoreSlim _globalGate;
//Limit btrfs gate on half of threads given to compactor.
private static readonly SemaphoreSlim _btrfsGate = new(4, 4);
private readonly BatchFilefragService _fragBatch;
@@ -83,8 +84,11 @@ public sealed class FileCompactor : IDisposable
_fragBatch = new BatchFilefragService(
useShell: _dalamudUtilService.IsWine,
log: _logger,
batchSize: 128,
flushMs: 25);
batchSize: 64,
flushMs: 25,
runDirect: RunProcessDirect,
runShell: RunProcessShell
);
_logger.LogInformation("FileCompactor started with {workers} workers", workerCount);
}
@@ -196,21 +200,15 @@ public sealed class FileCompactor : IDisposable
bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
var (_, linuxPath) = ResolvePathsForBtrfs(fileInfo.FullName);
var (ok1, out1, err1, code1) =
var (ok, output, err, code) =
isWindowsProc
? RunProcessShell($"stat -c %b -- {QuoteSingle(linuxPath)}", null, 10000)
: RunProcessDirect("stat", ["-c", "%b", "--", linuxPath], null, 10000);
? RunProcessShell($"stat -c='%b' {QuoteSingle(linuxPath)}", workingDir: null, 10000)
: RunProcessDirect("stat", ["-c='%b'", linuxPath], workingDir: null, 10000);
if (ok1 && long.TryParse(out1.Trim(), out long blocks))
if (ok && long.TryParse(output.Trim(), out long blocks))
return (false, blocks * 512L); // st_blocks are always 512B units
var (ok2, out2, err2, code2) = RunProcessShell($"du -B1 -- {QuoteSingle(linuxPath)} | cut -f1", workingDir: null, 10000); // use shell for the pipe
if (ok2 && long.TryParse(out2.Trim(), out long bytes))
return (false, bytes);
_logger.LogDebug("Btrfs size probe failed for {linux} (stat {code1}, du {code2}). Falling back to Length.",
linuxPath, code1, code2);
_logger.LogDebug("Btrfs size probe failed for {linux} (stat {code}, err {err}). Falling back to Length.", linuxPath, code, err);
return (false, fileInfo.Length);
}
catch (Exception ex)
@@ -360,7 +358,7 @@ public sealed class FileCompactor : IDisposable
}
/// <summary>
/// Decompress an BTRFS File
/// Decompress an BTRFS File on Wine/Linux
/// </summary>
/// <param name="path">Path of the compressed file</param>
/// <returns>Decompressing state</returns>
@@ -370,44 +368,55 @@ public sealed class FileCompactor : IDisposable
{
try
{
var (winPath, linuxPath) = ResolvePathsForBtrfs(path);
bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
bool isWine = _dalamudUtilService?.IsWine ?? false;
string linuxPath = isWine ? ToLinuxPathIfWine(path, isWine) : path;
var opts = GetMountOptionsForPath(linuxPath);
if (opts.Contains("compress", StringComparison.OrdinalIgnoreCase))
bool hasCompress = opts.Contains("compress", StringComparison.OrdinalIgnoreCase);
bool hasCompressForce = opts.Contains("compress-force", StringComparison.OrdinalIgnoreCase);
if (hasCompressForce)
{
_logger.LogWarning(
"Cannot safely decompress {file}: filesystem mounted with compression ({opts}). Remount with 'compress=no'.",
linuxPath, opts);
_logger.LogWarning("Cannot safely decompress {file}: mount options contains compress-force ({opts}).", linuxPath, opts);
return false;
}
if (hasCompress)
{
var setCmd = $"btrfs property set -- {QuoteDouble(linuxPath)} compression none";
var (okSet, _, errSet, codeSet) = isWine
? RunProcessShell(setCmd)
: RunProcessDirect("btrfs", ["property", "set", "--", linuxPath, "compression", "none"]);
if (!okSet)
{
_logger.LogWarning("Failed to set 'compression none' on {file}, please check drive options (exit code is: {code}): {err}", linuxPath, codeSet, errSet);
return false;
}
_logger.LogTrace("Set per-file 'compression none' on {file}", linuxPath);
}
if (!IsBtrfsCompressedFile(linuxPath))
{
_logger.LogTrace("Btrfs: not compressed, skip {file}", linuxPath);
_logger.LogTrace("{file} is not compressed, skipping decompression completely", linuxPath);
return true;
}
if (!ProbeFileReadableForBtrfs(winPath, linuxPath))
return false;
// Rewrite file uncompressed
(bool ok, string stdout, string stderr, int code) =
isWindowsProc
? RunProcessShell($"btrfs filesystem defragment -- {QuoteSingle(linuxPath)}")
: RunProcessDirect("btrfs", ["filesystem", "defragment", "--", linuxPath]);
var (ok, stdout, stderr, code) = isWine
? RunProcessShell($"btrfs filesystem defragment -- {QuoteDouble(linuxPath)}")
: RunProcessDirect("btrfs", ["filesystem", "defragment", "--", linuxPath]);
if (!ok)
{
_logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit {code}): {stderr}",
_logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit code is: {code}): {stderr}",
linuxPath, code, stderr);
return false;
}
if (!string.IsNullOrWhiteSpace(stdout))
_logger.LogTrace("btrfs (decompress) output {file}: {out}", linuxPath, stdout.Trim());
_logger.LogTrace("btrfs defragment output for {file}: {out}", linuxPath, stdout.Trim());
_logger.LogInformation("Decompressed (rewritten) Btrfs file: {file}", linuxPath);
_logger.LogInformation("Decompressed (rewritten uncompressed) Btrfs file: {file}", linuxPath);
return true;
}
catch (Exception ex)
@@ -467,19 +476,19 @@ public sealed class FileCompactor : IDisposable
/// <param name="path">Path that has to be converted</param>
/// <param name="isWine">Extra check if using the wine enviroment</param>
/// <returns>Converted path to be used in Linux</returns>
private static string ToLinuxPathIfWine(string path, bool isWine)
private string ToLinuxPathIfWine(string path, bool isWine, bool preferShell = true)
{
if (!isWine || !IsProbablyWine())
return path;
if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase))
return "/" + path[3..].Replace('\\', '/');
return ("/" + path[3..].Replace('\\', '/')).Replace("//", "/", StringComparison.Ordinal);
if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase))
{
const string usersPrefix = "C:\\Users\\";
var p = path.Replace('/', '\\');
const string usersPrefix = "C:\\Users\\";
if (p.StartsWith(usersPrefix, StringComparison.OrdinalIgnoreCase))
{
int afterUsers = usersPrefix.Length;
@@ -493,48 +502,38 @@ public sealed class FileCompactor : IDisposable
var linuxUser = Environment.GetEnvironmentVariable("USER") ?? Environment.UserName;
home = "/home/" + linuxUser;
}
// Join as Unix path
return (home.TrimEnd('/') + "/" + rel).Replace("//", "/", StringComparison.Ordinal);
return (home!.TrimEnd('/') + "/" + rel).Replace("//", "/", StringComparison.Ordinal);
}
}
try
{
var inner = "winepath -u " + "'" + path.Replace("'", "'\\''", StringComparison.Ordinal) + "'";
var psi = new ProcessStartInfo
(bool ok, string stdout, string stderr, int code) = preferShell
? RunProcessShell($"winepath -u {QuoteSingle(path)}", timeoutMs: 5000, workingDir: "/")
: RunProcessDirect("winepath", ["-u", path], workingDir: "/", timeoutMs: 5000);
if (ok)
{
FileName = "/bin/bash",
Arguments = "-c " + "\"" + inner.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal) + "\"",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true,
WorkingDirectory = "/"
};
using var proc = Process.Start(psi);
var outp = proc?.StandardOutput.ReadToEnd().Trim();
try
{
proc?.WaitForExit();
}
catch
{
/* Wine can throw here; ignore */
var outp = (stdout ?? "").Trim();
if (!string.IsNullOrEmpty(outp) && outp.StartsWith('/'))
return outp.Replace("//", "/", StringComparison.Ordinal);
}
else
{
_logger.LogTrace("winepath failed for {path} (exit {code}): {err}", path, code, stderr);
}
if (!string.IsNullOrEmpty(outp) && outp.StartsWith("/", StringComparison.Ordinal))
return outp;
}
catch
catch (Exception ex)
{
/* ignore and fall through the floor! */
_logger.LogTrace(ex, "winepath invocation failed for {path}", path);
}
}
return path.Replace('\\', '/');
return path.Replace('\\', '/').Replace("//", "/", StringComparison.Ordinal);
}
/// <summary>
/// Compress an WOF File
/// Compress an File using the WOF methods (NTFS)
/// </summary>
/// <param name="path">Path of the decompressed/normal file</param>
/// <returns>Compessing state</returns>
@@ -585,7 +584,7 @@ public sealed class FileCompactor : IDisposable
}
/// <summary>
/// Checks if an File is compacted with WOF compression
/// Checks if an File is compacted with WOF compression (NTFS)
/// </summary>
/// <param name="path">Path of the file</param>
/// <returns>State of the file</returns>
@@ -612,7 +611,7 @@ public sealed class FileCompactor : IDisposable
}
/// <summary>
/// Checks if an File is compacted any WOF compression with an WOF backing
/// Checks if an File is compacted any WOF compression with an WOF backing (NTFS)
/// </summary>
/// <param name="path">Path of the file</param>
/// <returns>State of the file, if its an external (no backing) and which algorithm if detected</returns>
@@ -821,7 +820,8 @@ public sealed class FileCompactor : IDisposable
CreateNoWindow = true
};
if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir;
// Use a Login shell so PATH includes /usr/sbin etc. AKA -lc
// Use a Login shell so PATH includes /usr/sbin etc. AKA -lc for login shell
psi.ArgumentList.Add("-lc");
psi.ArgumentList.Add(QuoteDouble(command));
EnsureUnixPathEnv(psi);
@@ -1011,7 +1011,7 @@ public sealed class FileCompactor : IDisposable
}
/// <summary>
/// Ensures the Unix pathing to be included into the process
/// Ensures the Unix pathing to be included into the process start
/// </summary>
/// <param name="psi">Process</param>
private static void EnsureUnixPathEnv(ProcessStartInfo psi)
@@ -1034,10 +1034,8 @@ public sealed class FileCompactor : IDisposable
if (!isWindowsProc)
return (path, path);
// Prefer winepath -u; fall back to your existing mapper
var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(path)}", null, 5000);
var linux = (ok && !string.IsNullOrWhiteSpace(outp)) ? outp.Trim()
: ToLinuxPathIfWine(path, isWine: true);
var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(path)}", workingDir: null, 5000);
var linux = (ok && !string.IsNullOrWhiteSpace(outp)) ? outp.Trim() : ToLinuxPathIfWine(path, isWine: true);
return (path, linux);
}

91
LightlessSync/Services/Compression/BatchFileFragService.cs → LightlessSync/Services/Compactor/BatchFileFragService.cs
View File

@@ -1,9 +1,8 @@
using Microsoft.Extensions.Logging;
using System.Diagnostics;
using System.Text.RegularExpressions;
using System.Threading.Channels;
namespace LightlessSync.Services.Compression
namespace LightlessSync.Services.Compactor
{
/// <summary>
/// This batch service is made for the File Frag command, because of each file needing to use this command.
@@ -19,13 +18,26 @@ namespace LightlessSync.Services.Compression
private readonly TimeSpan _flushDelay;
private readonly CancellationTokenSource _cts = new();
public BatchFilefragService(bool useShell, ILogger log, int batchSize = 128, int flushMs = 25)
public delegate (bool ok, string stdout, string stderr, int exitCode) RunDirect(string fileName, IEnumerable<string> args, string? workingDir, int timeoutMs);
private readonly RunDirect _runDirect;
public delegate (bool ok, string stdout, string stderr, int exitCode) RunShell(string command, string? workingDir, int timeoutMs);
private readonly RunShell _runShell;
public BatchFilefragService(bool useShell, ILogger log, int batchSize = 128, int flushMs = 25, RunDirect? runDirect = null, RunShell? runShell = null)
{
_useShell = useShell;
_log = log;
_batchSize = Math.Max(8, batchSize);
_flushDelay = TimeSpan.FromMilliseconds(Math.Max(5, flushMs));
_ch = Channel.CreateUnbounded<(string, TaskCompletionSource<bool>)>(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false });
// require runners to be setup, wouldnt start otherwise
if (runDirect is null || runShell is null)
throw new ArgumentNullException(nameof(runDirect), "Provide process runners from FileCompactor");
_runDirect = runDirect;
_runShell = runShell;
_worker = Task.Run(ProcessAsync, _cts.Token);
}
@@ -92,7 +104,7 @@ namespace LightlessSync.Services.Compression
try
{
var map = await RunBatchAsync(pending.Select(p => p.path)).ConfigureAwait(false);
var map = RunBatch(pending.Select(p => p.path));
foreach (var (path, tcs) in pending)
{
tcs.TrySetResult(map.TryGetValue(path, out var c) && c);
@@ -124,75 +136,33 @@ namespace LightlessSync.Services.Compression
/// <param name="paths">Paths that are needed for the command building for the batch return</param>
/// <returns>Path of the file and if it went correctly</returns>
/// <exception cref="InvalidOperationException">Failing to start filefrag on the system if this exception is found</exception>
private async Task<Dictionary<string, bool>> RunBatchAsync(IEnumerable<string> paths)
private Dictionary<string, bool> RunBatch(IEnumerable<string> paths)
{
var list = paths.Distinct(StringComparer.Ordinal).ToList();
var result = list.ToDictionary(p => p, _ => false, StringComparer.Ordinal);
ProcessStartInfo psi;
(bool ok, string stdout, string stderr, int code) res;
if (_useShell)
{
var inner = "filefrag -v -- " + string.Join(' ', list.Select(QuoteSingle));
psi = new ProcessStartInfo
{
FileName = "/bin/bash",
Arguments = "-lc " + QuoteDouble(inner),
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true,
WorkingDirectory = "/"
};
if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p))
psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin";
else
psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p;
var inner = "filefrag -v " + string.Join(' ', list.Select(QuoteSingle));
res = _runShell(inner, timeoutMs: 15000, workingDir: "/");
}
else
{
psi = new ProcessStartInfo
{
FileName = "filefrag",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p))
psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin";
else
psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p;
psi.ArgumentList.Add("-v");
psi.ArgumentList.Add("--");
var args = new List<string> { "-v" };
foreach (var path in list)
psi.ArgumentList.Add(path);
{
args.Add(' ' + path);
}
res = _runDirect("filefrag", args, workingDir: "/", timeoutMs: 15000);
}
using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Failed to start filefrag");
if (!string.IsNullOrWhiteSpace(res.stderr))
_log.LogTrace("filefrag stderr (batch): {err}", res.stderr.Trim());
var outTask = proc.StandardOutput.ReadToEndAsync(_cts.Token);
var errTask = proc.StandardError.ReadToEndAsync(_cts.Token);
var timeout = TimeSpan.FromSeconds(15);
var combined = Task.WhenAll(outTask, errTask);
var finished = await Task.WhenAny(combined, Task.Delay(timeout, _cts.Token)).ConfigureAwait(false);
if (finished != combined)
{
try { proc.Kill(entireProcessTree: true); } catch { /* ignore */ }
try { await combined.ConfigureAwait(false); } catch { /* ignore */ }
}
var stdout = outTask.IsCompletedSuccessfully ? await outTask.ConfigureAwait(false) : "";
var stderr = errTask.IsCompletedSuccessfully ? await errTask.ConfigureAwait(false) : "";
if (!string.IsNullOrWhiteSpace(stderr))
_log.LogTrace("filefrag stderr (batch): {err}", stderr.Trim());
ParseFilefrag(stdout, result);
ParseFilefrag(res.stdout, result);
return result;
}
@@ -225,7 +195,6 @@ namespace LightlessSync.Services.Compression
}
private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'";
private static string QuoteDouble(string s) => "\"" + s.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal).Replace("$", "\\$", StringComparison.Ordinal).Replace("`", "\\`", StringComparison.Ordinal) + "\"";
/// <summary>
/// Regex of the File Size return on the Linux/Wine systems, giving back the amount

10
LightlessSync/UI/SettingsUi.cs
View File

@@ -1236,7 +1236,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
UIColors.Get("LightlessYellow"));
}
if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled();
if (!_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled();
if (ImGui.Checkbox("Use file compactor", ref useFileCompactor))
{
_configService.Current.UseCompactor = useFileCompactor;
@@ -1281,20 +1281,20 @@ public class SettingsUi : WindowMediatorSubscriberBase
UIColors.Get("LightlessYellow"));
}
if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS)
if (!_cacheMonitor.StorageisNTFS)
{
ImGui.EndDisabled();
ImGui.TextUnformatted("The file compactor is only available on BTRFS and NTFS drives.");
ImGui.TextUnformatted("The file compactor is only available NTFS drives, soon for btrfs.");
}
if (_cacheMonitor.StorageisNTFS)
{
ImGui.TextUnformatted("The file compactor is running on NTFS Drive.");
ImGui.TextUnformatted("The file compactor detected an NTFS Drive.");
}
if (_cacheMonitor.StorageIsBtrfs)
{
ImGui.TextUnformatted("The file compactor is running on Btrfs Drive.");
ImGui.TextUnformatted("The file compactor detected an Btrfs Drive.");
}
ImGuiHelpers.ScaledDummy(new Vector2(10, 10));