Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 51 Wiki Activity

Refactored

Browse Source
This commit is contained in:
cake
2025-11-10 06:25:35 +01:00
parent d7182e9d57
commit 7de72471bb
1 changed files with 102 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
LightlessSync/FileCache/FileCompactor.cs
View File

@@ -6,6 +6,7 @@ using Microsoft.Win32.SafeHandles;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Channels;
using static LightlessSync.Utils.FileSystemHelper;
@@ -82,7 +83,7 @@ public sealed class FileCompactor : IDisposable
_fragBatch = new BatchFilefragService(
useShell: _dalamudUtilService.IsWine,
log: _logger,
batchSize: 256,
batchSize: 128,
flushMs: 25);
_logger.LogInformation("FileCompactor started with {workers} workers", workerCount);
@@ -198,13 +199,12 @@ public sealed class FileCompactor : IDisposable
var (ok1, out1, err1, code1) =
isWindowsProc
? RunProcessShell($"stat -c %b -- {QuoteSingle(linuxPath)}", null, 10000)
: RunProcessDirect("stat", new[] { "-c", "%b", "--", linuxPath }, null, 10000);
: RunProcessDirect("stat", ["-c", "%b", "--", linuxPath], null, 10000);
if (ok1 && long.TryParse(out1.Trim(), out long blocks))
return (false, blocks * 512L); // st_blocks are 512B units
return (false, blocks * 512L); // st_blocks are always 512B units
// Fallback: du -B1 (true on-disk bytes)
var (ok2, out2, err2, code2) = RunProcessShell($"du -B1 -- {QuoteSingle(linuxPath)} | cut -f1", null, 10000); // use shell for the pipe
var (ok2, out2, err2, code2) = RunProcessShell($"du -B1 -- {QuoteSingle(linuxPath)} | cut -f1", workingDir: null, 10000); // use shell for the pipe
if (ok2 && long.TryParse(out2.Trim(), out long bytes))
return (false, bytes);
@@ -425,6 +425,7 @@ public sealed class FileCompactor : IDisposable
/// <returns>Decompressing state</returns>
private bool DecompressWOFFile(string path)
{
//Check if its already been compressed
if (TryIsWofExternal(path, out bool isExternal, out int algo))
{
if (!isExternal)
@@ -436,6 +437,7 @@ public sealed class FileCompactor : IDisposable
_logger.LogTrace("WOF compression (algo={algo}) detected for {file}", compressString, path);
}
//This will attempt to start WOF thread.
return WithFileHandleForWOF(path, FileAccess.ReadWrite, h =>
{
if (!DeviceIoControl(h, FSCTL_DELETE_EXTERNAL_BACKING,
@@ -524,7 +526,7 @@ public sealed class FileCompactor : IDisposable
}
catch
{
/* ignore and fall through */
/* ignore and fall through the floor! */
}
}
@@ -550,7 +552,7 @@ public sealed class FileCompactor : IDisposable
{
int ret = WofSetFileDataLocation(h, WOF_PROVIDER_FILE, efInfoPtr, length);
// 0x80070158 is the benign "already compressed/unsupported" style return
// 0x80070158 is the being "already compressed/unsupported" style return
if (ret != 0 && ret != unchecked((int)0x80070158))
{
_logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X"));
@@ -791,38 +793,12 @@ public sealed class FileCompactor : IDisposable
using var proc = Process.Start(psi);
if (proc is null) return (false, "", "failed to start process", -1);
var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token);
var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token);
var both = Task.WhenAll(outTask, errTask);
if (_dalamudUtilService.IsWine)
var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token);
if (!success)
{
var finished = Task.WhenAny(both, Task.Delay(timeoutMs, _compactionCts.Token)).GetAwaiter().GetResult();
if (finished != both)
{
try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ }
try { Task.WaitAll(new[] { outTask, errTask }, 1000, _compactionCts.Token); } catch { /* ignore this */ }
var so = outTask.IsCompleted ? outTask.Result : "";
var se = errTask.IsCompleted ? errTask.Result : "timeout";
return (false, so, se, -1);
}
var stdout = outTask.Result;
var stderr = errTask.Result;
var ok = string.IsNullOrWhiteSpace(stderr);
return (ok, stdout, stderr, ok ? 0 : -1);
return (false, so2, se2, -1);
}
if (!proc.WaitForExit(timeoutMs))
{
try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ }
try { Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); } catch { /* ignore this */ }
return (false, outTask.IsCompleted ? outTask.Result : "", "timeout", -1);
}
Task.WaitAll(outTask, errTask);
var so2 = outTask.Result;
var se2 = errTask.Result;
int code;
try { code = proc.ExitCode; } catch { code = -1; }
return (code == 0, so2, se2, code);
@@ -836,7 +812,7 @@ public sealed class FileCompactor : IDisposable
/// <returns>State of the process, output of the process and error with exit code</returns>
private (bool ok, string stdout, string stderr, int exitCode) RunProcessShell(string command, string? workingDir = null, int timeoutMs = 60000)
{
// Use a LOGIN shell so PATH includes /usr/sbin etc.
var psi = new ProcessStartInfo("/bin/bash")
{
RedirectStandardOutput = true,
@@ -845,7 +821,7 @@ public sealed class FileCompactor : IDisposable
CreateNoWindow = true
};
if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir;
// Use a Login shell so PATH includes /usr/sbin etc. AKA -lc
psi.ArgumentList.Add("-lc");
psi.ArgumentList.Add(QuoteDouble(command));
EnsureUnixPathEnv(psi);
@@ -853,43 +829,74 @@ public sealed class FileCompactor : IDisposable
using var proc = Process.Start(psi);
if (proc is null) return (false, "", "failed to start /bin/bash", -1);
var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token);
var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token);
var both = Task.WhenAll(outTask, errTask);
if (_dalamudUtilService.IsWine)
var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token);
if (!success)
{
var finished = Task.WhenAny(both, Task.Delay(timeoutMs, _compactionCts.Token)).GetAwaiter().GetResult();
if (finished != both)
{
try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ }
try { Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); } catch { /* ignore this */ }
var so = outTask.IsCompleted ? outTask.Result : "";
var se = errTask.IsCompleted ? errTask.Result : "timeout";
return (false, so, se, -1);
}
var stdout = outTask.Result;
var stderr = errTask.Result;
var ok = string.IsNullOrWhiteSpace(stderr);
return (ok, stdout, stderr, ok ? 0 : -1);
return (false, so2, se2, -1);
}
if (!proc.WaitForExit(timeoutMs))
{
try { proc.Kill(entireProcessTree: true); } catch { /* ignore this */ }
try { Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); } catch { /* ignore this */ }
return (false, outTask.IsCompleted ? outTask.Result : "", "timeout", -1);
}
Task.WaitAll(outTask, errTask);
var so2 = outTask.Result;
var se2 = errTask.Result;
int code;
try { code = proc.ExitCode; } catch { code = -1; }
return (code == 0, so2, se2, code);
}
/// <summary>
/// Checking the process result for shell or direct processes
/// </summary>
/// <param name="proc">Process</param>
/// <param name="timeoutMs">How long when timeout is gotten</param>
/// <param name="token">Cancellation Token</param>
/// <returns>Multiple variables</returns>
private (bool success, string testy, string testi) CheckProcessResult(Process proc, int timeoutMs, CancellationToken token)
{
var outTask = proc.StandardOutput.ReadToEndAsync(token);
var errTask = proc.StandardError.ReadToEndAsync(token);
var bothTasks = Task.WhenAll(outTask, errTask);
//On wine, we dont wanna use waitforexit as it will be always broken and giving an error.
if (_dalamudUtilService.IsWine)
{
var finished = Task.WhenAny(bothTasks, Task.Delay(timeoutMs, token)).GetAwaiter().GetResult();
if (finished != bothTasks)
{
try
{
proc.Kill(entireProcessTree: true);
Task.WaitAll([outTask, errTask], 1000, token);
}
catch
{
// ignore this
}
var so = outTask.IsCompleted ? outTask.Result : "";
var se = errTask.IsCompleted ? errTask.Result : "timeout";
return (false, so, se);
}
var stderr = errTask.Result;
var ok = string.IsNullOrWhiteSpace(stderr);
return (ok, outTask.Result, stderr);
}
// On linux, we can use it as we please
if (!proc.WaitForExit(timeoutMs))
{
try
{
proc.Kill(entireProcessTree: true);
Task.WaitAll([outTask, errTask], 1000, token);
}
catch
{
// ignore this
}
return (false, outTask.IsCompleted ? outTask.Result : "", "timeout");
}
Task.WaitAll(outTask, errTask);
return (true, outTask.Result, errTask.Result);
}
/// <summary>
/// Enqueues the compaction/decompation of an filepath.
/// </summary>
@@ -991,6 +998,11 @@ public sealed class FileCompactor : IDisposable
}
}
/// <summary>
/// Resolves linux path from wine pathing
/// </summary>
/// <param name="windowsPath">Windows path given from Wine</param>
/// <returns>Linux path to be used in Linux</returns>
private string ResolveLinuxPathForWine(string windowsPath)
{
var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(windowsPath)}", null, 5000);
@@ -998,6 +1010,10 @@ public sealed class FileCompactor : IDisposable
return ToLinuxPathIfWine(windowsPath, isWine: true);
}
/// <summary>
/// Ensures the Unix pathing to be included into the process
/// </summary>
/// <param name="psi">Process</param>
private static void EnsureUnixPathEnv(ProcessStartInfo psi)
{
if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p))
@@ -1006,6 +1022,11 @@ public sealed class FileCompactor : IDisposable
psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p;
}
/// <summary>
/// Resolves paths for Btrfs to be used on wine or linux and windows in case
/// </summary>
/// <param name="path">Path given t</param>
/// <returns></returns>
private (string windowsPath, string linuxPath) ResolvePathsForBtrfs(string path)
{
bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
@@ -1021,13 +1042,19 @@ public sealed class FileCompactor : IDisposable
return (path, linux);
}
private bool ProbeFileReadableForBtrfs(string windowsPath, string linuxPath)
/// <summary>
/// Probes file if its readable to be used
/// </summary>
/// <param name="winePath">Windows path</param>
/// <param name="linuxPath">Linux path</param>
/// <returns>Succesfully probed or not</returns>
private bool ProbeFileReadableForBtrfs(string winePath, string linuxPath)
{
try
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
using var _ = new FileStream(windowsPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
using var _ = new FileStream(winePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
}
else
{
@@ -1038,6 +1065,12 @@ public sealed class FileCompactor : IDisposable
catch { return false; }
}
/// <summary>
/// Running functions into the Btrfs Gate/Threading.
/// </summary>
/// <typeparam name="T">Type of the function that wants to be run inside Btrfs Gate</typeparam>
/// <param name="body">Body of the function</param>
/// <returns>Task</returns>
private T RunWithBtrfsGate<T>(Func<T> body)
{
bool acquired = false;