Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 53 Wiki Activity

Changed some commands in file getting, redone compression check commands and turned off btrfs compactor for 1.12.4

Browse Source
This commit is contained in:
cake
2025-11-11 17:09:50 +01:00
parent 7de72471bb
commit 1862689b1b
3 changed files with 106 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
LightlessSync/FileCache/FileCompactor.cs
View File

@@ -1,12 +1,11 @@
using LightlessSync.LightlessConfiguration;
using LightlessSync.Services;
using LightlessSync.Services.Compression;
using LightlessSync.Services.Compactor;
using Microsoft.Extensions.Logging;
using Microsoft.Win32.SafeHandles;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Channels;
using static LightlessSync.Utils.FileSystemHelper;
@@ -28,6 +27,8 @@ public sealed class FileCompactor : IDisposable
private readonly List<Task> _workers = [];
private readonly SemaphoreSlim _globalGate;
//Limit btrfs gate on half of threads given to compactor.
private static readonly SemaphoreSlim _btrfsGate = new(4, 4);
private readonly BatchFilefragService _fragBatch;
@@ -83,8 +84,11 @@ public sealed class FileCompactor : IDisposable
_fragBatch = new BatchFilefragService(
useShell: _dalamudUtilService.IsWine,
log: _logger,
batchSize: 128,
flushMs: 25);
batchSize: 64,
flushMs: 25,
runDirect: RunProcessDirect,
runShell: RunProcessShell
);
_logger.LogInformation("FileCompactor started with {workers} workers", workerCount);
}
@@ -196,21 +200,15 @@ public sealed class FileCompactor : IDisposable
bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
var (_, linuxPath) = ResolvePathsForBtrfs(fileInfo.FullName);
var (ok1, out1, err1, code1) =
var (ok, output, err, code) =
isWindowsProc
? RunProcessShell($"stat -c %b -- {QuoteSingle(linuxPath)}", null, 10000)
: RunProcessDirect("stat", ["-c", "%b", "--", linuxPath], null, 10000);
? RunProcessShell($"stat -c='%b' {QuoteSingle(linuxPath)}", workingDir: null, 10000)
: RunProcessDirect("stat", ["-c='%b'", linuxPath], workingDir: null, 10000);
if (ok1 && long.TryParse(out1.Trim(), out long blocks))
if (ok && long.TryParse(output.Trim(), out long blocks))
return (false, blocks * 512L); // st_blocks are always 512B units
var (ok2, out2, err2, code2) = RunProcessShell($"du -B1 -- {QuoteSingle(linuxPath)} | cut -f1", workingDir: null, 10000); // use shell for the pipe
if (ok2 && long.TryParse(out2.Trim(), out long bytes))
return (false, bytes);
_logger.LogDebug("Btrfs size probe failed for {linux} (stat {code1}, du {code2}). Falling back to Length.",
linuxPath, code1, code2);
_logger.LogDebug("Btrfs size probe failed for {linux} (stat {code}, err {err}). Falling back to Length.", linuxPath, code, err);
return (false, fileInfo.Length);
}
catch (Exception ex)
@@ -360,7 +358,7 @@ public sealed class FileCompactor : IDisposable
}
/// <summary>
/// Decompress an BTRFS File
/// Decompress an BTRFS File on Wine/Linux
/// </summary>
/// <param name="path">Path of the compressed file</param>
/// <returns>Decompressing state</returns>
@@ -370,44 +368,55 @@ public sealed class FileCompactor : IDisposable
{
try
{
var (winPath, linuxPath) = ResolvePathsForBtrfs(path);
bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
bool isWine = _dalamudUtilService?.IsWine ?? false;
string linuxPath = isWine ? ToLinuxPathIfWine(path, isWine) : path;
var opts = GetMountOptionsForPath(linuxPath);
if (opts.Contains("compress", StringComparison.OrdinalIgnoreCase))
bool hasCompress = opts.Contains("compress", StringComparison.OrdinalIgnoreCase);
bool hasCompressForce = opts.Contains("compress-force", StringComparison.OrdinalIgnoreCase);
if (hasCompressForce)
{
_logger.LogWarning(
"Cannot safely decompress {file}: filesystem mounted with compression ({opts}). Remount with 'compress=no'.",
linuxPath, opts);
_logger.LogWarning("Cannot safely decompress {file}: mount options contains compress-force ({opts}).", linuxPath, opts);
return false;
}
if (hasCompress)
{
var setCmd = $"btrfs property set -- {QuoteDouble(linuxPath)} compression none";
var (okSet, _, errSet, codeSet) = isWine
? RunProcessShell(setCmd)
: RunProcessDirect("btrfs", ["property", "set", "--", linuxPath, "compression", "none"]);
if (!okSet)
{
_logger.LogWarning("Failed to set 'compression none' on {file}, please check drive options (exit code is: {code}): {err}", linuxPath, codeSet, errSet);
return false;
}
_logger.LogTrace("Set per-file 'compression none' on {file}", linuxPath);
}
if (!IsBtrfsCompressedFile(linuxPath))
{
_logger.LogTrace("Btrfs: not compressed, skip {file}", linuxPath);
_logger.LogTrace("{file} is not compressed, skipping decompression completely", linuxPath);
return true;
}
if (!ProbeFileReadableForBtrfs(winPath, linuxPath))
return false;
// Rewrite file uncompressed
(bool ok, string stdout, string stderr, int code) =
isWindowsProc
? RunProcessShell($"btrfs filesystem defragment -- {QuoteSingle(linuxPath)}")
: RunProcessDirect("btrfs", ["filesystem", "defragment", "--", linuxPath]);
var (ok, stdout, stderr, code) = isWine
? RunProcessShell($"btrfs filesystem defragment -- {QuoteDouble(linuxPath)}")
: RunProcessDirect("btrfs", ["filesystem", "defragment", "--", linuxPath]);
if (!ok)
{
_logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit {code}): {stderr}",
_logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit code is: {code}): {stderr}",
linuxPath, code, stderr);
return false;
}
if (!string.IsNullOrWhiteSpace(stdout))
_logger.LogTrace("btrfs (decompress) output {file}: {out}", linuxPath, stdout.Trim());
_logger.LogTrace("btrfs defragment output for {file}: {out}", linuxPath, stdout.Trim());
_logger.LogInformation("Decompressed (rewritten) Btrfs file: {file}", linuxPath);
_logger.LogInformation("Decompressed (rewritten uncompressed) Btrfs file: {file}", linuxPath);
return true;
}
catch (Exception ex)
@@ -467,19 +476,19 @@ public sealed class FileCompactor : IDisposable
/// <param name="path">Path that has to be converted</param>
/// <param name="isWine">Extra check if using the wine enviroment</param>
/// <returns>Converted path to be used in Linux</returns>
private static string ToLinuxPathIfWine(string path, bool isWine)
private string ToLinuxPathIfWine(string path, bool isWine, bool preferShell = true)
{
if (!isWine || !IsProbablyWine())
return path;
if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase))
return "/" + path[3..].Replace('\\', '/');
return ("/" + path[3..].Replace('\\', '/')).Replace("//", "/", StringComparison.Ordinal);
if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase))
{
const string usersPrefix = "C:\\Users\\";
var p = path.Replace('/', '\\');
const string usersPrefix = "C:\\Users\\";
if (p.StartsWith(usersPrefix, StringComparison.OrdinalIgnoreCase))
{
int afterUsers = usersPrefix.Length;
@@ -493,48 +502,38 @@ public sealed class FileCompactor : IDisposable
var linuxUser = Environment.GetEnvironmentVariable("USER") ?? Environment.UserName;
home = "/home/" + linuxUser;
}
// Join as Unix path
return (home.TrimEnd('/') + "/" + rel).Replace("//", "/", StringComparison.Ordinal);
return (home!.TrimEnd('/') + "/" + rel).Replace("//", "/", StringComparison.Ordinal);
}
}
try
{
var inner = "winepath -u " + "'" + path.Replace("'", "'\\''", StringComparison.Ordinal) + "'";
var psi = new ProcessStartInfo
(bool ok, string stdout, string stderr, int code) = preferShell
? RunProcessShell($"winepath -u {QuoteSingle(path)}", timeoutMs: 5000, workingDir: "/")
: RunProcessDirect("winepath", ["-u", path], workingDir: "/", timeoutMs: 5000);
if (ok)
{
FileName = "/bin/bash",
Arguments = "-c " + "\"" + inner.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal) + "\"",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true,
WorkingDirectory = "/"
};
using var proc = Process.Start(psi);
var outp = proc?.StandardOutput.ReadToEnd().Trim();
try
{
proc?.WaitForExit();
}
catch
{
/* Wine can throw here; ignore */
var outp = (stdout ?? "").Trim();
if (!string.IsNullOrEmpty(outp) && outp.StartsWith('/'))
return outp.Replace("//", "/", StringComparison.Ordinal);
}
else
{
_logger.LogTrace("winepath failed for {path} (exit {code}): {err}", path, code, stderr);
}
if (!string.IsNullOrEmpty(outp) && outp.StartsWith("/", StringComparison.Ordinal))
return outp;
}
catch
catch (Exception ex)
{
/* ignore and fall through the floor! */
_logger.LogTrace(ex, "winepath invocation failed for {path}", path);
}
}
return path.Replace('\\', '/');
return path.Replace('\\', '/').Replace("//", "/", StringComparison.Ordinal);
}
/// <summary>
/// Compress an WOF File
/// Compress an File using the WOF methods (NTFS)
/// </summary>
/// <param name="path">Path of the decompressed/normal file</param>
/// <returns>Compessing state</returns>
@@ -585,7 +584,7 @@ public sealed class FileCompactor : IDisposable
}
/// <summary>
/// Checks if an File is compacted with WOF compression
/// Checks if an File is compacted with WOF compression (NTFS)
/// </summary>
/// <param name="path">Path of the file</param>
/// <returns>State of the file</returns>
@@ -612,7 +611,7 @@ public sealed class FileCompactor : IDisposable
}
/// <summary>
/// Checks if an File is compacted any WOF compression with an WOF backing
/// Checks if an File is compacted any WOF compression with an WOF backing (NTFS)
/// </summary>
/// <param name="path">Path of the file</param>
/// <returns>State of the file, if its an external (no backing) and which algorithm if detected</returns>
@@ -821,7 +820,8 @@ public sealed class FileCompactor : IDisposable
CreateNoWindow = true
};
if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir;
// Use a Login shell so PATH includes /usr/sbin etc. AKA -lc
// Use a Login shell so PATH includes /usr/sbin etc. AKA -lc for login shell
psi.ArgumentList.Add("-lc");
psi.ArgumentList.Add(QuoteDouble(command));
EnsureUnixPathEnv(psi);
@@ -1011,7 +1011,7 @@ public sealed class FileCompactor : IDisposable
}
/// <summary>
/// Ensures the Unix pathing to be included into the process
/// Ensures the Unix pathing to be included into the process start
/// </summary>
/// <param name="psi">Process</param>
private static void EnsureUnixPathEnv(ProcessStartInfo psi)
@@ -1034,10 +1034,8 @@ public sealed class FileCompactor : IDisposable
if (!isWindowsProc)
return (path, path);
// Prefer winepath -u; fall back to your existing mapper
var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(path)}", null, 5000);
var linux = (ok && !string.IsNullOrWhiteSpace(outp)) ? outp.Trim()
: ToLinuxPathIfWine(path, isWine: true);
var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(path)}", workingDir: null, 5000);
var linux = (ok && !string.IsNullOrWhiteSpace(outp)) ? outp.Trim() : ToLinuxPathIfWine(path, isWine: true);
return (path, linux);
}