using LightlessSync.LightlessConfiguration; using LightlessSync.Services; using LightlessSync.Services.Compactor; using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.InteropServices; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; namespace LightlessSync.FileCache; public sealed class FileCompactor : IDisposable { public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U; public const ulong WOF_PROVIDER_FILE = 2UL; public const int _maxRetries = 3; private readonly ConcurrentDictionary _pendingCompactions; private readonly ILogger _logger; private readonly LightlessConfigService _lightlessConfigService; private readonly DalamudUtilService _dalamudUtilService; private readonly Channel _compactionQueue; private readonly CancellationTokenSource _compactionCts = new(); private readonly List _workers = []; private readonly SemaphoreSlim _globalGate; //Limit btrfs gate on half of threads given to compactor. private static readonly SemaphoreSlim _btrfsGate = new(4, 4); private readonly BatchFilefragService _fragBatch; private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo = new() { Algorithm = (int)CompressionAlgorithm.XPRESS8K, Flags = 0 }; [StructLayout(LayoutKind.Sequential, Pack = 1)] private struct WOF_FILE_COMPRESSION_INFO_V1 { public int Algorithm; public ulong Flags; } private enum CompressionAlgorithm { NO_COMPRESSION = -2, LZNT1 = -1, XPRESS4K = 0, LZX = 1, XPRESS8K = 2, XPRESS16K = 3 } public FileCompactor(ILogger logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService) { _pendingCompactions = new(StringComparer.OrdinalIgnoreCase); _logger = logger; _lightlessConfigService = lightlessConfigService; _dalamudUtilService = dalamudUtilService; _compactionQueue = Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = false, SingleWriter = false }); int workers = Math.Clamp(Math.Min(Environment.ProcessorCount / 2, 4), 1, 8); _globalGate = new SemaphoreSlim(workers, workers); int workerCount = Math.Max(workers * 2, workers); for (int i = 0; i < workerCount; i++) { _workers.Add(Task.Factory.StartNew( () => ProcessQueueWorkerAsync(_compactionCts.Token), _compactionCts.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default).Unwrap()); } _fragBatch = new BatchFilefragService( useShell: _dalamudUtilService.IsWine, log: _logger, batchSize: 64, flushMs: 25, runDirect: RunProcessDirect, runShell: RunProcessShell ); _logger.LogInformation("FileCompactor started with {workers} workers", workerCount); } public bool MassCompactRunning { get; private set; } public string Progress { get; private set; } = string.Empty; /// /// Compact the storage of the Cache Folder /// /// Used to check if files needs to be compressed public void CompactStorage(bool compress) { MassCompactRunning = true; try { var allFiles = Directory.EnumerateFiles(_lightlessConfigService.Current.CacheFolder).ToList(); int total = allFiles.Count; int current = 0; foreach (var file in allFiles) { current++; Progress = $"{current}/{total}"; try { // Compress or decompress files if (compress) CompactFile(file); else DecompressFile(file); } catch (IOException ioEx) { _logger.LogDebug(ioEx, "File {file} locked or busy, skipping", file); } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting/decompressing file {file}", file); } } } finally { MassCompactRunning = false; Progress = string.Empty; } } /// /// Write all bytes into a directory async /// /// Bytes will be writen to this filepath /// Bytes that have to be written /// Cancellation Token for interupts /// Writing Task public async Task WriteAllBytesAsync(string filePath, byte[] bytes, CancellationToken token) { var dir = Path.GetDirectoryName(filePath); if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) Directory.CreateDirectory(dir); await File.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false); if (_lightlessConfigService.Current.UseCompactor) EnqueueCompaction(filePath); } /// /// Gets the File size for an BTRFS or NTFS file system for the given FileInfo /// /// Amount of blocks used in the disk public long GetFileSizeOnDisk(FileInfo fileInfo) { var fsType = GetFilesystemType(fileInfo.FullName, _dalamudUtilService.IsWine); if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { (bool flowControl, long value) = GetFileSizeNTFS(fileInfo); if (!flowControl) { return value; } } if (fsType == FilesystemType.Btrfs) { (bool flowControl, long value) = GetFileSizeBtrfs(fileInfo); if (!flowControl) { return value; } } return fileInfo.Length; } /// /// Get File Size in an Btrfs file system (Linux/Wine). /// /// File that you want the size from. /// Succesful check and value of the filesize. /// Fails on the Process in StartProcessInfo private (bool flowControl, long value) GetFileSizeBtrfs(FileInfo fileInfo) { try { bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); var (_, linuxPath) = ResolvePathsForBtrfs(fileInfo.FullName); var (ok, output, err, code) = isWindowsProc ? RunProcessShell($"stat -c='%b' {QuoteSingle(linuxPath)}", workingDir: null, 10000) : RunProcessDirect("stat", ["-c='%b'", linuxPath], workingDir: null, 10000); if (ok && long.TryParse(output.Trim(), out long blocks)) return (false, blocks * 512L); // st_blocks are always 512B units _logger.LogDebug("Btrfs size probe failed for {linux} (stat {code}, err {err}). Falling back to Length.", linuxPath, code, err); return (false, fileInfo.Length); } catch (Exception ex) { _logger.LogDebug(ex, "Failed Btrfs size probe for {file}, using Length", fileInfo.FullName); return (false, fileInfo.Length); } } /// /// Get File Size in an NTFS file system (Windows). /// /// File that you want the size from. /// Succesful check and value of the filesize. private (bool flowControl, long value) GetFileSizeNTFS(FileInfo fileInfo) { try { var blockSize = GetBlockSizeForPath(fileInfo.FullName, _logger, _dalamudUtilService.IsWine); var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); var size = (long)hosize << 32 | losize; return (flowControl: false, value: ((size + blockSize - 1) / blockSize) * blockSize); } catch (Exception ex) { _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); } return (flowControl: true, value: default); } /// /// Compressing the given path with BTRFS or NTFS file system. /// /// Path of the decompressed/normal file private void CompactFile(string filePath) { var fi = new FileInfo(filePath); if (!fi.Exists) { _logger.LogTrace("Skip compaction: missing {file}", filePath); return; } var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); var oldSize = fi.Length; int blockSize = GetBlockSizeForPath(fi.FullName, _logger, _dalamudUtilService.IsWine); // We skipping small files (128KiB) as they slow down the system a lot for BTRFS. as BTRFS has a different blocksize it requires an different calculation. long minSizeBytes = fsType == FilesystemType.Btrfs ? Math.Max(blockSize * 2L, 128 * 1024L) : Math.Max(blockSize, 8 * 1024L); if (oldSize < minSizeBytes) { _logger.LogTrace("Skip compaction: {file} ({size} B) < threshold ({th} B)", filePath, oldSize, minSizeBytes); return; } if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { if (!IsWOFCompactedFile(filePath)) { _logger.LogDebug("NTFS compaction XPRESS8K: {file}", filePath); if (WOFCompressFile(filePath)) { var newSize = GetFileSizeOnDisk(fi); _logger.LogDebug("NTFS compressed {file} {old} -> {new}", filePath, oldSize, newSize); } else { _logger.LogWarning("NTFS compression failed or unavailable for {file}", filePath); } } else { _logger.LogTrace("Already NTFS-compressed: {file}", filePath); } return; } if (fsType == FilesystemType.Btrfs) { if (!IsBtrfsCompressedFile(filePath)) { _logger.LogDebug("Btrfs compression zstd: {file}", filePath); if (BtrfsCompressFile(filePath)) { var newSize = GetFileSizeOnDisk(fi); _logger.LogDebug("Btrfs compressed {file} {old} -> {new}", filePath, oldSize, newSize); } else { _logger.LogWarning("Btrfs compression failed or unavailable for {file}", filePath); } } else { _logger.LogTrace("Already Btrfs-compressed: {file}", filePath); } return; } _logger.LogTrace("Skip compact: unsupported FS for {file}", filePath); } /// /// Decompressing the given path with BTRFS file system or NTFS file system. /// /// Path of the compressed file private void DecompressFile(string path) { _logger.LogDebug("Decompress request: {file}", path); var fsType = GetFilesystemType(path, _dalamudUtilService.IsWine); if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { try { bool flowControl = DecompressWOFFile(path); if (!flowControl) { return; } } catch (Exception ex) { _logger.LogWarning(ex, "NTFS decompress error {file}", path); } } if (fsType == FilesystemType.Btrfs) { try { bool flowControl = DecompressBtrfsFile(path); if (!flowControl) { return; } } catch (Exception ex) { _logger.LogWarning(ex, "Btrfs decompress error {file}", path); } } } /// /// Decompress an BTRFS File on Wine/Linux /// /// Path of the compressed file /// Decompressing state private bool DecompressBtrfsFile(string path) { return RunWithBtrfsGate(() => { try { bool isWine = _dalamudUtilService?.IsWine ?? false; string linuxPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; var opts = GetMountOptionsForPath(linuxPath); bool hasCompress = opts.Contains("compress", StringComparison.OrdinalIgnoreCase); bool hasCompressForce = opts.Contains("compress-force", StringComparison.OrdinalIgnoreCase); if (hasCompressForce) { _logger.LogWarning("Cannot safely decompress {file}: mount options contains compress-force ({opts}).", linuxPath, opts); return false; } if (hasCompress) { var setCmd = $"btrfs property set -- {QuoteDouble(linuxPath)} compression none"; var (okSet, _, errSet, codeSet) = isWine ? RunProcessShell(setCmd) : RunProcessDirect("btrfs", ["property", "set", "--", linuxPath, "compression", "none"]); if (!okSet) { _logger.LogWarning("Failed to set 'compression none' on {file}, please check drive options (exit code is: {code}): {err}", linuxPath, codeSet, errSet); return false; } _logger.LogTrace("Set per-file 'compression none' on {file}", linuxPath); } if (!IsBtrfsCompressedFile(linuxPath)) { _logger.LogTrace("{file} is not compressed, skipping decompression completely", linuxPath); return true; } var (ok, stdout, stderr, code) = isWine ? RunProcessShell($"btrfs filesystem defragment -- {QuoteDouble(linuxPath)}") : RunProcessDirect("btrfs", ["filesystem", "defragment", "--", linuxPath]); if (!ok) { _logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit code is: {code}): {stderr}", linuxPath, code, stderr); return false; } if (!string.IsNullOrWhiteSpace(stdout)) _logger.LogTrace("btrfs defragment output for {file}: {out}", linuxPath, stdout.Trim()); _logger.LogInformation("Decompressed (rewritten uncompressed) Btrfs file: {file}", linuxPath); return true; } catch (Exception ex) { _logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path); return false; } }); } /// /// Decompress an NTFS File /// /// Path of the compressed file /// Decompressing state private bool DecompressWOFFile(string path) { //Check if its already been compressed if (TryIsWofExternal(path, out bool isExternal, out int algo)) { if (!isExternal) { _logger.LogTrace("Already decompressed file: {file}", path); return true; } var compressString = ((CompressionAlgorithm)algo).ToString(); _logger.LogTrace("WOF compression (algo={algo}) detected for {file}", compressString, path); } //This will attempt to start WOF thread. return WithFileHandleForWOF(path, FileAccess.ReadWrite, h => { if (!DeviceIoControl(h, FSCTL_DELETE_EXTERNAL_BACKING, IntPtr.Zero, 0, IntPtr.Zero, 0, out uint _, IntPtr.Zero)) { int err = Marshal.GetLastWin32Error(); // 342 error code means its been decompressed after the control, we handle it as it succesfully been decompressed. if (err == 342) { _logger.LogTrace("Successfully decompressed NTFS file {file}", path); return true; } _logger.LogWarning("DeviceIoControl failed for {file} with Win32 error {err}", path, err); return false; } _logger.LogTrace("Successfully decompressed NTFS file {file}", path); return true; }); } /// /// Converts to Linux Path if its using Wine (diferent pathing system in Wine) /// /// Path that has to be converted /// Extra check if using the wine enviroment /// Converted path to be used in Linux private string ToLinuxPathIfWine(string path, bool isWine, bool preferShell = true) { if (!isWine || !IsProbablyWine()) return path; if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) return ("/" + path[3..].Replace('\\', '/')).Replace("//", "/", StringComparison.Ordinal); if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) { const string usersPrefix = "C:\\Users\\"; var p = path.Replace('/', '\\'); if (p.StartsWith(usersPrefix, StringComparison.OrdinalIgnoreCase)) { int afterUsers = usersPrefix.Length; int slash = p.IndexOf('\\', afterUsers); if (slash > 0 && slash + 1 < p.Length) { var rel = p[(slash + 1)..].Replace('\\', '/'); var home = Environment.GetEnvironmentVariable("HOME"); if (string.IsNullOrEmpty(home)) { var linuxUser = Environment.GetEnvironmentVariable("USER") ?? Environment.UserName; home = "/home/" + linuxUser; } return (home!.TrimEnd('/') + "/" + rel).Replace("//", "/", StringComparison.Ordinal); } } try { (bool ok, string stdout, string stderr, int code) = preferShell ? RunProcessShell($"winepath -u {QuoteSingle(path)}", timeoutMs: 5000, workingDir: "/") : RunProcessDirect("winepath", ["-u", path], workingDir: "/", timeoutMs: 5000); if (ok) { var outp = (stdout ?? "").Trim(); if (!string.IsNullOrEmpty(outp) && outp.StartsWith('/')) return outp.Replace("//", "/", StringComparison.Ordinal); } else { _logger.LogTrace("winepath failed for {path} (exit {code}): {err}", path, code, stderr); } } catch (Exception ex) { _logger.LogTrace(ex, "winepath invocation failed for {path}", path); } } return path.Replace('\\', '/').Replace("//", "/", StringComparison.Ordinal); } /// /// Compress an File using the WOF methods (NTFS) /// /// Path of the decompressed/normal file /// Compessing state private bool WOFCompressFile(string path) { int size = Marshal.SizeOf(); IntPtr efInfoPtr = Marshal.AllocHGlobal(size); try { Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: false); ulong length = (ulong)size; return WithFileHandleForWOF(path, FileAccess.ReadWrite, h => { int ret = WofSetFileDataLocation(h, WOF_PROVIDER_FILE, efInfoPtr, length); // 0x80070158 is the being "already compressed/unsupported" style return if (ret != 0 && ret != unchecked((int)0x80070158)) { _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); return false; } return true; }); } catch (DllNotFoundException ex) { _logger.LogTrace(ex, "WofUtil not available; skipping NTFS compaction for {file}", path); return false; } catch (EntryPointNotFoundException ex) { _logger.LogTrace(ex, "WOF entrypoint missing on this system (Wine/older OS); skipping NTFS compaction for {file}", path); return false; } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting file {path}", path); return false; } finally { if (efInfoPtr != IntPtr.Zero) Marshal.FreeHGlobal(efInfoPtr); } } /// /// Checks if an File is compacted with WOF compression (NTFS) /// /// Path of the file /// State of the file private static bool IsWOFCompactedFile(string filePath) { try { uint buf = (uint)Marshal.SizeOf(); int result = WofIsExternalFile(filePath, out int isExternal, out _, out var info, ref buf); if (result != 0 || isExternal == 0) return false; return info.Algorithm == (int)CompressionAlgorithm.XPRESS8K || info.Algorithm == (int)CompressionAlgorithm.XPRESS4K || info.Algorithm == (int)CompressionAlgorithm.XPRESS16K || info.Algorithm == (int)CompressionAlgorithm.LZX || info.Algorithm == (int)CompressionAlgorithm.LZNT1 || info.Algorithm == (int)CompressionAlgorithm.NO_COMPRESSION; } catch { return false; } } /// /// Checks if an File is compacted any WOF compression with an WOF backing (NTFS) /// /// Path of the file /// State of the file, if its an external (no backing) and which algorithm if detected private static bool TryIsWofExternal(string path, out bool isExternal, out int algorithm) { isExternal = false; algorithm = 0; try { uint buf = (uint)Marshal.SizeOf(); int hr = WofIsExternalFile(path, out int ext, out _, out var info, ref buf); if (hr == 0 && ext != 0) { isExternal = true; algorithm = info.Algorithm; } return true; } catch (DllNotFoundException) { return false; } catch (EntryPointNotFoundException) { return false; } } /// /// Checks if an File is compacted with Btrfs compression /// /// Path of the file /// State of the file private bool IsBtrfsCompressedFile(string path) { return RunWithBtrfsGate(() => { try { bool windowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); string linuxPath = windowsProc ? ResolveLinuxPathForWine(path) : path; var task = _fragBatch.IsCompressedAsync(linuxPath, _compactionCts.Token); if (task.Wait(TimeSpan.FromSeconds(5), _compactionCts.Token) && task.IsCompletedSuccessfully) return task.Result; _logger.LogTrace("filefrag batch timed out for {file}", linuxPath); return false; } catch (OperationCanceledException) { return false; } catch (Exception ex) { _logger.LogDebug(ex, "filefrag batch check failed for {file}", path); return false; } }); } /// /// Compress an Btrfs File /// /// Path of the decompressed/normal file /// Compessing state private bool BtrfsCompressFile(string path) { return RunWithBtrfsGate(() => { try { var (winPath, linuxPath) = ResolvePathsForBtrfs(path); bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); if (IsBtrfsCompressedFile(linuxPath)) { _logger.LogTrace("Already Btrfs compressed: {file} (linux={linux})", winPath, linuxPath); return true; } if (!ProbeFileReadableForBtrfs(winPath, linuxPath)) { _logger.LogTrace("Probe failed; cannot open file for compress: {file} (linux={linux})", winPath, linuxPath); return false; } (bool ok, string stdout, string stderr, int code) = isWindowsProc ? RunProcessShell($"btrfs filesystem defragment -clzo -- {QuoteSingle(linuxPath)}") : RunProcessDirect("btrfs", ["filesystem", "defragment", "-clzo", "--", linuxPath]); if (!ok) { _logger.LogWarning("btrfs defragment failed for {file} (linux={linux}) exit {code}: {stderr}", winPath, linuxPath, code, stderr); return false; } if (!string.IsNullOrWhiteSpace(stdout)) _logger.LogTrace("btrfs output for {file}: {out}", winPath, stdout.Trim()); _logger.LogInformation("Compressed btrfs file successfully: {file} (linux={linux})", winPath, linuxPath); return true; } catch (Exception ex) { _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); return false; } }); } /// /// Attempt opening file stream for WOF functions /// /// File that has to be accessed /// Permissions for the file /// Access of the file stream for the WOF function to handle. /// State of the attempt for the file private bool WithFileHandleForWOF(string path, FileAccess access, Func body) { const FileShare share = FileShare.ReadWrite | FileShare.Delete; for (int attempt = 0; attempt < _maxRetries; attempt++) { try { using var fs = new FileStream(path, FileMode.Open, access, share); var handle = fs.SafeFileHandle; if (handle.IsInvalid) { _logger.LogWarning("Invalid file handle for {file}", path); return false; } return body(handle); } catch (IOException ex) { if (attempt == _maxRetries - 1) { _logger.LogWarning(ex, "File still in use after {attempts} attempts, skipping {file}", _maxRetries, path); return false; } int delay = 150 * (attempt + 1); _logger.LogTrace(ex, "File busy, retrying in {delay}ms for {file}", delay, path); Thread.Sleep(delay); } } return false; } /// /// Runs an nonshell process meant for Linux enviroments /// /// File that has to be excuted /// Arguments meant for the file/command /// Working directory used to execute the file with/without arguments /// Timeout timer for the process /// State of the process, output of the process and error with exit code private (bool ok, string stdout, string stderr, int exitCode) RunProcessDirect(string fileName, IEnumerable args, string? workingDir = null, int timeoutMs = 60000) { var psi = new ProcessStartInfo(fileName) { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true }; if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir; foreach (var a in args) psi.ArgumentList.Add(a); EnsureUnixPathEnv(psi); using var proc = Process.Start(psi); if (proc is null) return (false, "", "failed to start process", -1); var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token); if (!success) { return (false, so2, se2, -1); } int code; try { code = proc.ExitCode; } catch { code = -1; } return (code == 0, so2, se2, code); } /// /// Runs an shell using '/bin/bash'/ command meant for Linux/Wine enviroments /// /// Command that has to be excuted /// Timeout timer for the process /// State of the process, output of the process and error with exit code private (bool ok, string stdout, string stderr, int exitCode) RunProcessShell(string command, string? workingDir = null, int timeoutMs = 60000) { var psi = new ProcessStartInfo("/bin/bash") { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true }; if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir; // Use a Login shell so PATH includes /usr/sbin etc. AKA -lc for login shell psi.ArgumentList.Add("-lc"); psi.ArgumentList.Add(QuoteDouble(command)); EnsureUnixPathEnv(psi); using var proc = Process.Start(psi); if (proc is null) return (false, "", "failed to start /bin/bash", -1); var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token); if (!success) { return (false, so2, se2, -1); } int code; try { code = proc.ExitCode; } catch { code = -1; } return (code == 0, so2, se2, code); } /// /// Checking the process result for shell or direct processes /// /// Process /// How long when timeout is gotten /// Cancellation Token /// Multiple variables private (bool success, string testy, string testi) CheckProcessResult(Process proc, int timeoutMs, CancellationToken token) { var outTask = proc.StandardOutput.ReadToEndAsync(token); var errTask = proc.StandardError.ReadToEndAsync(token); var bothTasks = Task.WhenAll(outTask, errTask); //On wine, we dont wanna use waitforexit as it will be always broken and giving an error. if (_dalamudUtilService.IsWine) { var finished = Task.WhenAny(bothTasks, Task.Delay(timeoutMs, token)).GetAwaiter().GetResult(); if (finished != bothTasks) { try { proc.Kill(entireProcessTree: true); Task.WaitAll([outTask, errTask], 1000, token); } catch { // ignore this } var so = outTask.IsCompleted ? outTask.Result : ""; var se = errTask.IsCompleted ? errTask.Result : "timeout"; return (false, so, se); } var stderr = errTask.Result; var ok = string.IsNullOrWhiteSpace(stderr); return (ok, outTask.Result, stderr); } // On linux, we can use it as we please if (!proc.WaitForExit(timeoutMs)) { try { proc.Kill(entireProcessTree: true); Task.WaitAll([outTask, errTask], 1000, token); } catch { // ignore this } return (false, outTask.IsCompleted ? outTask.Result : "", "timeout"); } Task.WaitAll(outTask, errTask); return (true, outTask.Result, errTask.Result); } /// /// Enqueues the compaction/decompation of an filepath. /// /// Filepath that will be enqueued private void EnqueueCompaction(string filePath) { // Safe-checks if (string.IsNullOrWhiteSpace(filePath)) return; if (!_lightlessConfigService.Current.UseCompactor) return; if (!File.Exists(filePath)) return; if (!_pendingCompactions.TryAdd(filePath, 0)) return; bool enqueued = false; try { bool isWine = _dalamudUtilService?.IsWine ?? false; var fsType = GetFilesystemType(filePath, isWine); // If under Wine, we should skip NTFS because its not Windows but might return NTFS. if (fsType == FilesystemType.NTFS && isWine) { _logger.LogTrace("Skip enqueue (NTFS under Wine) {file}", filePath); return; } // Unknown file system should be skipped. if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) { _logger.LogTrace("Skip enqueue (unsupported fs) {fs} {file}", fsType, filePath); return; } // Channel got closed, skip enqueue on file if (!_compactionQueue.Writer.TryWrite(filePath)) { _logger.LogTrace("Skip enqueue: compaction channel is/got closed {file}", filePath); return; } enqueued = true; _logger.LogTrace("Queued compaction for {file} (fs={fs})", filePath, fsType); } finally { if (!enqueued) _pendingCompactions.TryRemove(filePath, out _); } } /// /// Process the queue with, meant for a worker/thread /// /// Cancellation token for the worker whenever it needs to be stopped private async Task ProcessQueueWorkerAsync(CancellationToken token) { try { while (await _compactionQueue.Reader.WaitToReadAsync(token).ConfigureAwait(false)) { while (_compactionQueue.Reader.TryRead(out var filePath)) { try { token.ThrowIfCancellationRequested(); await _globalGate.WaitAsync(token).ConfigureAwait(false); try { if (_lightlessConfigService.Current.UseCompactor && File.Exists(filePath)) CompactFile(filePath); } finally { _globalGate.Release(); } } catch (OperationCanceledException) { return; } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting file {file}", filePath); } finally { _pendingCompactions.TryRemove(filePath, out _); } } } } catch (OperationCanceledException) { // Shutting down worker, this exception is expected } } /// /// Resolves linux path from wine pathing /// /// Windows path given from Wine /// Linux path to be used in Linux private string ResolveLinuxPathForWine(string windowsPath) { var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(windowsPath)}", null, 5000); if (ok && !string.IsNullOrWhiteSpace(outp)) return outp.Trim(); return ToLinuxPathIfWine(windowsPath, isWine: true); } /// /// Ensures the Unix pathing to be included into the process start /// /// Process private static void EnsureUnixPathEnv(ProcessStartInfo psi) { if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p)) psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin"; else if (!p.Contains("/usr/sbin", StringComparison.Ordinal)) psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p; } /// /// Resolves paths for Btrfs to be used on wine or linux and windows in case /// /// Path given t /// private (string windowsPath, string linuxPath) ResolvePathsForBtrfs(string path) { bool isWindowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); if (!isWindowsProc) return (path, path); var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(path)}", workingDir: null, 5000); var linux = (ok && !string.IsNullOrWhiteSpace(outp)) ? outp.Trim() : ToLinuxPathIfWine(path, isWine: true); return (path, linux); } /// /// Probes file if its readable to be used /// /// Windows path /// Linux path /// Succesfully probed or not private bool ProbeFileReadableForBtrfs(string winePath, string linuxPath) { try { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { using var _ = new FileStream(winePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); } else { using var _ = new FileStream(linuxPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); } return true; } catch { return false; } } /// /// Running functions into the Btrfs Gate/Threading. /// /// Type of the function that wants to be run inside Btrfs Gate /// Body of the function /// Task private T RunWithBtrfsGate(Func body) { bool acquired = false; try { _btrfsGate.Wait(_compactionCts.Token); acquired = true; return body(); } finally { if (acquired) _btrfsGate.Release(); } } [DllImport("kernel32.dll", SetLastError = true)] private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped); [DllImport("kernel32.dll")] private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh); [DllImport("WofUtil.dll")] private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength); [DllImport("WofUtil.dll", SetLastError = true)] private static extern int WofSetFileDataLocation(SafeFileHandle FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; private static string QuoteDouble(string s) => "\"" + s.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal).Replace("$", "\\$", StringComparison.Ordinal).Replace("`", "\\`", StringComparison.Ordinal) + "\""; public void Dispose() { _fragBatch?.Dispose(); _compactionQueue.Writer.TryComplete(); _compactionCts.Cancel(); try { Task.WaitAll([.. _workers.Where(t => t != null)], TimeSpan.FromSeconds(5)); } catch { // Ignore this catch on the dispose } finally { _compactionCts.Dispose(); } GC.SuppressFinalize(this); } }