using LightlessSync.LightlessConfiguration; using LightlessSync.Services; using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.InteropServices; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; namespace LightlessSync.FileCache; public sealed class FileCompactor : IDisposable { public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U; public const ulong WOF_PROVIDER_FILE = 2UL; private readonly ConcurrentDictionary _pendingCompactions; private readonly ILogger _logger; private readonly LightlessConfigService _lightlessConfigService; private readonly DalamudUtilService _dalamudUtilService; private readonly Channel _compactionQueue; private readonly CancellationTokenSource _compactionCts = new(); private readonly Task _compactionWorker; private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo = new() { Algorithm = (int)CompressionAlgorithm.XPRESS8K, Flags = 0 }; private enum CompressionAlgorithm { NO_COMPRESSION = -2, LZNT1 = -1, XPRESS4K = 0, LZX = 1, XPRESS8K = 2, XPRESS16K = 3 } public FileCompactor(ILogger logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService) { _pendingCompactions = new(StringComparer.OrdinalIgnoreCase); _logger = logger; _lightlessConfigService = lightlessConfigService; _dalamudUtilService = dalamudUtilService; _compactionQueue = Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); _compactionWorker = Task.Factory.StartNew(() => ProcessQueueAsync(_compactionCts.Token), _compactionCts.Token, TaskCreationOptions.LongRunning,TaskScheduler.Default).Unwrap(); } public bool MassCompactRunning { get; private set; } public string Progress { get; private set; } = string.Empty; public void CompactStorage(bool compress) { MassCompactRunning = true; try { var allFiles = Directory.EnumerateFiles(_lightlessConfigService.Current.CacheFolder).ToList(); int total = allFiles.Count; int current = 0; foreach (var file in allFiles) { current++; Progress = $"{current}/{total}"; try { if (compress) CompactFile(file); else DecompressFile(file); } catch (IOException ioEx) { _logger.LogDebug(ioEx, "File {file} locked or busy, skipping", file); } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting/decompressing file {file}", file); } } } finally { MassCompactRunning = false; Progress = string.Empty; } } /// /// Write all bytes into a directory async /// /// Bytes will be writen to this filepath /// Bytes that have to be written /// Cancellation Token for interupts /// Writing Task public async Task WriteAllBytesAsync(string filePath, byte[] bytes, CancellationToken token) { var dir = Path.GetDirectoryName(filePath); if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) Directory.CreateDirectory(dir); await File.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false); if (_lightlessConfigService.Current.UseCompactor) EnqueueCompaction(filePath); } /// /// Gets the File size for an BTRFS or NTFS file system for the given FileInfo /// /// Amount of blocks used in the disk public long GetFileSizeOnDisk(FileInfo fileInfo) { var fsType = GetFilesystemType(fileInfo.FullName, _dalamudUtilService.IsWine); if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { var blockSize = GetBlockSizeForPath(fileInfo.FullName, _logger, _dalamudUtilService.IsWine); var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); var size = (long)hosize << 32 | losize; return ((size + blockSize - 1) / blockSize) * blockSize; } if (fsType == FilesystemType.Btrfs) { try { var realPath = fileInfo.FullName.Replace("\"", "\\\"", StringComparison.Ordinal); var psi = new ProcessStartInfo("stat", $"-c %b \"{realPath}\"") { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true }; using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat"); var outp = proc.StandardOutput.ReadToEnd(); var err = proc.StandardError.ReadToEnd(); proc.WaitForExit(); if (proc.ExitCode != 0) throw new InvalidOperationException($"stat failed: {err}"); if (!long.TryParse(outp.Trim(), out var blocks)) throw new InvalidOperationException($"invalid stat output: {outp}"); // st_blocks are always 512-byte on Linux enviroment. return blocks * 512L; } catch (Exception ex) { _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); } } return fileInfo.Length; } /// /// Compressing the given path with BTRFS or NTFS file system. /// /// Path of the decompressed/normal file private void CompactFile(string filePath) { var fi = new FileInfo(filePath); if (!fi.Exists) { _logger.LogTrace("Skip compact: missing {file}", filePath); return; } var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); _logger.LogTrace("Detected filesystem {fs} for {file} (isWine={wine})", fsType, filePath, _dalamudUtilService.IsWine); var oldSize = fi.Length; int blockSize = GetBlockSizeForPath(fi.FullName, _logger, _dalamudUtilService.IsWine); if (oldSize < Math.Max(blockSize, 8 * 1024)) { _logger.LogTrace("Skip compact: {file} < block {block}", filePath, blockSize); return; } if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { if (!IsWOFCompactedFile(filePath)) { _logger.LogDebug("NTFS compact XPRESS8K: {file}", filePath); if (WOFCompressFile(filePath)) { var newSize = GetFileSizeOnDisk(fi); _logger.LogDebug("NTFS compressed {file} {old} -> {new}", filePath, oldSize, newSize); } else { _logger.LogWarning("NTFS compression failed or unavailable for {file}", filePath); } } else { _logger.LogTrace("Already NTFS-compressed: {file}", filePath); } return; } if (fsType == FilesystemType.Btrfs) { if (!IsBtrfsCompressedFile(filePath)) { _logger.LogDebug("Btrfs compress zstd: {file}", filePath); if (BtrfsCompressFile(filePath)) { var newSize = GetFileSizeOnDisk(fi); _logger.LogDebug("Btrfs compressed {file} {old} -> {new}", filePath, oldSize, newSize); } else { _logger.LogWarning("Btrfs compression failed or unavailable for {file}", filePath); } } else { _logger.LogTrace("Already Btrfs-compressed: {file}", filePath); } return; } _logger.LogTrace("Skip compact: unsupported FS for {file}", filePath); } /// /// Decompressing the given path with BTRFS file system or NTFS file system. /// /// Path of the compressed file private void DecompressFile(string path) { _logger.LogDebug("Decompress request: {file}", path); var fsType = GetFilesystemType(path, _dalamudUtilService.IsWine); if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { try { bool flowControl = DecompressWOFFile(path, out FileStream fs); if (!flowControl) { return; } } catch (Exception ex) { _logger.LogWarning(ex, "NTFS decompress error {file}", path); } } if (fsType == FilesystemType.Btrfs) { try { bool flowControl = DecompressBtrfsFile(path); if (!flowControl) { return; } } catch (Exception ex) { _logger.LogWarning(ex, "Btrfs decompress error {file}", path); } } } /// /// Decompress an BTRFS File /// /// Path of the compressed file /// Decompessing state private bool DecompressBtrfsFile(string path) { try { var opts = GetMountOptionsForPath(path); if (opts.Contains("compress", StringComparison.OrdinalIgnoreCase)) { _logger.LogWarning("Cannot safely decompress {file}: mount options include compression ({opts})", path, opts); return false; } string realPath = ToLinuxPathIfWine(path, _dalamudUtilService.IsWine); var psi = new ProcessStartInfo { FileName = _dalamudUtilService.IsWine ? "/bin/bash" : "btrfs", Arguments = _dalamudUtilService.IsWine ? $"-c \"btrfs filesystem defragment -- '{EscapeSingle(realPath)}'\"" : $"filesystem defragment -- \"{realPath}\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true, WorkingDirectory = "/" }; using var proc = Process.Start(psi); if (proc == null) { _logger.LogWarning("Failed to start btrfs defragment for {file}", path); return false; } var stdout = proc.StandardOutput.ReadToEnd(); var stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); if (proc.ExitCode != 0) { _logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr); return false; } if (!string.IsNullOrWhiteSpace(stdout)) _logger.LogTrace("btrfs defragment output {file}: {out}", path, stdout.Trim()); _logger.LogInformation("Btrfs rewritten uncompressed: {file}", path); } catch (Exception ex) { _logger.LogWarning(ex, "Btrfs decompress error {file}", path); } return true; } /// /// Decompress an NTFS File /// /// Path of the compressed file /// Decompessing state private bool DecompressWOFFile(string path, out FileStream fs) { fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); var handle = fs.SafeFileHandle; if (handle.IsInvalid) { _logger.LogWarning("Invalid handle: {file}", path); return false; } if (!DeviceIoControl(handle, FSCTL_DELETE_EXTERNAL_BACKING, IntPtr.Zero, 0, IntPtr.Zero, 0, out _, IntPtr.Zero)) { int err = Marshal.GetLastWin32Error(); if (err == 342) { _logger.LogTrace("File {file} not externally backed (already decompressed)", path); } else { _logger.LogWarning("DeviceIoControl failed for {file} with Win32 error {err}", path, err); } } else { _logger.LogTrace("Successfully decompressed NTFS file {file}", path); } return true; } private static string EscapeSingle(string p) => p.Replace("'", "'\\'", StringComparison.Ordinal); private static string ToLinuxPathIfWine(string path, bool isWine) { if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return path; if (!IsProbablyWine() && !isWine) return path; string realPath = path; if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) realPath = "/" + path[3..].Replace('\\', '/'); else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) realPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/'); return realPath; } /// /// Compress an WOF File /// /// Path of the decompressed/normal file /// Compessing state private bool WOFCompressFile(string path) { FileStream? fs = null; int size = Marshal.SizeOf(); IntPtr efInfoPtr = Marshal.AllocHGlobal(size); try { Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: false); ulong length = (ulong)size; const int maxRetries = 3; for (int attempt = 0; attempt < maxRetries; attempt++) { try { fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); break; } catch (IOException) { if (attempt == maxRetries - 1) { _logger.LogWarning("File still in use after {attempts} attempts, skipping compression for {file}", maxRetries, path); return false; } int delay = 150 * (attempt + 1); _logger.LogTrace("File in use, retrying in {delay}ms for {file}", delay, path); Thread.Sleep(delay); } } if (fs == null) { _logger.LogWarning("Failed to open {file} for compression; skipping", path); return false; } var handle = fs.SafeFileHandle; if (handle.IsInvalid) { _logger.LogWarning("Invalid file handle for {file}", path); return false; } int ret = WofSetFileDataLocation(handle, WOF_PROVIDER_FILE, efInfoPtr, length); // 0x80070158 is WOF error whenever compression fails in an non-fatal way. if (ret != 0 && ret != unchecked((int)0x80070158)) { _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); return false; } return true; } catch (DllNotFoundException) { _logger.LogTrace("WofUtil.dll not available; skipping NTFS compaction for {file}", path); return false; } catch (EntryPointNotFoundException) { _logger.LogTrace("WOF entrypoint missing (Wine/older OS); skipping NTFS compaction for {file}", path); return false; } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting file {path}", path); return false; } finally { fs?.Dispose(); if (efInfoPtr != IntPtr.Zero) { Marshal.FreeHGlobal(efInfoPtr); } } } /// /// Checks if an File is compacted with WOF compression /// /// Path of the file /// State of the file private static bool IsWOFCompactedFile(string filePath) { try { uint buf = (uint)Marshal.SizeOf(); int result = WofIsExternalFile(filePath, out int isExternal, out _, out var info, ref buf); if (result != 0 || isExternal == 0) return false; return info.Algorithm == (int)CompressionAlgorithm.XPRESS8K || info.Algorithm == (int)CompressionAlgorithm.XPRESS4K || info.Algorithm == (int)CompressionAlgorithm.XPRESS16K || info.Algorithm == (int)CompressionAlgorithm.LZX || info.Algorithm == (int)CompressionAlgorithm.LZNT1 || info.Algorithm == (int)CompressionAlgorithm.NO_COMPRESSION; } catch { return false; } } /// /// Checks if an File is compacted with Btrfs compression /// /// Path of the file /// State of the file private bool IsBtrfsCompressedFile(string path) { try { bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; var psi = new ProcessStartInfo { FileName = isWine ? "/bin/bash" : "filefrag", Arguments = isWine ? $"-c \"filefrag -v '{EscapeSingle(realPath)}'\"" : $"-v \"{realPath}\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true, WorkingDirectory = "/" }; using var proc = Process.Start(psi); if (proc == null) { _logger.LogWarning("Failed to start filefrag for {file}", path); return false; } string stdout = proc.StandardOutput.ReadToEnd(); string stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) { _logger.LogTrace("filefrag exited with code {code}: {stderr}", proc.ExitCode, stderr); } bool compressed = stdout.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); _logger.LogTrace("Btrfs compression check for {file}: {compressed}", path, compressed); return compressed; } catch (Exception ex) { _logger.LogDebug(ex, "Failed to detect Btrfs compression for {file}", path); return false; } } /// /// Compress an Btrfs File /// /// Path of the decompressed/normal file /// Compessing state private bool BtrfsCompressFile(string path) { try { bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; if (isWine && IsProbablyWine()) { if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) { realPath = "/" + path[3..].Replace('\\', '/'); } else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) { string linuxHome = Environment.GetEnvironmentVariable("HOME") ?? "/home"; realPath = Path.Combine(linuxHome, path[3..].Replace('\\', '/')).Replace('\\', '/'); } _logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", realPath); } const int maxRetries = 3; for (int attempt = 0; attempt < maxRetries; attempt++) { try { using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); break; } catch (IOException) { if (attempt == maxRetries - 1) { _logger.LogWarning("File still in use after {attempts} attempts; skipping btrfs compression for {file}", maxRetries, path); return false; } int delay = 150 * (attempt + 1); _logger.LogTrace("File busy, retrying in {delay}ms for {file}", delay, path); Thread.Sleep(delay); } } string command = $"btrfs filesystem defragment -czstd -- \"{realPath}\""; var psi = new ProcessStartInfo { FileName = isWine ? "/bin/bash" : "btrfs", Arguments = isWine ? $"-c \"{command}\"" : $"filesystem defragment -czstd -- \"{realPath}\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true, WorkingDirectory = "/" }; using var proc = Process.Start(psi); if (proc == null) { _logger.LogWarning("Failed to start btrfs defragment for compression of {file}", path); return false; } string stdout = proc.StandardOutput.ReadToEnd(); string stderr = proc.StandardError.ReadToEnd(); try { proc.WaitForExit(); } catch (Exception ex) { _logger.LogTrace(ex, "Process.WaitForExit threw under Wine for {file}", path); } if (proc.ExitCode != 0) { _logger.LogWarning("btrfs defragment failed for {file}: {stderr}", path, stderr); return false; } if (!string.IsNullOrWhiteSpace(stdout)) _logger.LogTrace("btrfs defragment output for {file}: {stdout}", path, stdout.Trim()); _logger.LogInformation("Compressed btrfs file successfully: {file}", path); return true; } catch (Exception ex) { _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); return false; } } private void EnqueueCompaction(string filePath) { if (!_pendingCompactions.TryAdd(filePath, 0)) return; var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) { _logger.LogTrace("Skip enqueue (unsupported fs) {fs} {file}", fsType, filePath); _pendingCompactions.TryRemove(filePath, out _); return; } if (!_compactionQueue.Writer.TryWrite(filePath)) { _pendingCompactions.TryRemove(filePath, out _); _logger.LogDebug("Failed to enqueue compaction {file}", filePath); } } private async Task ProcessQueueAsync(CancellationToken token) { try { while (await _compactionQueue.Reader.WaitToReadAsync(token).ConfigureAwait(false)) { while (_compactionQueue.Reader.TryRead(out var filePath)) { try { if (token.IsCancellationRequested) { return; } if (!_lightlessConfigService.Current.UseCompactor) { continue; } if (!File.Exists(filePath)) { _logger.LogTrace("Skip compact (missing) {file}", filePath); continue; } CompactFile(filePath); } catch (OperationCanceledException) { return; } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting file {file}", filePath); } finally { _pendingCompactions.TryRemove(filePath, out _); } } } } catch (OperationCanceledException) { _logger.LogDebug("Compaction queue cancelled"); } } [StructLayout(LayoutKind.Sequential, Pack = 1)] private struct WOF_FILE_COMPRESSION_INFO_V1 { public int Algorithm; public ulong Flags; } [DllImport("kernel32.dll", SetLastError = true)] private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped); [DllImport("kernel32.dll")] private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh); [DllImport("WofUtil.dll")] private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength); [DllImport("WofUtil.dll", SetLastError = true)] private static extern int WofSetFileDataLocation(SafeFileHandle FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); public void Dispose() { _compactionQueue.Writer.TryComplete(); _compactionCts.Cancel(); try { _compactionWorker.Wait(TimeSpan.FromSeconds(5)); } catch { //ignore on catch ^^ } finally { _compactionCts.Dispose(); } GC.SuppressFinalize(this); } }