using LightlessSync.LightlessConfiguration; using LightlessSync.Services; using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.InteropServices; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; namespace LightlessSync.FileCache; public sealed class FileCompactor : IDisposable { public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U; public const ulong WOF_PROVIDER_FILE = 2UL; public const int _maxRetries = 3; private readonly ConcurrentDictionary _pendingCompactions; private readonly ILogger _logger; private readonly LightlessConfigService _lightlessConfigService; private readonly DalamudUtilService _dalamudUtilService; private readonly Channel _compactionQueue; private readonly CancellationTokenSource _compactionCts = new(); private readonly Task _compactionWorker; private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo = new() { Algorithm = (int)CompressionAlgorithm.XPRESS8K, Flags = 0 }; [StructLayout(LayoutKind.Sequential, Pack = 1)] private struct WOF_FILE_COMPRESSION_INFO_V1 { public int Algorithm; public ulong Flags; } private enum CompressionAlgorithm { NO_COMPRESSION = -2, LZNT1 = -1, XPRESS4K = 0, LZX = 1, XPRESS8K = 2, XPRESS16K = 3 } public FileCompactor(ILogger logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService) { _pendingCompactions = new(StringComparer.OrdinalIgnoreCase); _logger = logger; _lightlessConfigService = lightlessConfigService; _dalamudUtilService = dalamudUtilService; _compactionQueue = Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); _compactionWorker = Task.Factory.StartNew(() => ProcessQueueAsync(_compactionCts.Token), _compactionCts.Token, TaskCreationOptions.LongRunning,TaskScheduler.Default).Unwrap(); } public bool MassCompactRunning { get; private set; } public string Progress { get; private set; } = string.Empty; /// /// Compact the storage of the Cache Folder /// /// Used to check if files needs to be compressed public void CompactStorage(bool compress) { MassCompactRunning = true; try { var allFiles = Directory.EnumerateFiles(_lightlessConfigService.Current.CacheFolder).ToList(); int total = allFiles.Count; int current = 0; foreach (var file in allFiles) { current++; Progress = $"{current}/{total}"; try { // Compress or decompress files if (compress) CompactFile(file); else DecompressFile(file); } catch (IOException ioEx) { _logger.LogDebug(ioEx, "File {file} locked or busy, skipping", file); } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting/decompressing file {file}", file); } } } finally { MassCompactRunning = false; Progress = string.Empty; } } /// /// Write all bytes into a directory async /// /// Bytes will be writen to this filepath /// Bytes that have to be written /// Cancellation Token for interupts /// Writing Task public async Task WriteAllBytesAsync(string filePath, byte[] bytes, CancellationToken token) { var dir = Path.GetDirectoryName(filePath); if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) Directory.CreateDirectory(dir); await File.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false); if (_lightlessConfigService.Current.UseCompactor) EnqueueCompaction(filePath); } /// /// Gets the File size for an BTRFS or NTFS file system for the given FileInfo /// /// Amount of blocks used in the disk public long GetFileSizeOnDisk(FileInfo fileInfo) { var fsType = GetFilesystemType(fileInfo.FullName, _dalamudUtilService.IsWine); if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { (bool flowControl, long value) = GetFileSizeNTFS(fileInfo); if (!flowControl) { return value; } } if (fsType == FilesystemType.Btrfs) { (bool flowControl, long value) = GetFileSizeBtrfs(fileInfo); if (!flowControl) { return value; } } return fileInfo.Length; } /// /// Get File Size in an Btrfs file system (Linux/Wine). /// /// File that you want the size from. /// Succesful check and value of the filesize. /// Fails on the Process in StartProcessInfo private (bool flowControl, long value) GetFileSizeBtrfs(FileInfo fileInfo) { try { bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = isWine ? ToLinuxPathIfWine(fileInfo.FullName, isWine) : fileInfo.FullName; var fileName = "stat"; var arguments = $"-c %b \"{realPath}\""; (bool processControl, bool success) = StartProcessInfo(realPath, fileName, arguments, out Process? proc, out string stdout); if (!processControl && !success) throw new InvalidOperationException($"stat failed: {proc}"); if (!long.TryParse(stdout.Trim(), out var blocks)) throw new InvalidOperationException($"invalid stat output: {stdout}"); // st_blocks are always 512-byte on Linux enviroment. return (flowControl: false, value: blocks * 512L); } catch (Exception ex) { _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); } return (flowControl: true, value: default); } /// /// Get File Size in an NTFS file system (Windows). /// /// File that you want the size from. /// Succesful check and value of the filesize. private (bool flowControl, long value) GetFileSizeNTFS(FileInfo fileInfo) { try { var blockSize = GetBlockSizeForPath(fileInfo.FullName, _logger, _dalamudUtilService.IsWine); var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); var size = (long)hosize << 32 | losize; return (flowControl: false, value: ((size + blockSize - 1) / blockSize) * blockSize); } catch (Exception ex) { _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); } return (flowControl: true, value: default); } /// /// Compressing the given path with BTRFS or NTFS file system. /// /// Path of the decompressed/normal file private void CompactFile(string filePath) { var fi = new FileInfo(filePath); if (!fi.Exists) { _logger.LogTrace("Skip compact: missing {file}", filePath); return; } var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); _logger.LogTrace("Detected filesystem {fs} for {file} (isWine={wine})", fsType, filePath, _dalamudUtilService.IsWine); var oldSize = fi.Length; int blockSize = GetBlockSizeForPath(fi.FullName, _logger, _dalamudUtilService.IsWine); if (oldSize < Math.Max(blockSize, 8 * 1024)) { _logger.LogTrace("Skip compact: {file} < block {block}", filePath, blockSize); return; } if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { if (!IsWOFCompactedFile(filePath)) { _logger.LogDebug("NTFS compact XPRESS8K: {file}", filePath); if (WOFCompressFile(filePath)) { var newSize = GetFileSizeOnDisk(fi); _logger.LogDebug("NTFS compressed {file} {old} -> {new}", filePath, oldSize, newSize); } else { _logger.LogWarning("NTFS compression failed or unavailable for {file}", filePath); } } else { _logger.LogTrace("Already NTFS-compressed: {file}", filePath); } return; } if (fsType == FilesystemType.Btrfs) { if (!IsBtrfsCompressedFile(filePath)) { _logger.LogDebug("Btrfs compress zstd: {file}", filePath); if (BtrfsCompressFile(filePath)) { var newSize = GetFileSizeOnDisk(fi); _logger.LogDebug("Btrfs compressed {file} {old} -> {new}", filePath, oldSize, newSize); } else { _logger.LogWarning("Btrfs compression failed or unavailable for {file}", filePath); } } else { _logger.LogTrace("Already Btrfs-compressed: {file}", filePath); } return; } _logger.LogTrace("Skip compact: unsupported FS for {file}", filePath); } /// /// Decompressing the given path with BTRFS file system or NTFS file system. /// /// Path of the compressed file private void DecompressFile(string path) { _logger.LogDebug("Decompress request: {file}", path); var fsType = GetFilesystemType(path, _dalamudUtilService.IsWine); if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { try { bool flowControl = DecompressWOFFile(path, out FileStream fs); if (!flowControl) { return; } } catch (Exception ex) { _logger.LogWarning(ex, "NTFS decompress error {file}", path); } } if (fsType == FilesystemType.Btrfs) { try { bool flowControl = DecompressBtrfsFile(path); if (!flowControl) { return; } } catch (Exception ex) { _logger.LogWarning(ex, "Btrfs decompress error {file}", path); } } } /// /// Decompress an BTRFS File /// /// Path of the compressed file /// Decompressing state private bool DecompressBtrfsFile(string path) { var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); try { bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; var mountOptions = GetMountOptionsForPath(realPath); if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) { _logger.LogWarning( "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " + "Remount with 'compress=no' before running decompression.", realPath, mountOptions); return false; } if (!IsBtrfsCompressedFile(realPath)) { _logger.LogTrace("File {file} is not compressed, skipping decompression.", realPath); return true; } (bool flowControl, bool value) = FileStreamOpening(realPath, ref fs); if (!flowControl) { return value; } string fileName = isWine ? "/bin/bash" : "btrfs"; string command = isWine ? $"-c \"filesystem defragment -- \"{realPath}\"\"" : $"filesystem defragment -- \"{realPath}\""; (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); if (!processControl && !success) { return value; } if (!string.IsNullOrWhiteSpace(stdout)) _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); _logger.LogInformation("Decompressed btrfs file successfully: {file}", realPath); return true; } catch (Exception ex) { _logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path); return false; } } /// /// Decompress an NTFS File /// /// Path of the compressed file /// Decompressing state private bool DecompressWOFFile(string path, out FileStream fs) { fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); var handle = fs.SafeFileHandle; if (handle.IsInvalid) { _logger.LogWarning("Invalid handle: {file}", path); return false; } if (!DeviceIoControl(handle, FSCTL_DELETE_EXTERNAL_BACKING, IntPtr.Zero, 0, IntPtr.Zero, 0, out _, IntPtr.Zero)) { int err = Marshal.GetLastWin32Error(); if (err == 342) { _logger.LogTrace("File {file} not externally backed (already decompressed)", path); } else { _logger.LogWarning("DeviceIoControl failed for {file} with Win32 error {err}", path, err); } } else { _logger.LogTrace("Successfully decompressed NTFS file {file}", path); } return true; } /// /// Converts to Linux Path if its using Wine (diferent pathing system in Wine) /// /// Path that has to be converted /// Extra check if using the wine enviroment /// Converted path to be used in Linux private string ToLinuxPathIfWine(string path, bool isWine) { if (!IsProbablyWine() && !isWine) return path; string linuxPath = path; if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) linuxPath = "/" + path[3..].Replace('\\', '/'); else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) linuxPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/'); _logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", linuxPath); return linuxPath; } /// /// Compress an WOF File /// /// Path of the decompressed/normal file /// Compessing state private bool WOFCompressFile(string path) { FileStream? fs = null; int size = Marshal.SizeOf(); IntPtr efInfoPtr = Marshal.AllocHGlobal(size); try { Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: false); ulong length = (ulong)size; (bool flowControl, bool value) = FileStreamOpening(path, ref fs); if (!flowControl) { return value; } if (fs == null) { _logger.LogWarning("Failed to open {file} for compression; skipping", path); return false; } var handle = fs.SafeFileHandle; if (handle.IsInvalid) { _logger.LogWarning("Invalid file handle for {file}", path); return false; } int ret = WofSetFileDataLocation(handle, WOF_PROVIDER_FILE, efInfoPtr, length); // 0x80070158 is WOF error whenever compression fails in an non-fatal way. if (ret != 0 && ret != unchecked((int)0x80070158)) { _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); return false; } return true; } catch (DllNotFoundException ex) { _logger.LogTrace(ex, "WofUtil.dll not available, this DLL is needed for compression; skipping NTFS compaction for {file}", path); return false; } catch (EntryPointNotFoundException ex) { _logger.LogTrace(ex, "WOF entrypoint missing (Wine/older OS); skipping NTFS compaction for {file}", path); return false; } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting file {path}", path); return false; } finally { fs?.Dispose(); if (efInfoPtr != IntPtr.Zero) { Marshal.FreeHGlobal(efInfoPtr); } } } /// /// Checks if an File is compacted with WOF compression /// /// Path of the file /// State of the file private static bool IsWOFCompactedFile(string filePath) { try { uint buf = (uint)Marshal.SizeOf(); int result = WofIsExternalFile(filePath, out int isExternal, out _, out var info, ref buf); if (result != 0 || isExternal == 0) return false; return info.Algorithm == (int)CompressionAlgorithm.XPRESS8K || info.Algorithm == (int)CompressionAlgorithm.XPRESS4K || info.Algorithm == (int)CompressionAlgorithm.XPRESS16K || info.Algorithm == (int)CompressionAlgorithm.LZX || info.Algorithm == (int)CompressionAlgorithm.LZNT1 || info.Algorithm == (int)CompressionAlgorithm.NO_COMPRESSION; } catch { return false; } } /// /// Checks if an File is compacted with Btrfs compression /// /// Path of the file /// State of the file private bool IsBtrfsCompressedFile(string path) { try { bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; var fi = new FileInfo(realPath); if (fi == null) { _logger.LogWarning("Failed to open {file} for checking on compression; skipping", realPath); return false; } string fileName = isWine ? "/bin/bash" : "filefrag"; string command = isWine ? $"-c \"filefrag -v '{EscapeSingle(realPath)}'\"" : $"-v \"{realPath}\""; (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); if (!processControl && !success) { return success; } bool compressed = stdout.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); _logger.LogTrace("Btrfs compression check for {file}: {compressed}", realPath, compressed); return compressed; } catch (Exception ex) { _logger.LogDebug(ex, "Failed to detect Btrfs compression for {file}", path); return false; } } /// /// Compress an Btrfs File /// /// Path of the decompressed/normal file /// Compessing state private bool BtrfsCompressFile(string path) { FileStream? fs = null; try { bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; var fi = new FileInfo(realPath); if (fi == null) { _logger.LogWarning("Failed to open {file} for compression; skipping", realPath); return false; } if (IsBtrfsCompressedFile(realPath)) { _logger.LogTrace("File {file} already compressed (Btrfs), skipping file", realPath); return true; } (bool flowControl, bool value) = FileStreamOpening(realPath, ref fs); if (!flowControl) { return value; } string fileName = isWine ? "/bin/bash" : "btrfs"; string command = isWine ? $"-c \"btrfs filesystem defragment -czstd:1 -- \"{realPath}\"\"" : $"btrfs filesystem defragment -czstd:1 -- \"{realPath}\""; (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); if (!processControl && !success) { return value; } if (!string.IsNullOrWhiteSpace(stdout)) _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); _logger.LogInformation("Compressed btrfs file successfully: {file}", realPath); return true; } catch (Exception ex) { _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); return false; } } /// /// Trying opening file stream in certain amount of tries. /// /// Path where the file is located /// Filestream used for the function /// State of the filestream opening private (bool flowControl, bool value) FileStreamOpening(string path, ref FileStream? fs) { for (int attempt = 0; attempt < _maxRetries; attempt++) { try { fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); break; } catch (IOException) { if (attempt == _maxRetries - 1) { _logger.LogWarning("File still in use after {attempts} attempts, skipping compression for {file}", _maxRetries, path); return (flowControl: false, value: false); } int delay = 150 * (attempt + 1); _logger.LogTrace("File in use, retrying in {delay}ms for {file}", delay, path); Thread.Sleep(delay); } } return (flowControl: true, value: default); } /// /// Starts an process with given Filename and Arguments /// /// Path you want to use for the process (Compression is using these) /// File of the command /// Arguments used for the command /// Returns process of the given command /// Returns output of the given command /// Returns if the process been done succesfully or not private (bool processControl, bool success) StartProcessInfo(string path, string fileName, string arguments, out Process? proc, out string stdout) { var psi = new ProcessStartInfo { FileName = fileName, Arguments = arguments, RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true, WorkingDirectory = "/" }; proc = Process.Start(psi); if (proc == null) { _logger.LogWarning("Failed to start {arguments} for {file}", arguments, path); stdout = string.Empty; return (processControl: false, success: false); } stdout = proc.StandardOutput.ReadToEnd(); string stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) { _logger.LogTrace("{arguments} exited with code {code}: {stderr}", arguments, proc.ExitCode, stderr); return (processControl: false, success: false); } return (processControl: true, success: default); } private static string EscapeSingle(string p) => p.Replace("'", "'\\'", StringComparison.Ordinal); private void EnqueueCompaction(string filePath) { // Safe-checks if (string.IsNullOrWhiteSpace(filePath)) return; if (!_lightlessConfigService.Current.UseCompactor) return; if (!File.Exists(filePath)) return; if (!_pendingCompactions.TryAdd(filePath, 0)) return; bool enqueued = false; try { bool isWine = _dalamudUtilService?.IsWine ?? false; var fsType = GetFilesystemType(filePath, isWine); // If under Wine, we should skip NTFS because its not Windows but might return NTFS. if (fsType == FilesystemType.NTFS && isWine) { _logger.LogTrace("Skip enqueue (NTFS under Wine) {file}", filePath); return; } // Unknown file system should be skipped. if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) { _logger.LogTrace("Skip enqueue (unsupported fs) {fs} {file}", fsType, filePath); return; } if (!_compactionQueue.Writer.TryWrite(filePath)) { _logger.LogTrace("Skip enqueue: compaction channel is closed {file}", filePath); return; } enqueued = true; _logger.LogTrace("Queued compaction for {file} (fs={fs})", filePath, fsType); } finally { if (!enqueued) _pendingCompactions.TryRemove(filePath, out _); } } private async Task ProcessQueueAsync(CancellationToken token) { try { while (await _compactionQueue.Reader.WaitToReadAsync(token).ConfigureAwait(false)) { while (_compactionQueue.Reader.TryRead(out var filePath)) { try { if (token.IsCancellationRequested) { return; } if (!_lightlessConfigService.Current.UseCompactor) { continue; } if (!File.Exists(filePath)) { _logger.LogTrace("Skip compact (missing) {file}", filePath); continue; } CompactFile(filePath); } catch (OperationCanceledException) { return; } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting file {file}", filePath); } finally { _pendingCompactions.TryRemove(filePath, out _); } } } } catch (OperationCanceledException) { _logger.LogDebug("Compaction queue cancelled"); } } [DllImport("kernel32.dll", SetLastError = true)] private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped); [DllImport("kernel32.dll")] private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh); [DllImport("WofUtil.dll")] private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength); [DllImport("WofUtil.dll", SetLastError = true)] private static extern int WofSetFileDataLocation(SafeFileHandle FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); public void Dispose() { _compactionQueue.Writer.TryComplete(); _compactionCts.Cancel(); try { _compactionWorker.Wait(TimeSpan.FromSeconds(5)); } catch { //ignore on catch ^^ } finally { _compactionCts.Dispose(); } GC.SuppressFinalize(this); } }