using LightlessSync.LightlessConfiguration; using LightlessSync.Services; using LightlessSync.Utils; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.InteropServices; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; namespace LightlessSync.FileCache; public sealed class FileCompactor : IDisposable { public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U; public const ulong WOF_PROVIDER_FILE = 2UL; private readonly Dictionary _clusterSizes; private readonly ConcurrentDictionary _pendingCompactions; private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo; private readonly ILogger _logger; private readonly LightlessConfigService _lightlessConfigService; private readonly DalamudUtilService _dalamudUtilService; private readonly Channel _compactionQueue; private readonly CancellationTokenSource _compactionCts = new(); private readonly Task _compactionWorker; public FileCompactor(ILogger logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService) { _clusterSizes = new(StringComparer.Ordinal); _pendingCompactions = new(StringComparer.OrdinalIgnoreCase); _logger = logger; _lightlessConfigService = lightlessConfigService; _dalamudUtilService = dalamudUtilService; _efInfo = new WOF_FILE_COMPRESSION_INFO_V1 { Algorithm = CompressionAlgorithm.XPRESS8K, Flags = 0 }; _compactionQueue = Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); _compactionWorker = Task.Factory.StartNew( () => ProcessQueueAsync(_compactionCts.Token), _compactionCts.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default) .Unwrap(); } private enum CompressionAlgorithm { NO_COMPRESSION = -2, LZNT1 = -1, XPRESS4K = 0, LZX = 1, XPRESS8K = 2, XPRESS16K = 3 } public bool MassCompactRunning { get; private set; } = false; public string Progress { get; private set; } = string.Empty; public void CompactStorage(bool compress) { MassCompactRunning = true; int currentFile = 1; var allFiles = Directory.EnumerateFiles(_lightlessConfigService.Current.CacheFolder).ToList(); int allFilesCount = allFiles.Count; foreach (var file in allFiles) { Progress = $"{currentFile}/{allFilesCount}"; if (compress) CompactFile(file); else DecompressFile(file); currentFile++; } MassCompactRunning = false; } public long GetFileSizeOnDisk(FileInfo fileInfo) { var fsType = GetFilesystemType(fileInfo.FullName, _dalamudUtilService.IsWine); if (fsType != FilesystemType.Btrfs && fsType != FilesystemType.NTFS) { return fileInfo.Length; } if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { return GetFileSizeOnDisk(fileInfo, GetClusterSize); } if (fsType == FilesystemType.Btrfs) { try { long blocks = RunStatGetBlocks(fileInfo.FullName); //st_blocks are always calculated in 512-byte units, hence we use 512L return blocks * 512L; } catch (Exception ex) { _logger.LogDebug(ex, "Failed to get on-disk size via stat for {file}, falling back to Length", fileInfo.FullName); return fileInfo.Length; } } return fileInfo.Length; } public async Task WriteAllBytesAsync(string filePath, byte[] decompressedFile, CancellationToken token) { var dir = Path.GetDirectoryName(filePath); if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) Directory.CreateDirectory(dir); await File.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false); if (!_lightlessConfigService.Current.UseCompactor) return; EnqueueCompaction(filePath); } public void Dispose() { _compactionQueue.Writer.TryComplete(); _compactionCts.Cancel(); try { if (!_compactionWorker.Wait(TimeSpan.FromSeconds(5))) { _logger.LogDebug("Compaction worker did not shut down within timeout"); } } catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogDebug(ex, "Error shutting down compaction worker"); } finally { _compactionCts.Dispose(); } GC.SuppressFinalize(this); } [DllImport("libc", SetLastError = true)] private static extern int statvfs(string path, out Statvfs buf); [StructLayout(LayoutKind.Sequential)] private struct Statvfs { public ulong f_bsize; /* Filesystem block size */ public ulong f_frsize; /* Fragment size */ public ulong f_blocks; /* Size of fs in f_frsize units */ public ulong f_bfree; /* Number of free blocks */ public ulong f_bavail; /* Number of free blocks for unprivileged users */ public ulong f_files; /* Number of inodes */ public ulong f_ffree; /* Number of free inodes */ public ulong f_favail; /* Number of free inodes for unprivileged users */ public ulong f_fsid; /* Filesystem ID */ public ulong f_flag; /* Mount flags */ public ulong f_namemax; /* Maximum filename length */ } private static int GetLinuxBlockSize(string path) { try { int result = statvfs(path, out var buf); if (result != 0) return -1; //return fragment size of Linux file system return (int)buf.f_frsize; } catch { return -1; } } [DllImport("kernel32.dll")] private static extern int DeviceIoControl(IntPtr hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out IntPtr lpBytesReturned, out IntPtr lpOverlapped); [DllImport("kernel32.dll")] private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh); [DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)] private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters, out uint lpTotalNumberOfClusters); [DllImport("WoFUtil.dll")] private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength); [DllImport("WofUtil.dll")] private static extern int WofSetFileDataLocation(IntPtr FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); private void CompactFile(string filePath) { var fi = new FileInfo(filePath); if (!fi.Exists) { _logger.LogDebug("Skipping compaction for missing file {file}", filePath); return; } var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); var oldSize = fi.Length; int clusterSize = GetClusterSize(fi); if (oldSize < Math.Max(clusterSize, 8 * 1024)) { _logger.LogDebug("File {file} is smaller than cluster size ({size}), ignoring", filePath, clusterSize); return; } // NTFS Compression. if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { if (!IsWOFCompactedFile(filePath)) { _logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath); var success = WOFCompressFile(filePath); if (success) { var newSize = GetFileSizeOnDisk(fi); _logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); } else { _logger.LogWarning("NTFS compression failed or not available for {file}", filePath); } } else { _logger.LogDebug("File {file} already compressed (NTFS)", filePath); } } // BTRFS Compression if (fsType == FilesystemType.Btrfs) { if (!IsBtrfsCompressedFile(filePath)) { _logger.LogDebug("Attempting btrfs compression for {file}", filePath); var success = BtrfsCompressFile(filePath); if (success) { var newSize = GetFileSizeOnDisk(fi); _logger.LogDebug("Btrfs-compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); } else { _logger.LogWarning("Btrfs compression failed or not available for {file}", filePath); } } else { _logger.LogDebug("File {file} already compressed (Btrfs)", filePath); } } } private static long GetFileSizeOnDisk(FileInfo fileInfo, Func getClusterSize) { int clusterSize = getClusterSize(fileInfo); if (clusterSize <= 0) return fileInfo.Length; uint low = GetCompressedFileSizeW(fileInfo.FullName, out uint high); long compressed = ((long)high << 32) | low; return ((compressed + clusterSize - 1) / clusterSize) * clusterSize; } private static long RunStatGetBlocks(string path) { var psi = new ProcessStartInfo("stat", $"-c %b \"{path}\"") { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true }; using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat process"); var outp = proc.StandardOutput.ReadToEnd(); var err = proc.StandardError.ReadToEnd(); proc.WaitForExit(); if (proc.ExitCode != 0) { throw new InvalidOperationException($"stat failed: {err}"); } if (!long.TryParse(outp.Trim(), out var blocks)) { throw new InvalidOperationException($"invalid stat output: {outp}"); } return blocks; } private void DecompressFile(string path) { _logger.LogDebug("Removing compression from {file}", path); var fsType = GetFilesystemType(path, _dalamudUtilService.IsWine); //NTFS Decompression if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { try { using var fs = new FileStream(path, FileMode.Open); #pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called var hDevice = fs.SafeFileHandle.DangerousGetHandle(); #pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _); } catch (Exception ex) { _logger.LogWarning(ex, "Error decompressing file {path}", path); } return; } //BTRFS Decompression if (fsType == FilesystemType.Btrfs) { try { var mountOptions = GetMountOptionsForPath(path); if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) { _logger.LogWarning("Cannot safely decompress {file}: filesystem mounted with compression ({opts}). Remount with 'compress=no' before running decompression.", path, mountOptions); return; } _logger.LogDebug("Rewriting {file} to remove btrfs compression...", path); var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -- \"{path}\"") { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true }; using var proc = Process.Start(psi); if (proc == null) { _logger.LogWarning("Failed to start btrfs defragment for decompression of {file}", path); return; } //End stream of process to read the files var stdout = proc.StandardOutput.ReadToEnd(); var stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); if (proc.ExitCode != 0) { _logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr); } else { if (!string.IsNullOrWhiteSpace(stdout)) _logger.LogDebug("btrfs defragment output for {file}: {out}", path, stdout.Trim()); _logger.LogInformation("Decompressed (rewritten uncompressed) btrfs file: {file}", path); } } catch (Exception ex) { _logger.LogWarning(ex, "Error rewriting {file} for decompression", path); } } } private int GetClusterSize(FileInfo fi) { try { if (!fi.Exists) return -1; var root = fi.Directory?.Root.FullName; if (string.IsNullOrEmpty(root)) return -1; root = root.ToLowerInvariant(); if (_clusterSizes.TryGetValue(root, out int cached)) return cached; _logger.LogDebug("Determining cluster/block size for {path} (root: {root})", fi.FullName, root); int clusterSize; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !_dalamudUtilService.IsWine) { int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _); if (result == 0) { _logger.LogWarning("GetDiskFreeSpaceW failed for {root}", root); return -1; } clusterSize = (int)(sectorsPerCluster * bytesPerSector); } else { clusterSize = GetLinuxBlockSize(root); if (clusterSize <= 0) { _logger.LogWarning("Failed to determine block size for {root}", root); return -1; } } _clusterSizes[root] = clusterSize; _logger.LogDebug("Determined cluster/block size for {root}: {cluster} bytes", root, clusterSize); return clusterSize; } catch (Exception ex) { _logger.LogWarning(ex, "Error determining cluster size for {file}", fi.FullName); return -1; } } public static bool UseSafeHandle(SafeHandle handle, Func action) { bool addedRef = false; try { handle.DangerousAddRef(ref addedRef); IntPtr ptr = handle.DangerousGetHandle(); return action(ptr); } finally { if (addedRef) handle.DangerousRelease(); } } private static bool IsWOFCompactedFile(string filePath) { uint buf = 8; _ = WofIsExternalFile(filePath, out int isExtFile, out uint _, out var info, ref buf); if (isExtFile == 0) return false; return info.Algorithm == CompressionAlgorithm.XPRESS8K; } private bool IsBtrfsCompressedFile(string path) { try { var psi = new ProcessStartInfo("filefrag", $"-v \"{path}\"") { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true }; using var proc = Process.Start(psi); if (proc == null) { _logger.LogWarning("Failed to start filefrag for {file}", path); return false; } string output = proc.StandardOutput.ReadToEnd(); proc.WaitForExit(); bool compressed = output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); _logger.LogTrace("Btrfs compression check for {file}: {compressed}", path, compressed); return compressed; } catch (Exception ex) { _logger.LogDebug(ex, "Failed to detect btrfs compression for {file}", path); return false; } } private bool WOFCompressFile(string path) { var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo)); Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: true); ulong length = (ulong)Marshal.SizeOf(_efInfo); try { using var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None); var handle = fs.SafeFileHandle; if (handle.IsInvalid) { _logger.LogWarning("Invalid file handle to {file}", path); return false; } return UseSafeHandle(handle, hFile => { int ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length); if (ret != 0 && ret != unchecked((int)0x80070158)) { _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); return false; } return true; }); } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting file {path}", path); return false; } finally { Marshal.FreeHGlobal(efInfoPtr); } } private bool BtrfsCompressFile(string path) { try { var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -czstd -- \"{path}\"") { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true }; using var proc = Process.Start(psi); if (proc == null) { _logger.LogWarning("Failed to start btrfs process for {file}", path); return false; } var stdout = proc.StandardOutput.ReadToEnd(); var stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); if (proc.ExitCode != 0) { _logger.LogWarning("btrfs defrag returned {code} for {file}: {err}", proc.ExitCode, path, stderr); return false; } _logger.LogDebug("btrfs output: {out}", stdout); return true; } catch (Exception ex) { _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); return false; } } private struct WOF_FILE_COMPRESSION_INFO_V1 { public CompressionAlgorithm Algorithm; public ulong Flags; } private void EnqueueCompaction(string filePath) { if (!_pendingCompactions.TryAdd(filePath, 0)) return; var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) { _logger.LogTrace("Skipping compaction enqueue for unsupported filesystem {fs} ({file})", fsType, filePath); _pendingCompactions.TryRemove(filePath, out _); return; } if (!_compactionQueue.Writer.TryWrite(filePath)) { _pendingCompactions.TryRemove(filePath, out _); _logger.LogDebug("Failed to enqueue compaction job for {file}", filePath); } else { _logger.LogTrace("Queued compaction job for {file} (fs={fs})", filePath, fsType); } } private async Task ProcessQueueAsync(CancellationToken token) { try { while (await _compactionQueue.Reader.WaitToReadAsync(token).ConfigureAwait(false)) { while (_compactionQueue.Reader.TryRead(out var filePath)) { try { if (token.IsCancellationRequested) { return; } if (!_lightlessConfigService.Current.UseCompactor) { continue; } if (!File.Exists(filePath)) { _logger.LogTrace("Skipping compaction for missing file {file}", filePath); continue; } CompactFile(filePath); } catch (OperationCanceledException) { return; } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting file {file}", filePath); } finally { _pendingCompactions.TryRemove(filePath, out _); } } } } catch (OperationCanceledException) { _logger.LogDebug("Queue has been cancelled by token"); } } }