diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 64910f3..85fb30e 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -137,7 +137,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase if (fsType == FileSystemHelper.FilesystemType.Btrfs) { StorageIsBtrfs = true; - Logger.LogInformation("Lightless Storage is on BTRFS drive: {isNtfs}", StorageIsBtrfs); + Logger.LogInformation("Lightless Storage is on BTRFS drive: {isBtrfs}", StorageIsBtrfs); } Logger.LogDebug("Initializing Lightless FSW on {path}", lightlessPath); @@ -661,44 +661,44 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase if (ct.IsCancellationRequested) return; - // scan new files - if (allScannedFiles.Any(c => !c.Value)) + var newFiles = allScannedFiles.Where(c => !c.Value).Select(c => c.Key).ToList(); + foreach (var cachePath in newFiles) { - Parallel.ForEach(allScannedFiles.Where(c => !c.Value).Select(c => c.Key), - new ParallelOptions() - { - MaxDegreeOfParallelism = threadCount, - CancellationToken = ct - }, (cachePath) => - { - if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null) - { - Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}", _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null); - return; - } + if (ct.IsCancellationRequested) break; + ProcessOne(cachePath); + Interlocked.Increment(ref _currentFileProgress); + } - if (ct.IsCancellationRequested) return; + Logger.LogTrace("Scanner added {count} new files to db", newFiles.Count); - if (!_ipcManager.Penumbra.APIAvailable) - { - Logger.LogWarning("Penumbra not available"); - return; - } + void ProcessOne(string? cachePath) + { + if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null) + { + Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}", + _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null); + return; + } - try - { - var entry = _fileDbManager.CreateFileEntry(cachePath); - if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath); - } - catch (Exception ex) - { - Logger.LogWarning(ex, "Failed adding {file}", cachePath); - } + if (!_ipcManager.Penumbra.APIAvailable) + { + Logger.LogWarning("Penumbra not available"); + return; + } - Interlocked.Increment(ref _currentFileProgress); - }); - - Logger.LogTrace("Scanner added {notScanned} new files to db", allScannedFiles.Count(c => !c.Value)); + try + { + var entry = _fileDbManager.CreateFileEntry(cachePath); + if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath); + } + catch (IOException ioex) + { + Logger.LogDebug(ioex, "File busy or locked: {file}", cachePath); + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Failed adding {file}", cachePath); + } } Logger.LogDebug("Scan complete"); diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 5f05549..a917b59 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,6 +1,7 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; using Microsoft.Extensions.Logging; +using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.InteropServices; @@ -14,44 +15,21 @@ public sealed class FileCompactor : IDisposable public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U; public const ulong WOF_PROVIDER_FILE = 2UL; - private readonly Dictionary _clusterSizes; private readonly ConcurrentDictionary _pendingCompactions; - private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo; private readonly ILogger _logger; - private readonly LightlessConfigService _lightlessConfigService; private readonly DalamudUtilService _dalamudUtilService; + private readonly Channel _compactionQueue; private readonly CancellationTokenSource _compactionCts = new(); private readonly Task _compactionWorker; - - public FileCompactor(ILogger logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService) + + private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo = new() { - _clusterSizes = new(StringComparer.Ordinal); - _pendingCompactions = new(StringComparer.OrdinalIgnoreCase); - _logger = logger; - _lightlessConfigService = lightlessConfigService; - _dalamudUtilService = dalamudUtilService; - _efInfo = new WOF_FILE_COMPRESSION_INFO_V1 - { - Algorithm = CompressionAlgorithm.XPRESS8K, - Flags = 0 - }; - - _compactionQueue = Channel.CreateUnbounded(new UnboundedChannelOptions - { - SingleReader = true, - SingleWriter = false - }); - _compactionWorker = Task.Factory.StartNew( - () => ProcessQueueAsync(_compactionCts.Token), - _compactionCts.Token, - TaskCreationOptions.LongRunning, - TaskScheduler.Default) - .Unwrap(); - } - - private enum CompressionAlgorithm + Algorithm = (int)CompressionAlgorithm.XPRESS8K, + Flags = 0 + }; + private enum CompressionAlgorithm { NO_COMPRESSION = -2, LZNT1 = -1, @@ -61,493 +39,500 @@ public sealed class FileCompactor : IDisposable XPRESS16K = 3 } - public bool MassCompactRunning { get; private set; } = false; + public FileCompactor(ILogger logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService) + { + _pendingCompactions = new(StringComparer.OrdinalIgnoreCase); + _logger = logger; + _lightlessConfigService = lightlessConfigService; + _dalamudUtilService = dalamudUtilService; + _compactionQueue = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = true, + SingleWriter = false + }); + + _compactionWorker = Task.Factory.StartNew(() => ProcessQueueAsync(_compactionCts.Token), _compactionCts.Token, TaskCreationOptions.LongRunning,TaskScheduler.Default).Unwrap(); + } + + public bool MassCompactRunning { get; private set; } public string Progress { get; private set; } = string.Empty; public void CompactStorage(bool compress) { MassCompactRunning = true; - - int currentFile = 1; - var allFiles = Directory.EnumerateFiles(_lightlessConfigService.Current.CacheFolder).ToList(); - int allFilesCount = allFiles.Count; - foreach (var file in allFiles) + try { - Progress = $"{currentFile}/{allFilesCount}"; - if (compress) - CompactFile(file); - else - DecompressFile(file); - currentFile++; - } + var allFiles = Directory.EnumerateFiles(_lightlessConfigService.Current.CacheFolder).ToList(); + int total = allFiles.Count; + int current = 0; - MassCompactRunning = false; + foreach (var file in allFiles) + { + current++; + Progress = $"{current}/{total}"; + + try + { + if (compress) + CompactFile(file); + else + DecompressFile(file); + } + catch (IOException ioEx) + { + _logger.LogDebug(ioEx, "File {file} locked or busy, skipping", file); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error compacting/decompressing file {file}", file); + } + } + } + finally + { + MassCompactRunning = false; + Progress = string.Empty; + } } + /// + /// Write all bytes into a directory async + /// + /// Bytes will be writen to this filepath + /// Bytes that have to be written + /// Cancellation Token for interupts + /// Writing Task + public async Task WriteAllBytesAsync(string filePath, byte[] bytes, CancellationToken token) + { + var dir = Path.GetDirectoryName(filePath); + if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) + Directory.CreateDirectory(dir); + + await File.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false); + + if (_lightlessConfigService.Current.UseCompactor) + EnqueueCompaction(filePath); + } + + /// + /// Gets the File size for an BTRFS or NTFS file system for the given FileInfo + /// + /// Amount of blocks used in the disk public long GetFileSizeOnDisk(FileInfo fileInfo) { var fsType = GetFilesystemType(fileInfo.FullName, _dalamudUtilService.IsWine); - if (fsType != FilesystemType.Btrfs && fsType != FilesystemType.NTFS) - { - return fileInfo.Length; - } - if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { - return GetFileSizeOnDisk(fileInfo, GetClusterSize); + var blockSize = GetBlockSizeForPath(fileInfo.FullName, _logger, _dalamudUtilService.IsWine); + var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); + var size = (long)hosize << 32 | losize; + return ((size + blockSize - 1) / blockSize) * blockSize; } if (fsType == FilesystemType.Btrfs) { try { - long blocks = RunStatGetBlocks(fileInfo.FullName); - //st_blocks are always calculated in 512-byte units, hence we use 512L + var realPath = fileInfo.FullName.Replace("\"", "\\\"", StringComparison.Ordinal); + var psi = new ProcessStartInfo("stat", $"-c %b \"{realPath}\"") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat"); + var outp = proc.StandardOutput.ReadToEnd(); + var err = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + + if (proc.ExitCode != 0) + throw new InvalidOperationException($"stat failed: {err}"); + + if (!long.TryParse(outp.Trim(), out var blocks)) + throw new InvalidOperationException($"invalid stat output: {outp}"); + + // st_blocks are always 512-byte on Linux enviroment. return blocks * 512L; } catch (Exception ex) { - _logger.LogDebug(ex, "Failed to get on-disk size via stat for {file}, falling back to Length", fileInfo.FullName); - return fileInfo.Length; + _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); } } return fileInfo.Length; } - public async Task WriteAllBytesAsync(string filePath, byte[] decompressedFile, CancellationToken token) - { - var dir = Path.GetDirectoryName(filePath); - if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) - Directory.CreateDirectory(dir); - - await File.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false); - - if (!_lightlessConfigService.Current.UseCompactor) - return; - - EnqueueCompaction(filePath); - } - - public void Dispose() - { - _compactionQueue.Writer.TryComplete(); - _compactionCts.Cancel(); - try - { - if (!_compactionWorker.Wait(TimeSpan.FromSeconds(5), _compactionCts.Token)) - { - _logger.LogDebug("Compaction worker did not shut down within timeout"); - } - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogDebug(ex, "Error shutting down compaction worker"); - } - finally - { - _compactionCts.Dispose(); - } - - GC.SuppressFinalize(this); - } - - [DllImport("libc", SetLastError = true)] - private static extern int statvfs(string path, out Statvfs buf); - - [StructLayout(LayoutKind.Sequential)] - private struct Statvfs - { - public ulong f_bsize; /* Filesystem block size */ - public ulong f_frsize; /* Fragment size */ - public ulong f_blocks; /* Size of fs in f_frsize units */ - public ulong f_bfree; /* Number of free blocks */ - public ulong f_bavail; /* Number of free blocks for unprivileged users */ - public ulong f_files; /* Number of inodes */ - public ulong f_ffree; /* Number of free inodes */ - public ulong f_favail; /* Number of free inodes for unprivileged users */ - public ulong f_fsid; /* Filesystem ID */ - public ulong f_flag; /* Mount flags */ - public ulong f_namemax; /* Maximum filename length */ - } - - private static int GetLinuxBlockSize(string path) - { - try - { - int result = statvfs(path, out var buf); - if (result != 0) - return -1; - - //return fragment size of Linux file system - return (int)buf.f_frsize; - } - catch - { - return -1; - } - } - - private static string ConvertWinePathToLinux(string winePath) - { - if (winePath.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - return "/" + winePath.Substring(3).Replace('\\', '/'); - if (winePath.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), - winePath.Substring(3).Replace('\\', '/')).Replace('\\', '/'); - return winePath.Replace('\\', '/'); - } - - [DllImport("kernel32.dll")] - private static extern int DeviceIoControl(IntPtr hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out IntPtr lpBytesReturned, out IntPtr lpOverlapped); - - [DllImport("kernel32.dll")] - private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, - [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh); - - [DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)] - private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, - out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters, - out uint lpTotalNumberOfClusters); - - [DllImport("WoFUtil.dll")] - private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength); - - [DllImport("WofUtil.dll")] - private static extern int WofSetFileDataLocation(IntPtr FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); - + /// + /// Compressing the given path with BTRFS or NTFS file system. + /// + /// Path of the decompressed/normal file private void CompactFile(string filePath) { var fi = new FileInfo(filePath); if (!fi.Exists) { - _logger.LogDebug("Skipping compaction for missing file {file}", filePath); + _logger.LogTrace("Skip compact: missing {file}", filePath); return; } var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); + _logger.LogTrace("Detected filesystem {fs} for {file} (isWine={wine})", fsType, filePath, _dalamudUtilService.IsWine); var oldSize = fi.Length; - int clusterSize = GetClusterSize(fi); - if (oldSize < Math.Max(clusterSize, 8 * 1024)) + int blockSize = GetBlockSizeForPath(fi.FullName, _logger, _dalamudUtilService.IsWine); + if (oldSize < Math.Max(blockSize, 8 * 1024)) { - _logger.LogDebug("File {file} is smaller than cluster size ({size}), ignoring", filePath, clusterSize); + _logger.LogTrace("Skip compact: {file} < block {block}", filePath, blockSize); return; } - // NTFS Compression. if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { if (!IsWOFCompactedFile(filePath)) { - _logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath); - var success = WOFCompressFile(filePath); - - if (success) + _logger.LogDebug("NTFS compact XPRESS8K: {file}", filePath); + if (WOFCompressFile(filePath)) { var newSize = GetFileSizeOnDisk(fi); - _logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); + _logger.LogDebug("NTFS compressed {file} {old} -> {new}", filePath, oldSize, newSize); } else { - _logger.LogWarning("NTFS compression failed or not available for {file}", filePath); + _logger.LogWarning("NTFS compression failed or unavailable for {file}", filePath); } - } else { - _logger.LogDebug("File {file} already compressed (NTFS)", filePath); - } - } - - // BTRFS Compression - if (fsType == FilesystemType.Btrfs) - { - if (!IsBtrfsCompressedFile(filePath)) - { - _logger.LogDebug("Attempting btrfs compression for {file}", filePath); - var success = BtrfsCompressFile(filePath); - - if (success) - { - var newSize = GetFileSizeOnDisk(fi); - _logger.LogDebug("Btrfs-compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); - } - else - { - _logger.LogWarning("Btrfs compression failed or not available for {file}", filePath); - } - } - else - { - _logger.LogDebug("File {file} already compressed (Btrfs)", filePath); - } - } - } - - private static long GetFileSizeOnDisk(FileInfo fileInfo, Func getClusterSize) - { - int clusterSize = getClusterSize(fileInfo); - if (clusterSize <= 0) - return fileInfo.Length; - - uint low = GetCompressedFileSizeW(fileInfo.FullName, out uint high); - long compressed = ((long)high << 32) | low; - return ((compressed + clusterSize - 1) / clusterSize) * clusterSize; - } - - private static long RunStatGetBlocks(string path) - { - var psi = new ProcessStartInfo("stat", $"-c %b \"{path}\"") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - - using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat process"); - var outp = proc.StandardOutput.ReadToEnd(); - var err = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); - if (proc.ExitCode != 0) - { - throw new InvalidOperationException($"stat failed: {err}"); - } - - if (!long.TryParse(outp.Trim(), out var blocks)) - { - throw new InvalidOperationException($"invalid stat output: {outp}"); - } - - return blocks; - } - - private void DecompressFile(string path) - { - _logger.LogDebug("Removing compression from {file}", path); - var fsType = GetFilesystemType(path, _dalamudUtilService.IsWine); - - //NTFS Decompression - if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) - { - try - { - using var fs = new FileStream(path, FileMode.Open); -#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called - var hDevice = fs.SafeFileHandle.DangerousGetHandle(); -#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called - _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Error decompressing file {path}", path); + _logger.LogTrace("Already NTFS-compressed: {file}", filePath); } return; } - //BTRFS Decompression if (fsType == FilesystemType.Btrfs) { - try + if (!IsBtrfsCompressedFile(filePath)) { - var mountOptions = GetMountOptionsForPath(path); - if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) + _logger.LogDebug("Btrfs compress zstd: {file}", filePath); + if (BtrfsCompressFile(filePath)) { - _logger.LogWarning( - "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). Remount with 'compress=no' before running decompression.", - path, mountOptions); - return; - } - - string realPath = path; - bool isWine = _dalamudUtilService?.IsWine ?? false; - if (isWine && IsProbablyWine()) - { - if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - { - realPath = "/" + path.Substring(3).Replace('\\', '/'); - } - else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - { - realPath = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.Personal), - path.Substring(3).Replace('\\', '/') - ).Replace('\\', '/'); - } - - _logger.LogTrace("Detected Wine environment. Converted path for decompression: {realPath}", realPath); - } - - string command = $"btrfs filesystem defragment -- \"{realPath}\""; - var psi = new ProcessStartInfo - { - FileName = isWine ? "/bin/bash" : "btrfs", - Arguments = isWine ? $"-c \"{command}\"" : $"filesystem defragment -- \"{realPath}\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; - - using var proc = Process.Start(psi); - if (proc == null) - { - _logger.LogWarning("Failed to start btrfs defragment for decompression of {file}", path); - return; - } - - var stdout = proc.StandardOutput.ReadToEnd(); - var stderr = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); - - if (proc.ExitCode != 0) - { - _logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr); + var newSize = GetFileSizeOnDisk(fi); + _logger.LogDebug("Btrfs compressed {file} {old} -> {new}", filePath, oldSize, newSize); } else { - if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogDebug("btrfs defragment output for {file}: {out}", path, stdout.Trim()); + _logger.LogWarning("Btrfs compression failed or unavailable for {file}", filePath); + } + } + else + { + _logger.LogTrace("Already Btrfs-compressed: {file}", filePath); + } + return; + } - _logger.LogInformation("Decompressed (rewritten uncompressed) btrfs file: {file}", path); + _logger.LogTrace("Skip compact: unsupported FS for {file}", filePath); + } + + /// + /// Decompressing the given path with BTRFS file system or NTFS file system. + /// + /// Path of the compressed file + private void DecompressFile(string path) + { + _logger.LogDebug("Decompress request: {file}", path); + var fsType = GetFilesystemType(path, _dalamudUtilService.IsWine); + + if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) + { + try + { + bool flowControl = DecompressWOFFile(path, out FileStream fs); + if (!flowControl) + { + return; } } catch (Exception ex) { - _logger.LogWarning(ex, "Error rewriting {file} for decompression", path); + _logger.LogWarning(ex, "NTFS decompress error {file}", path); + } + } + + if (fsType == FilesystemType.Btrfs) + { + try + { + bool flowControl = DecompressBtrfsFile(path); + if (!flowControl) + { + return; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Btrfs decompress error {file}", path); } } } - private int GetClusterSize(FileInfo fi) + /// + /// Decompress an BTRFS File + /// + /// Path of the compressed file + /// Decompessing state + private bool DecompressBtrfsFile(string path) { try { - if (!fi.Exists) - return -1; - - var root = fi.Directory?.Root.FullName; - if (string.IsNullOrEmpty(root)) - return -1; - - root = root.ToLowerInvariant(); - - if (_clusterSizes.TryGetValue(root, out int cached)) - return cached; - - _logger.LogDebug("Determining cluster/block size for {path} (root: {root})", fi.FullName, root); - - int clusterSize; - - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !_dalamudUtilService.IsWine) + var opts = GetMountOptionsForPath(path); + if (opts.Contains("compress", StringComparison.OrdinalIgnoreCase)) { - int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _); - - if (result == 0) - { - _logger.LogWarning("GetDiskFreeSpaceW failed for {root}", root); - return -1; - } - - clusterSize = (int)(sectorsPerCluster * bytesPerSector); - } - else - { - clusterSize = GetLinuxBlockSize(root); - if (clusterSize <= 0) - { - _logger.LogWarning("Failed to determine block size for {root}", root); - return -1; - } + _logger.LogWarning("Cannot safely decompress {file}: mount options include compression ({opts})", path, opts); + return false; } - _clusterSizes[root] = clusterSize; - _logger.LogDebug("Determined cluster/block size for {root}: {cluster} bytes", root, clusterSize); - return clusterSize; + string realPath = ToLinuxPathIfWine(path, _dalamudUtilService.IsWine); + var psi = new ProcessStartInfo + { + FileName = _dalamudUtilService.IsWine ? "/bin/bash" : "btrfs", + Arguments = _dalamudUtilService.IsWine + ? $"-c \"btrfs filesystem defragment -- '{EscapeSingle(realPath)}'\"" + : $"filesystem defragment -- \"{realPath}\"", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = "/" + }; + + using var proc = Process.Start(psi); + if (proc == null) + { + _logger.LogWarning("Failed to start btrfs defragment for {file}", path); + return false; + } + + var stdout = proc.StandardOutput.ReadToEnd(); + var stderr = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + + if (proc.ExitCode != 0) + { + _logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr); + return false; + } + + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs defragment output {file}: {out}", path, stdout.Trim()); + + _logger.LogInformation("Btrfs rewritten uncompressed: {file}", path); } catch (Exception ex) { - _logger.LogWarning(ex, "Error determining cluster size for {file}", fi.FullName); - return -1; + _logger.LogWarning(ex, "Btrfs decompress error {file}", path); } + + return true; } - public static bool UseSafeHandle(SafeHandle handle, Func action) + /// + /// Decompress an NTFS File + /// + /// Path of the compressed file + /// Decompessing state + private bool DecompressWOFFile(string path, out FileStream fs) { - bool addedRef = false; + fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + var handle = fs.SafeFileHandle; + + if (handle.IsInvalid) + { + _logger.LogWarning("Invalid handle: {file}", path); + return false; + } + + if (!DeviceIoControl(handle, FSCTL_DELETE_EXTERNAL_BACKING, + IntPtr.Zero, 0, IntPtr.Zero, 0, + out _, IntPtr.Zero)) + { + int err = Marshal.GetLastWin32Error(); + + if (err == 342) + { + _logger.LogTrace("File {file} not externally backed (already decompressed)", path); + } + else + { + _logger.LogWarning("DeviceIoControl failed for {file} with Win32 error {err}", path, err); + } + } + else + { + _logger.LogTrace("Successfully decompressed NTFS file {file}", path); + } + + return true; + } + + private static string EscapeSingle(string p) => p.Replace("'", "'\\'", StringComparison.Ordinal); + + private static string ToLinuxPathIfWine(string path, bool isWine) + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + return path; + + if (!IsProbablyWine() && !isWine) + return path; + + string realPath = path; + if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + realPath = "/" + path[3..].Replace('\\', '/'); + else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + realPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/'); + + return realPath; + } + + /// + /// Compress an WOF File + /// + /// Path of the decompressed/normal file + /// Compessing state + private bool WOFCompressFile(string path) + { + FileStream? fs = null; + int size = Marshal.SizeOf(); + IntPtr efInfoPtr = Marshal.AllocHGlobal(size); + try { - handle.DangerousAddRef(ref addedRef); - IntPtr ptr = handle.DangerousGetHandle(); - return action(ptr); + Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: false); + ulong length = (ulong)size; + + const int maxRetries = 3; + + for (int attempt = 0; attempt < maxRetries; attempt++) + { + try + { + fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + break; + } + catch (IOException) + { + if (attempt == maxRetries - 1) + { + _logger.LogWarning("File still in use after {attempts} attempts, skipping compression for {file}", maxRetries, path); + return false; + } + + int delay = 150 * (attempt + 1); + _logger.LogTrace("File in use, retrying in {delay}ms for {file}", delay, path); + Thread.Sleep(delay); + } + } + + if (fs == null) + { + _logger.LogWarning("Failed to open {file} for compression; skipping", path); + return false; + } + + var handle = fs.SafeFileHandle; + + if (handle.IsInvalid) + { + _logger.LogWarning("Invalid file handle for {file}", path); + return false; + } + + int ret = WofSetFileDataLocation(handle, WOF_PROVIDER_FILE, efInfoPtr, length); + + // 0x80070158 is WOF error whenever compression fails in an non-fatal way. + if (ret != 0 && ret != unchecked((int)0x80070158)) + { + _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); + return false; + } + + return true; + } + catch (DllNotFoundException) + { + _logger.LogTrace("WofUtil.dll not available; skipping NTFS compaction for {file}", path); + return false; + } + catch (EntryPointNotFoundException) + { + _logger.LogTrace("WOF entrypoint missing (Wine/older OS); skipping NTFS compaction for {file}", path); + return false; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error compacting file {path}", path); + return false; } finally { - if (addedRef) - handle.DangerousRelease(); + fs?.Dispose(); + + if (efInfoPtr != IntPtr.Zero) + { + Marshal.FreeHGlobal(efInfoPtr); + } } } + /// + /// Checks if an File is compacted with WOF compression + /// + /// Path of the file + /// State of the file private static bool IsWOFCompactedFile(string filePath) { try { uint buf = (uint)Marshal.SizeOf(); - int result = WofIsExternalFile(filePath, out int isExternal, out uint _, out var info, ref buf); - + int result = WofIsExternalFile(filePath, out int isExternal, out _, out var info, ref buf); if (result != 0 || isExternal == 0) return false; - return info.Algorithm == CompressionAlgorithm.XPRESS8K || info.Algorithm == CompressionAlgorithm.XPRESS4K - || info.Algorithm == CompressionAlgorithm.XPRESS16K || info.Algorithm == CompressionAlgorithm.LZX; + return info.Algorithm == (int)CompressionAlgorithm.XPRESS8K + || info.Algorithm == (int)CompressionAlgorithm.XPRESS4K + || info.Algorithm == (int)CompressionAlgorithm.XPRESS16K + || info.Algorithm == (int)CompressionAlgorithm.LZX + || info.Algorithm == (int)CompressionAlgorithm.LZNT1 + || info.Algorithm == (int)CompressionAlgorithm.NO_COMPRESSION; } - catch (DllNotFoundException) + catch { - // WofUtil.dll not available - return false; - } - catch (EntryPointNotFoundException) - { - // Running under Wine or non-NTFS systems - return false; - } - catch (Exception) - { - // Exception happened return false; } } + /// + /// Checks if an File is compacted with Btrfs compression + /// + /// Path of the file + /// State of the file private bool IsBtrfsCompressedFile(string path) { try { bool isWine = _dalamudUtilService?.IsWine ?? false; - string realPath = path; + string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; - if (isWine && IsProbablyWine()) - { - if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - { - realPath = "/" + path.Substring(3).Replace('\\', '/'); - } - else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - { - realPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), path.Substring(3).Replace('\\', '/')).Replace('\\', '/'); - } - - _logger.LogTrace("Detected Wine environment. Converted path for filefrag: {realPath}", realPath); - } - - string command = $"filefrag -v -- \"{realPath}\""; var psi = new ProcessStartInfo { FileName = isWine ? "/bin/bash" : "filefrag", - Arguments = isWine ? $"-c \"{command}\"" : $"-v -- \"{realPath}\"", + Arguments = isWine + ? $"-c \"filefrag -v '{EscapeSingle(realPath)}'\"" + : $"-v \"{realPath}\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, @@ -562,109 +547,116 @@ public sealed class FileCompactor : IDisposable return false; } - string output = proc.StandardOutput.ReadToEnd(); + string stdout = proc.StandardOutput.ReadToEnd(); string stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); - if (proc.ExitCode != 0) + if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) { - _logger.LogDebug("filefrag exited with {code} for {file}. stderr: {stderr}", - proc.ExitCode, path, stderr); - return false; + _logger.LogTrace("filefrag exited with code {code}: {stderr}", proc.ExitCode, stderr); } - bool compressed = output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); + bool compressed = stdout.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); _logger.LogTrace("Btrfs compression check for {file}: {compressed}", path, compressed); return compressed; } catch (Exception ex) { - _logger.LogDebug(ex, "Failed to detect btrfs compression for {file}", path); + _logger.LogDebug(ex, "Failed to detect Btrfs compression for {file}", path); return false; } } - private bool WOFCompressFile(string path) - { - var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo)); - Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: true); - ulong length = (ulong)Marshal.SizeOf(_efInfo); - try - { - using var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None); - var handle = fs.SafeFileHandle; - - if (handle.IsInvalid) - { - _logger.LogWarning("Invalid file handle to {file}", path); - return false; - } - - return UseSafeHandle(handle, hFile => - { - int ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length); - if (ret != 0 && ret != unchecked((int)0x80070158)) - { - _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); - return false; - } - return true; - }); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Error compacting file {path}", path); - return false; - } - finally - { - Marshal.FreeHGlobal(efInfoPtr); - } - } - + /// + /// Compress an Btrfs File + /// + /// Path of the decompressed/normal file + /// Compessing state private bool BtrfsCompressFile(string path) { try { - string realPath = path; - if (_dalamudUtilService.IsWine && IsProbablyWine()) + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; + + if (isWine && IsProbablyWine()) { - realPath = ConvertWinePathToLinux(path); - _logger.LogTrace("Detected Wine environment, remapped path: {realPath}", realPath); + if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + { + realPath = "/" + path[3..].Replace('\\', '/'); + } + else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + { + string linuxHome = Environment.GetEnvironmentVariable("HOME") ?? "/home"; + realPath = Path.Combine(linuxHome, path[3..].Replace('\\', '/')).Replace('\\', '/'); + } + + _logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", realPath); } - if (!File.Exists("/usr/bin/btrfs") && !File.Exists("/bin/btrfs")) + const int maxRetries = 3; + for (int attempt = 0; attempt < maxRetries; attempt++) { - _logger.LogWarning("Skipping Btrfs compression — btrfs binary not found"); - return false; + try + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + break; + } + catch (IOException) + { + if (attempt == maxRetries - 1) + { + _logger.LogWarning("File still in use after {attempts} attempts; skipping btrfs compression for {file}", maxRetries, path); + return false; + } + + int delay = 150 * (attempt + 1); + _logger.LogTrace("File busy, retrying in {delay}ms for {file}", delay, path); + Thread.Sleep(delay); + } } - var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -czstd -- \"{realPath}\"") + string command = $"btrfs filesystem defragment -czstd -- \"{realPath}\""; + var psi = new ProcessStartInfo { + FileName = isWine ? "/bin/bash" : "btrfs", + Arguments = isWine ? $"-c \"{command}\"" : $"filesystem defragment -czstd -- \"{realPath}\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, - CreateNoWindow = true + CreateNoWindow = true, + WorkingDirectory = "/" }; using var proc = Process.Start(psi); if (proc == null) { - _logger.LogWarning("Failed to start btrfs process for {file}", realPath); + _logger.LogWarning("Failed to start btrfs defragment for compression of {file}", path); return false; } - var stdout = proc.StandardOutput.ReadToEnd(); - var stderr = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); + string stdout = proc.StandardOutput.ReadToEnd(); + string stderr = proc.StandardError.ReadToEnd(); + + try + { + proc.WaitForExit(); + } + catch (Exception ex) + { + _logger.LogTrace(ex, "Process.WaitForExit threw under Wine for {file}", path); + } if (proc.ExitCode != 0) { - _logger.LogWarning("btrfs defrag returned {code} for {file}: {err}", proc.ExitCode, realPath, stderr); + _logger.LogWarning("btrfs defragment failed for {file}: {stderr}", path, stderr); return false; } - _logger.LogDebug("btrfs output: {out}", stdout); + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs defragment output for {file}: {stdout}", path, stdout.Trim()); + + _logger.LogInformation("Compressed btrfs file successfully: {file}", path); return true; } catch (Exception ex) @@ -674,22 +666,15 @@ public sealed class FileCompactor : IDisposable } } - private struct WOF_FILE_COMPRESSION_INFO_V1 - { - public CompressionAlgorithm Algorithm; - public ulong Flags; - } - private void EnqueueCompaction(string filePath) { if (!_pendingCompactions.TryAdd(filePath, 0)) return; - - var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); + var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) { - _logger.LogTrace("Skipping compaction enqueue for unsupported filesystem {fs} ({file})", fsType, filePath); + _logger.LogTrace("Skip enqueue (unsupported fs) {fs} {file}", fsType, filePath); _pendingCompactions.TryRemove(filePath, out _); return; } @@ -697,11 +682,7 @@ public sealed class FileCompactor : IDisposable if (!_compactionQueue.Writer.TryWrite(filePath)) { _pendingCompactions.TryRemove(filePath, out _); - _logger.LogDebug("Failed to enqueue compaction job for {file}", filePath); - } - else - { - _logger.LogTrace("Queued compaction job for {file} (fs={fs})", filePath, fsType); + _logger.LogDebug("Failed to enqueue compaction {file}", filePath); } } @@ -727,13 +708,13 @@ public sealed class FileCompactor : IDisposable if (!File.Exists(filePath)) { - _logger.LogTrace("Skipping compaction for missing file {file}", filePath); + _logger.LogTrace("Skip compact (missing) {file}", filePath); continue; } CompactFile(filePath); } - catch (OperationCanceledException) + catch (OperationCanceledException) { return; } @@ -750,9 +731,46 @@ public sealed class FileCompactor : IDisposable } catch (OperationCanceledException) { - _logger.LogDebug("Queue has been cancelled by token"); + _logger.LogDebug("Compaction queue cancelled"); } } - private static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); + [StructLayout(LayoutKind.Sequential, Pack = 1)] + private struct WOF_FILE_COMPRESSION_INFO_V1 + { + public int Algorithm; + public ulong Flags; + } + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped); + + [DllImport("kernel32.dll")] + private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh); + + [DllImport("WofUtil.dll")] + private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength); + + [DllImport("WofUtil.dll", SetLastError = true)] + private static extern int WofSetFileDataLocation(SafeFileHandle FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); + + public void Dispose() + { + _compactionQueue.Writer.TryComplete(); + _compactionCts.Cancel(); + try + { + _compactionWorker.Wait(TimeSpan.FromSeconds(5)); + } + catch + { + //ignore on catch ^^ + } + finally + { + _compactionCts.Dispose(); + } + + GC.SuppressFinalize(this); + } } diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs index 4bbfc75..af4c98b 100644 --- a/LightlessSync/Utils/FileSystemHelper.cs +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -1,4 +1,6 @@ -using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using System.Collections.Concurrent; +using System.Diagnostics; using System.Runtime.InteropServices; namespace LightlessSync.Utils @@ -20,6 +22,8 @@ namespace LightlessSync.Utils } private const string _mountPath = "/proc/mounts"; + private const int _defaultBlockSize = 4096; + private static readonly Dictionary _blockSizeCache = new(StringComparer.OrdinalIgnoreCase); private static readonly ConcurrentDictionary _filesystemTypeCache = new(StringComparer.OrdinalIgnoreCase); public static FilesystemType GetFilesystemType(string filePath, bool isWine = false) @@ -27,6 +31,7 @@ namespace LightlessSync.Utils try { string rootPath; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (!IsProbablyWine() || !isWine)) { var info = new FileInfo(filePath); @@ -49,6 +54,7 @@ namespace LightlessSync.Utils { var root = new DriveInfo(rootPath); var format = root.DriveFormat?.ToUpperInvariant() ?? string.Empty; + detected = format switch { "NTFS" => FilesystemType.NTFS, @@ -62,10 +68,28 @@ namespace LightlessSync.Utils detected = GetLinuxFilesystemType(filePath); } + if (isWine || IsProbablyWine()) + { + switch (detected) + { + case FilesystemType.NTFS: + case FilesystemType.Unknown: + { + var linuxDetected = GetLinuxFilesystemType(filePath); + if (linuxDetected != FilesystemType.Unknown) + { + detected = linuxDetected; + } + + break; + } + } + } + _filesystemTypeCache[rootPath] = detected; return detected; } - catch (Exception ex) + catch { return FilesystemType.Unknown; } @@ -175,7 +199,84 @@ namespace LightlessSync.Utils } } + public static int GetBlockSizeForPath(string path, ILogger? logger = null, bool isWine = false) + { + try + { + if (string.IsNullOrWhiteSpace(path)) + return _defaultBlockSize; + + var fi = new FileInfo(path); + if (!fi.Exists) + return _defaultBlockSize; + + var root = fi.Directory?.Root.FullName.ToLowerInvariant() ?? "/"; + if (_blockSizeCache.TryGetValue(root, out int cached)) + return cached; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !isWine) + { + int result = GetDiskFreeSpaceW(root, + out uint sectorsPerCluster, + out uint bytesPerSector, + out _, + out _); + + if (result == 0) + { + logger?.LogWarning("Failed to determine block size for {root}", root); + return _defaultBlockSize; + } + + int clusterSize = (int)(sectorsPerCluster * bytesPerSector); + _blockSizeCache[root] = clusterSize; + logger?.LogTrace("NTFS cluster size for {root}: {cluster}", root, clusterSize); + return clusterSize; + } + + string realPath = fi.FullName; + if (isWine && realPath.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + { + realPath = "/" + realPath.Substring(3).Replace('\\', '/'); + } + + var psi = new ProcessStartInfo + { + FileName = "/bin/bash", + Arguments = $"-c \"stat -f -c %s '{realPath.Replace("'", "'\\''")}'\"", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = "/" + }; + + using var proc = Process.Start(psi); + string stdout = proc?.StandardOutput.ReadToEnd().Trim() ?? ""; + proc?.WaitForExit(); + + if (int.TryParse(stdout, out int blockSize) && blockSize > 0) + { + _blockSizeCache[root] = blockSize; + logger?.LogTrace("Filesystem block size via stat for {root}: {block}", root, blockSize); + return blockSize; + } + + logger?.LogTrace("stat did not return valid block size for {file}, output: {out}", fi.FullName, stdout); + _blockSizeCache[root] = _defaultBlockSize; + return _defaultBlockSize; + } + catch (Exception ex) + { + logger?.LogTrace(ex, "Error determining block size for {path}", path); + return _defaultBlockSize; + } + } + + [DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)] + private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters, out uint lpTotalNumberOfClusters); + //Extra check on - private static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); + public static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); } }