diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 85fb30e..486e11e 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -403,57 +403,94 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase public void RecalculateFileCacheSize(CancellationToken token) { - if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder)) + if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || + !Directory.Exists(_configService.Current.CacheFolder)) { FileCacheSize = 0; return; } - FileCacheSize = -1; - - var drive = DriveInfo.GetDrives().FirstOrDefault(d => _configService.Current.CacheFolder.StartsWith(d.Name, StringComparison.Ordinal)); - if (drive == null) - { - return; - } + FileCacheSize = -1; + bool isWine = _dalamudUtil?.IsWine ?? false; try { - FileCacheDriveFree = drive.AvailableFreeSpace; + var drive = DriveInfo.GetDrives() + .FirstOrDefault(d => _configService.Current.CacheFolder + .StartsWith(d.Name, StringComparison.OrdinalIgnoreCase)); + + if (drive != null) + FileCacheDriveFree = drive.AvailableFreeSpace; } catch (Exception ex) { - Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder); + Logger.LogWarning(ex, "Could not determine drive size for storage folder {folder}", _configService.Current.CacheFolder); } - var files = Directory.EnumerateFiles(_configService.Current.CacheFolder).Select(f => new FileInfo(f)) - .OrderBy(f => f.LastAccessTime).ToList(); - FileCacheSize = files - .Sum(f => - { - token.ThrowIfCancellationRequested(); + var files = Directory.EnumerateFiles(_configService.Current.CacheFolder) + .Select(f => new FileInfo(f)) + .OrderBy(f => f.LastAccessTime) + .ToList(); - try + long totalSize = 0; + + foreach (var f in files) + { + token.ThrowIfCancellationRequested(); + + try + { + long size = 0; + + if (!isWine) { - return _fileCompactor.GetFileSizeOnDisk(f); + try + { + size = _fileCompactor.GetFileSizeOnDisk(f); + } + catch (Exception ex) + { + Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName); + size = f.Length; + } } - catch + else { - return 0; + size = f.Length; } - }); + + totalSize += size; + } + catch (Exception ex) + { + Logger.LogTrace(ex, "Error getting size for {file}", f.FullName); + } + } + + FileCacheSize = totalSize; var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d); - - if (FileCacheSize < maxCacheInBytes) return; + if (FileCacheSize < maxCacheInBytes) + return; var maxCacheBuffer = maxCacheInBytes * 0.05d; - while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer) + + while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer && files.Count > 0) { var oldestFile = files[0]; - FileCacheSize -= _fileCompactor.GetFileSizeOnDisk(oldestFile); - File.Delete(oldestFile.FullName); - files.Remove(oldestFile); + + try + { + long fileSize = oldestFile.Length; + File.Delete(oldestFile.FullName); + FileCacheSize -= fileSize; + } + catch (Exception ex) + { + Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullName); + } + + files.RemoveAt(0); } } diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index a917b59..b9c2118 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Diagnostics; +using System.IO; using System.Runtime.InteropServices; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; @@ -14,6 +15,7 @@ public sealed class FileCompactor : IDisposable { public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U; public const ulong WOF_PROVIDER_FILE = 2UL; + public const int _maxRetries = 3; private readonly ConcurrentDictionary _pendingCompactions; private readonly ILogger _logger; @@ -29,6 +31,14 @@ public sealed class FileCompactor : IDisposable Algorithm = (int)CompressionAlgorithm.XPRESS8K, Flags = 0 }; + + [StructLayout(LayoutKind.Sequential, Pack = 1)] + private struct WOF_FILE_COMPRESSION_INFO_V1 + { + public int Algorithm; + public ulong Flags; + } + private enum CompressionAlgorithm { NO_COMPRESSION = -2, @@ -58,6 +68,10 @@ public sealed class FileCompactor : IDisposable public bool MassCompactRunning { get; private set; } public string Progress { get; private set; } = string.Empty; + /// + /// Compact the storage of the Cache Folder + /// + /// Used to check if files needs to be compressed public void CompactStorage(bool compress) { MassCompactRunning = true; @@ -74,6 +88,7 @@ public sealed class FileCompactor : IDisposable try { + // Compress or decompress files if (compress) CompactFile(file); else @@ -135,25 +150,19 @@ public sealed class FileCompactor : IDisposable { try { - var realPath = fileInfo.FullName.Replace("\"", "\\\"", StringComparison.Ordinal); - var psi = new ProcessStartInfo("stat", $"-c %b \"{realPath}\"") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(fileInfo.FullName, isWine) : fileInfo.FullName; - using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat"); - var outp = proc.StandardOutput.ReadToEnd(); - var err = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); + var fileName = "stat"; + var arguments = $"-c %b \"{realPath}\""; - if (proc.ExitCode != 0) - throw new InvalidOperationException($"stat failed: {err}"); + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, arguments, out Process? proc, out string stdout); - if (!long.TryParse(outp.Trim(), out var blocks)) - throw new InvalidOperationException($"invalid stat output: {outp}"); + if (!processControl && !success) + throw new InvalidOperationException($"stat failed: {proc}"); + + if (!long.TryParse(stdout.Trim(), out var blocks)) + throw new InvalidOperationException($"invalid stat output: {stdout}"); // st_blocks are always 512-byte on Linux enviroment. return blocks * 512L; @@ -287,57 +296,57 @@ public sealed class FileCompactor : IDisposable /// Decompessing state private bool DecompressBtrfsFile(string path) { + var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + try { - var opts = GetMountOptionsForPath(path); - if (opts.Contains("compress", StringComparison.OrdinalIgnoreCase)) + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; + + var mountOptions = GetMountOptionsForPath(realPath); + if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) { - _logger.LogWarning("Cannot safely decompress {file}: mount options include compression ({opts})", path, opts); + _logger.LogWarning( + "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " + + "Remount with 'compress=no' before running decompression.", + realPath, mountOptions); return false; } - string realPath = ToLinuxPathIfWine(path, _dalamudUtilService.IsWine); - var psi = new ProcessStartInfo + if (!IsBtrfsCompressedFile(realPath)) { - FileName = _dalamudUtilService.IsWine ? "/bin/bash" : "btrfs", - Arguments = _dalamudUtilService.IsWine - ? $"-c \"btrfs filesystem defragment -- '{EscapeSingle(realPath)}'\"" - : $"filesystem defragment -- \"{realPath}\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; - - using var proc = Process.Start(psi); - if (proc == null) - { - _logger.LogWarning("Failed to start btrfs defragment for {file}", path); - return false; + _logger.LogTrace("File {file} is not compressed, skipping decompression.", realPath); + return true; } - var stdout = proc.StandardOutput.ReadToEnd(); - var stderr = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); + (bool flowControl, bool value) = FileStreamOpening(realPath, ref fs); - if (proc.ExitCode != 0) + if (!flowControl) { - _logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr); - return false; + return value; + } + + string fileName = isWine ? "/bin/bash" : "btrfs"; + string command = isWine ? $"-c \"filesystem defragment -- \"{realPath}\"\"" : $"filesystem defragment -- \"{realPath}\""; + + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); + if (!processControl && !success) + { + return value; } if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogTrace("btrfs defragment output {file}: {out}", path, stdout.Trim()); + _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); - _logger.LogInformation("Btrfs rewritten uncompressed: {file}", path); + _logger.LogInformation("Decompressed btrfs file successfully: {file}", realPath); + + return true; } catch (Exception ex) { - _logger.LogWarning(ex, "Btrfs decompress error {file}", path); + _logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path); + return false; } - - return true; } /// @@ -379,23 +388,25 @@ public sealed class FileCompactor : IDisposable return true; } - private static string EscapeSingle(string p) => p.Replace("'", "'\\'", StringComparison.Ordinal); - - private static string ToLinuxPathIfWine(string path, bool isWine) + /// + /// Converts to Linux Path if its using Wine (diferent pathing system in Wine) + /// + /// Path that has to be converted + /// Extra check if using the wine enviroment + /// Converted path to be used in Linux + private string ToLinuxPathIfWine(string path, bool isWine) { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - return path; - if (!IsProbablyWine() && !isWine) return path; - string realPath = path; + string linuxPath = path; if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - realPath = "/" + path[3..].Replace('\\', '/'); + linuxPath = "/" + path[3..].Replace('\\', '/'); else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - realPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/'); + linuxPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/'); - return realPath; + _logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", linuxPath); + return linuxPath; } /// @@ -414,27 +425,11 @@ public sealed class FileCompactor : IDisposable Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: false); ulong length = (ulong)size; - const int maxRetries = 3; + (bool flowControl, bool value) = FileStreamOpening(path, ref fs); - for (int attempt = 0; attempt < maxRetries; attempt++) + if (!flowControl) { - try - { - fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); - break; - } - catch (IOException) - { - if (attempt == maxRetries - 1) - { - _logger.LogWarning("File still in use after {attempts} attempts, skipping compression for {file}", maxRetries, path); - return false; - } - - int delay = 150 * (attempt + 1); - _logger.LogTrace("File in use, retrying in {delay}ms for {file}", delay, path); - Thread.Sleep(delay); - } + return value; } if (fs == null) @@ -462,14 +457,14 @@ public sealed class FileCompactor : IDisposable return true; } - catch (DllNotFoundException) + catch (DllNotFoundException ex) { - _logger.LogTrace("WofUtil.dll not available; skipping NTFS compaction for {file}", path); + _logger.LogTrace(ex, "WofUtil.dll not available, this DLL is needed for compression; skipping NTFS compaction for {file}", path); return false; } - catch (EntryPointNotFoundException) + catch (EntryPointNotFoundException ex) { - _logger.LogTrace("WOF entrypoint missing (Wine/older OS); skipping NTFS compaction for {file}", path); + _logger.LogTrace(ex, "WOF entrypoint missing (Wine/older OS); skipping NTFS compaction for {file}", path); return false; } catch (Exception ex) @@ -527,37 +522,24 @@ public sealed class FileCompactor : IDisposable bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; - var psi = new ProcessStartInfo - { - FileName = isWine ? "/bin/bash" : "filefrag", - Arguments = isWine - ? $"-c \"filefrag -v '{EscapeSingle(realPath)}'\"" - : $"-v \"{realPath}\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; + var fi = new FileInfo(realPath); - using var proc = Process.Start(psi); - if (proc == null) + if (fi == null) { - _logger.LogWarning("Failed to start filefrag for {file}", path); + _logger.LogWarning("Failed to open {file} for checking on compression; skipping", realPath); return false; } - string stdout = proc.StandardOutput.ReadToEnd(); - string stderr = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); - - if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) + string fileName = isWine ? "/bin/bash" : "filefrag"; + string command = isWine ? $"-c \"filefrag -v '{EscapeSingle(realPath)}'\"" : $"-v \"{realPath}\""; + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); + if (!processControl && !success) { - _logger.LogTrace("filefrag exited with code {code}: {stderr}", proc.ExitCode, stderr); + return success; } bool compressed = stdout.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); - _logger.LogTrace("Btrfs compression check for {file}: {compressed}", path, compressed); + _logger.LogTrace("Btrfs compression check for {file}: {compressed}", realPath, compressed); return compressed; } catch (Exception ex) @@ -574,89 +556,56 @@ public sealed class FileCompactor : IDisposable /// Compessing state private bool BtrfsCompressFile(string path) { + FileStream? fs = null; + try { bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; - if (isWine && IsProbablyWine()) + var fi = new FileInfo(realPath); + + if (fi == null) { - if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - { - realPath = "/" + path[3..].Replace('\\', '/'); - } - else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - { - string linuxHome = Environment.GetEnvironmentVariable("HOME") ?? "/home"; - realPath = Path.Combine(linuxHome, path[3..].Replace('\\', '/')).Replace('\\', '/'); - } - - _logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", realPath); - } - - const int maxRetries = 3; - for (int attempt = 0; attempt < maxRetries; attempt++) - { - try - { - using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); - break; - } - catch (IOException) - { - if (attempt == maxRetries - 1) - { - _logger.LogWarning("File still in use after {attempts} attempts; skipping btrfs compression for {file}", maxRetries, path); - return false; - } - - int delay = 150 * (attempt + 1); - _logger.LogTrace("File busy, retrying in {delay}ms for {file}", delay, path); - Thread.Sleep(delay); - } - } - - string command = $"btrfs filesystem defragment -czstd -- \"{realPath}\""; - var psi = new ProcessStartInfo - { - FileName = isWine ? "/bin/bash" : "btrfs", - Arguments = isWine ? $"-c \"{command}\"" : $"filesystem defragment -czstd -- \"{realPath}\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; - - using var proc = Process.Start(psi); - if (proc == null) - { - _logger.LogWarning("Failed to start btrfs defragment for compression of {file}", path); + _logger.LogWarning("Failed to open {file} for compression; skipping", realPath); return false; } - string stdout = proc.StandardOutput.ReadToEnd(); - string stderr = proc.StandardError.ReadToEnd(); - - try - { - proc.WaitForExit(); - } - catch (Exception ex) + //Skipping small files to make compression a bit faster, its not that effective on small files. + int blockSize = GetBlockSizeForPath(realPath, _logger, isWine); + if (fi.Length < Math.Max(blockSize * 2, 128 * 1024)) { - _logger.LogTrace(ex, "Process.WaitForExit threw under Wine for {file}", path); + _logger.LogTrace("Skipping Btrfs compression for small file {file} ({size} bytes)", realPath, fi.Length); + return true; } - if (proc.ExitCode != 0) + if (IsBtrfsCompressedFile(realPath)) { - _logger.LogWarning("btrfs defragment failed for {file}: {stderr}", path, stderr); - return false; + _logger.LogTrace("File {file} already compressed (Btrfs), skipping file", realPath); + return true; + } + + (bool flowControl, bool value) = FileStreamOpening(realPath, ref fs); + + if (!flowControl) + { + return value; + } + + string fileName = isWine ? "/bin/bash" : "btrfs"; + string command = isWine ? $"-c \"btrfs filesystem defragment -czstd:1 -- \"{realPath}\"\"" : $"btrfs filesystem defragment -czstd:1 -- \"{realPath}\""; + + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); + if (!processControl && !success) + { + return value; } if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogTrace("btrfs defragment output for {file}: {stdout}", path, stdout.Trim()); + _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); + + _logger.LogInformation("Compressed btrfs file successfully: {file}", realPath); - _logger.LogInformation("Compressed btrfs file successfully: {file}", path); return true; } catch (Exception ex) @@ -666,6 +615,84 @@ public sealed class FileCompactor : IDisposable } } + + /// + /// Trying opening file stream in certain amount of tries. + /// + /// Path where the file is located + /// Filestream used for the function + /// State of the filestream opening + private (bool flowControl, bool value) FileStreamOpening(string path, ref FileStream? fs) + { + for (int attempt = 0; attempt < _maxRetries; attempt++) + { + try + { + fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + break; + } + catch (IOException) + { + if (attempt == _maxRetries - 1) + { + _logger.LogWarning("File still in use after {attempts} attempts, skipping compression for {file}", _maxRetries, path); + return (flowControl: false, value: false); + } + + int delay = 150 * (attempt + 1); + _logger.LogTrace("File in use, retrying in {delay}ms for {file}", delay, path); + Thread.Sleep(delay); + } + } + + return (flowControl: true, value: default); + } + + /// + /// Starts an process with given Filename and Arguments + /// + /// Path you want to use for the process (Compression is using these) + /// File of the command + /// Arguments used for the command + /// Returns process of the given command + /// Returns output of the given command + /// Returns if the process been done succesfully or not + private (bool processControl, bool success) StartProcessInfo(string path, string fileName, string arguments, out Process? proc, out string stdout) + { + var psi = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = "/" + }; + proc = Process.Start(psi); + + if (proc == null) + { + _logger.LogWarning("Failed to start {arguments} for {file}", arguments, path); + stdout = string.Empty; + return (processControl: false, success: false); + } + + stdout = proc.StandardOutput.ReadToEnd(); + string stderr = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + + if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) + { + _logger.LogTrace("{arguments} exited with code {code}: {stderr}", arguments, proc.ExitCode, stderr); + return (processControl: false, success: false); + } + + return (processControl: true, success: default); + } + + private static string EscapeSingle(string p) => p.Replace("'", "'\\'", StringComparison.Ordinal); + private void EnqueueCompaction(string filePath) { if (!_pendingCompactions.TryAdd(filePath, 0)) @@ -735,13 +762,6 @@ public sealed class FileCompactor : IDisposable } } - [StructLayout(LayoutKind.Sequential, Pack = 1)] - private struct WOF_FILE_COMPRESSION_INFO_V1 - { - public int Algorithm; - public ulong Flags; - } - [DllImport("kernel32.dll", SetLastError = true)] private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped);