diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 3b41d85..486e11e 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -115,6 +115,8 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase public bool StorageisNTFS { get; private set; } = false; + public bool StorageIsBtrfs { get ; private set; } = false; + public void StartLightlessWatcher(string? lightlessPath) { LightlessWatcher?.Dispose(); @@ -124,10 +126,19 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase Logger.LogWarning("Lightless file path is not set, cannot start the FSW for Lightless."); return; } + var fsType = FileSystemHelper.GetFilesystemType(_configService.Current.CacheFolder, _dalamudUtil.IsWine); - DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName); - StorageisNTFS = string.Equals("NTFS", di.DriveFormat, StringComparison.OrdinalIgnoreCase); - Logger.LogInformation("Lightless Storage is on NTFS drive: {isNtfs}", StorageisNTFS); + if (fsType == FileSystemHelper.FilesystemType.NTFS) + { + StorageisNTFS = true; + Logger.LogInformation("Lightless Storage is on NTFS drive: {isNtfs}", StorageisNTFS); + } + + if (fsType == FileSystemHelper.FilesystemType.Btrfs) + { + StorageIsBtrfs = true; + Logger.LogInformation("Lightless Storage is on BTRFS drive: {isBtrfs}", StorageIsBtrfs); + } Logger.LogDebug("Initializing Lightless FSW on {path}", lightlessPath); LightlessWatcher = new() @@ -392,51 +403,94 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase public void RecalculateFileCacheSize(CancellationToken token) { - if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder)) + if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || + !Directory.Exists(_configService.Current.CacheFolder)) { FileCacheSize = 0; return; } FileCacheSize = -1; - DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName); + bool isWine = _dalamudUtil?.IsWine ?? false; + try { - FileCacheDriveFree = di.AvailableFreeSpace; + var drive = DriveInfo.GetDrives() + .FirstOrDefault(d => _configService.Current.CacheFolder + .StartsWith(d.Name, StringComparison.OrdinalIgnoreCase)); + + if (drive != null) + FileCacheDriveFree = drive.AvailableFreeSpace; } catch (Exception ex) { - Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder); + Logger.LogWarning(ex, "Could not determine drive size for storage folder {folder}", _configService.Current.CacheFolder); } - var files = Directory.EnumerateFiles(_configService.Current.CacheFolder).Select(f => new FileInfo(f)) - .OrderBy(f => f.LastAccessTime).ToList(); - FileCacheSize = files - .Sum(f => - { - token.ThrowIfCancellationRequested(); + var files = Directory.EnumerateFiles(_configService.Current.CacheFolder) + .Select(f => new FileInfo(f)) + .OrderBy(f => f.LastAccessTime) + .ToList(); - try + long totalSize = 0; + + foreach (var f in files) + { + token.ThrowIfCancellationRequested(); + + try + { + long size = 0; + + if (!isWine) { - return _fileCompactor.GetFileSizeOnDisk(f, StorageisNTFS); + try + { + size = _fileCompactor.GetFileSizeOnDisk(f); + } + catch (Exception ex) + { + Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName); + size = f.Length; + } } - catch + else { - return 0; + size = f.Length; } - }); + + totalSize += size; + } + catch (Exception ex) + { + Logger.LogTrace(ex, "Error getting size for {file}", f.FullName); + } + } + + FileCacheSize = totalSize; var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d); - - if (FileCacheSize < maxCacheInBytes) return; + if (FileCacheSize < maxCacheInBytes) + return; var maxCacheBuffer = maxCacheInBytes * 0.05d; - while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer) + + while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer && files.Count > 0) { var oldestFile = files[0]; - FileCacheSize -= _fileCompactor.GetFileSizeOnDisk(oldestFile); - File.Delete(oldestFile.FullName); - files.Remove(oldestFile); + + try + { + long fileSize = oldestFile.Length; + File.Delete(oldestFile.FullName); + FileCacheSize -= fileSize; + } + catch (Exception ex) + { + Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullName); + } + + files.RemoveAt(0); } } @@ -644,44 +698,44 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase if (ct.IsCancellationRequested) return; - // scan new files - if (allScannedFiles.Any(c => !c.Value)) + var newFiles = allScannedFiles.Where(c => !c.Value).Select(c => c.Key).ToList(); + foreach (var cachePath in newFiles) { - Parallel.ForEach(allScannedFiles.Where(c => !c.Value).Select(c => c.Key), - new ParallelOptions() - { - MaxDegreeOfParallelism = threadCount, - CancellationToken = ct - }, (cachePath) => - { - if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null) - { - Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}", _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null); - return; - } + if (ct.IsCancellationRequested) break; + ProcessOne(cachePath); + Interlocked.Increment(ref _currentFileProgress); + } - if (ct.IsCancellationRequested) return; + Logger.LogTrace("Scanner added {count} new files to db", newFiles.Count); - if (!_ipcManager.Penumbra.APIAvailable) - { - Logger.LogWarning("Penumbra not available"); - return; - } + void ProcessOne(string? cachePath) + { + if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null) + { + Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}", + _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null); + return; + } - try - { - var entry = _fileDbManager.CreateFileEntry(cachePath); - if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath); - } - catch (Exception ex) - { - Logger.LogWarning(ex, "Failed adding {file}", cachePath); - } + if (!_ipcManager.Penumbra.APIAvailable) + { + Logger.LogWarning("Penumbra not available"); + return; + } - Interlocked.Increment(ref _currentFileProgress); - }); - - Logger.LogTrace("Scanner added {notScanned} new files to db", allScannedFiles.Count(c => !c.Value)); + try + { + var entry = _fileDbManager.CreateFileEntry(cachePath); + if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath); + } + catch (IOException ioex) + { + Logger.LogDebug(ioex, "File busy or locked: {file}", cachePath); + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Failed adding {file}", cachePath); + } } Logger.LogDebug("Scan complete"); diff --git a/LightlessSync/FileCache/FileCacheManager.cs b/LightlessSync/FileCache/FileCacheManager.cs index 972c4d9..7ee6c99 100644 --- a/LightlessSync/FileCache/FileCacheManager.cs +++ b/LightlessSync/FileCache/FileCacheManager.cs @@ -203,42 +203,72 @@ public sealed class FileCacheManager : IHostedService return output; } - public Task> ValidateLocalIntegrity(IProgress<(int, int, FileCacheEntity)> progress, CancellationToken cancellationToken) + public async Task> ValidateLocalIntegrity(IProgress<(int completed, int total, FileCacheEntity current)> progress, CancellationToken cancellationToken) { _lightlessMediator.Publish(new HaltScanMessage(nameof(ValidateLocalIntegrity))); _logger.LogInformation("Validating local storage"); - var cacheEntries = _fileCaches.Values.SelectMany(v => v.Values.Where(e => e != null)).Where(v => v.IsCacheEntry).ToList(); - List brokenEntities = []; - int i = 0; - foreach (var fileCache in cacheEntries) + + var cacheEntries = _fileCaches.Values + .SelectMany(v => v.Values) + .Where(v => v.IsCacheEntry) + .ToList(); + + int total = cacheEntries.Count; + int processed = 0; + var brokenEntities = new ConcurrentBag(); + + _logger.LogInformation("Checking {count} cache entries...", total); + + await Parallel.ForEachAsync(cacheEntries, new ParallelOptions + { + MaxDegreeOfParallelism = Environment.ProcessorCount, + CancellationToken = cancellationToken + }, + async (fileCache, token) => { - if (cancellationToken.IsCancellationRequested) break; - - _logger.LogInformation("Validating {file}", fileCache.ResolvedFilepath); - - progress.Report((i, cacheEntries.Count, fileCache)); - i++; - if (!File.Exists(fileCache.ResolvedFilepath)) - { - brokenEntities.Add(fileCache); - continue; - } - try { - var computedHash = Crypto.GetFileHash(fileCache.ResolvedFilepath); + int current = Interlocked.Increment(ref processed); + if (current % 10 == 0) + progress.Report((current, total, fileCache)); + + if (!File.Exists(fileCache.ResolvedFilepath)) + { + brokenEntities.Add(fileCache); + return; + } + + string computedHash; + try + { + computedHash = await Crypto.GetFileHashAsync(fileCache.ResolvedFilepath, token).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error hashing {file}", fileCache.ResolvedFilepath); + brokenEntities.Add(fileCache); + return; + } + if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal)) { - _logger.LogInformation("Failed to validate {file}, got hash {computedHash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash); + _logger.LogInformation( + "Hash mismatch: {file} (got {computedHash}, expected {expected})", + fileCache.ResolvedFilepath, computedHash, fileCache.Hash); + brokenEntities.Add(fileCache); } } - catch (Exception e) + catch (OperationCanceledException) { - _logger.LogWarning(e, "Error during validation of {file}", fileCache.ResolvedFilepath); + _logger.LogError("Validation got cancelled for {file}", fileCache.ResolvedFilepath); + } + catch (Exception ex) + { + _logger.LogError(ex, "Unexpected error validating {file}", fileCache.ResolvedFilepath); brokenEntities.Add(fileCache); } - } + }).ConfigureAwait(false); foreach (var brokenEntity in brokenEntities) { @@ -250,12 +280,14 @@ public sealed class FileCacheManager : IHostedService } catch (Exception ex) { - _logger.LogWarning(ex, "Could not delete {file}", brokenEntity.ResolvedFilepath); + _logger.LogWarning(ex, "Failed to delete invalid cache file {file}", brokenEntity.ResolvedFilepath); } } _lightlessMediator.Publish(new ResumeScanMessage(nameof(ValidateLocalIntegrity))); - return Task.FromResult(brokenEntities); + _logger.LogInformation("Validation complete. Found {count} invalid entries.", brokenEntities.Count); + + return [.. brokenEntities]; } public string GetCacheFilePath(string hash, string extension) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 1a35ad6..4722b1f 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,11 +1,12 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; using Microsoft.Extensions.Logging; -using System.Runtime.InteropServices; +using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; -using System.Threading; +using System.Diagnostics; +using System.Runtime.InteropServices; using System.Threading.Channels; -using System.Threading.Tasks; +using static LightlessSync.Utils.FileSystemHelper; namespace LightlessSync.FileCache; @@ -13,45 +14,31 @@ public sealed class FileCompactor : IDisposable { public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U; public const ulong WOF_PROVIDER_FILE = 2UL; + public const int _maxRetries = 3; - private readonly Dictionary _clusterSizes; private readonly ConcurrentDictionary _pendingCompactions; - private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo; private readonly ILogger _logger; - private readonly LightlessConfigService _lightlessConfigService; private readonly DalamudUtilService _dalamudUtilService; + private readonly Channel _compactionQueue; private readonly CancellationTokenSource _compactionCts = new(); private readonly Task _compactionWorker; - - public FileCompactor(ILogger logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService) + + private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo = new() { - _clusterSizes = new(StringComparer.Ordinal); - _pendingCompactions = new(StringComparer.OrdinalIgnoreCase); - _logger = logger; - _lightlessConfigService = lightlessConfigService; - _dalamudUtilService = dalamudUtilService; - _efInfo = new WOF_FILE_COMPRESSION_INFO_V1 - { - Algorithm = CompressionAlgorithm.XPRESS8K, - Flags = 0 - }; + Algorithm = (int)CompressionAlgorithm.XPRESS8K, + Flags = 0 + }; - _compactionQueue = Channel.CreateUnbounded(new UnboundedChannelOptions - { - SingleReader = true, - SingleWriter = false - }); - _compactionWorker = Task.Factory.StartNew( - () => ProcessQueueAsync(_compactionCts.Token), - _compactionCts.Token, - TaskCreationOptions.LongRunning, - TaskScheduler.Default) - .Unwrap(); + [StructLayout(LayoutKind.Sequential, Pack = 1)] + private struct WOF_FILE_COMPRESSION_INFO_V1 + { + public int Algorithm; + public ulong Flags; } - private enum CompressionAlgorithm + private enum CompressionAlgorithm { NO_COMPRESSION = -2, LZNT1 = -1, @@ -61,226 +48,730 @@ public sealed class FileCompactor : IDisposable XPRESS16K = 3 } - public bool MassCompactRunning { get; private set; } = false; + public FileCompactor(ILogger logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService) + { + _pendingCompactions = new(StringComparer.OrdinalIgnoreCase); + _logger = logger; + _lightlessConfigService = lightlessConfigService; + _dalamudUtilService = dalamudUtilService; + _compactionQueue = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = true, + SingleWriter = false + }); + + _compactionWorker = Task.Factory.StartNew(() => ProcessQueueAsync(_compactionCts.Token), _compactionCts.Token, TaskCreationOptions.LongRunning,TaskScheduler.Default).Unwrap(); + } + + public bool MassCompactRunning { get; private set; } public string Progress { get; private set; } = string.Empty; + /// + /// Compact the storage of the Cache Folder + /// + /// Used to check if files needs to be compressed public void CompactStorage(bool compress) { MassCompactRunning = true; - - int currentFile = 1; - var allFiles = Directory.EnumerateFiles(_lightlessConfigService.Current.CacheFolder).ToList(); - int allFilesCount = allFiles.Count; - foreach (var file in allFiles) - { - Progress = $"{currentFile}/{allFilesCount}"; - if (compress) - CompactFile(file); - else - DecompressFile(file); - currentFile++; - } - - MassCompactRunning = false; - } - - public long GetFileSizeOnDisk(FileInfo fileInfo, bool? isNTFS = null) - { - bool ntfs = isNTFS ?? string.Equals(new DriveInfo(fileInfo.Directory!.Root.FullName).DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase); - - if (_dalamudUtilService.IsWine || !ntfs) return fileInfo.Length; - - var clusterSize = GetClusterSize(fileInfo); - if (clusterSize == -1) return fileInfo.Length; - var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); - var size = (long)hosize << 32 | losize; - return ((size + clusterSize - 1) / clusterSize) * clusterSize; - } - - public async Task WriteAllBytesAsync(string filePath, byte[] decompressedFile, CancellationToken token) - { - await File.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false); - - if (_dalamudUtilService.IsWine || !_lightlessConfigService.Current.UseCompactor) - { - return; - } - - EnqueueCompaction(filePath); - } - - public void Dispose() - { - _compactionQueue.Writer.TryComplete(); - _compactionCts.Cancel(); try { - if (!_compactionWorker.Wait(TimeSpan.FromSeconds(5))) + var allFiles = Directory.EnumerateFiles(_lightlessConfigService.Current.CacheFolder).ToList(); + int total = allFiles.Count; + int current = 0; + + foreach (var file in allFiles) { - _logger.LogDebug("Compaction worker did not shut down within timeout"); + current++; + Progress = $"{current}/{total}"; + + try + { + // Compress or decompress files + if (compress) + CompactFile(file); + else + DecompressFile(file); + } + catch (IOException ioEx) + { + _logger.LogDebug(ioEx, "File {file} locked or busy, skipping", file); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error compacting/decompressing file {file}", file); + } } } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogDebug(ex, "Error shutting down compaction worker"); - } finally { - _compactionCts.Dispose(); - } - - GC.SuppressFinalize(this); - } - - [DllImport("kernel32.dll")] - private static extern int DeviceIoControl(IntPtr hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out IntPtr lpBytesReturned, out IntPtr lpOverlapped); - - [DllImport("kernel32.dll")] - private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, - [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh); - - [DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)] - private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, - out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters, - out uint lpTotalNumberOfClusters); - - [DllImport("WoFUtil.dll")] - private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength); - - [DllImport("WofUtil.dll")] - private static extern int WofSetFileDataLocation(IntPtr FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); - - private void CompactFile(string filePath) - { - var fs = new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName); - bool isNTFS = string.Equals(fs.DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase); - if (!isNTFS) - { - _logger.LogWarning("Drive for file {file} is not NTFS", filePath); - return; - } - - var fi = new FileInfo(filePath); - var oldSize = fi.Length; - var clusterSize = GetClusterSize(fi); - - if (oldSize < Math.Max(clusterSize, 8 * 1024)) - { - _logger.LogDebug("File {file} is smaller than cluster size ({size}), ignoring", filePath, clusterSize); - return; - } - - if (!IsCompactedFile(filePath)) - { - _logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath); - - WOFCompressFile(filePath); - - var newSize = GetFileSizeOnDisk(fi); - - _logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); - } - else - { - _logger.LogDebug("File {file} already compressed", filePath); + MassCompactRunning = false; + Progress = string.Empty; } } - private void DecompressFile(string path) + /// + /// Write all bytes into a directory async + /// + /// Bytes will be writen to this filepath + /// Bytes that have to be written + /// Cancellation Token for interupts + /// Writing Task + public async Task WriteAllBytesAsync(string filePath, byte[] bytes, CancellationToken token) + { + var dir = Path.GetDirectoryName(filePath); + if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) + Directory.CreateDirectory(dir); + + await File.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false); + + if (_lightlessConfigService.Current.UseCompactor) + EnqueueCompaction(filePath); + } + + /// + /// Gets the File size for an BTRFS or NTFS file system for the given FileInfo + /// + /// Amount of blocks used in the disk + public long GetFileSizeOnDisk(FileInfo fileInfo) + { + var fsType = GetFilesystemType(fileInfo.FullName, _dalamudUtilService.IsWine); + + if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) + { + (bool flowControl, long value) = GetFileSizeNTFS(fileInfo); + if (!flowControl) + { + return value; + } + } + + if (fsType == FilesystemType.Btrfs) + { + (bool flowControl, long value) = GetFileSizeBtrfs(fileInfo); + if (!flowControl) + { + return value; + } + } + + return fileInfo.Length; + } + + /// + /// Get File Size in an Btrfs file system (Linux/Wine). + /// + /// File that you want the size from. + /// Succesful check and value of the filesize. + /// Fails on the Process in StartProcessInfo + private (bool flowControl, long value) GetFileSizeBtrfs(FileInfo fileInfo) { - _logger.LogDebug("Removing compression from {file}", path); try { - using (var fs = new FileStream(path, FileMode.Open)) - { -#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called - var hDevice = fs.SafeFileHandle.DangerousGetHandle(); -#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called - _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _); - } + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(fileInfo.FullName, isWine) : fileInfo.FullName; + + var fileName = "stat"; + var arguments = $"-c %b \"{realPath}\""; + + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, arguments, out Process? proc, out string stdout); + + if (!processControl && !success) + throw new InvalidOperationException($"stat failed: {proc}"); + + if (!long.TryParse(stdout.Trim(), out var blocks)) + throw new InvalidOperationException($"invalid stat output: {stdout}"); + + // st_blocks are always 512-byte on Linux enviroment. + return (flowControl: false, value: blocks * 512L); } catch (Exception ex) { - _logger.LogWarning(ex, "Error decompressing file {path}", path); + _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); } + + return (flowControl: true, value: default); } - private int GetClusterSize(FileInfo fi) + /// + /// Get File Size in an NTFS file system (Windows). + /// + /// File that you want the size from. + /// Succesful check and value of the filesize. + private (bool flowControl, long value) GetFileSizeNTFS(FileInfo fileInfo) { - if (!fi.Exists) return -1; - var root = fi.Directory?.Root.FullName.ToLower() ?? string.Empty; - if (string.IsNullOrEmpty(root)) return -1; - if (_clusterSizes.TryGetValue(root, out int value)) return value; - _logger.LogDebug("Getting Cluster Size for {path}, root {root}", fi.FullName, root); - int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _); - if (result == 0) return -1; - _clusterSizes[root] = (int)(sectorsPerCluster * bytesPerSector); - _logger.LogDebug("Determined Cluster Size for root {root}: {cluster}", root, _clusterSizes[root]); - return _clusterSizes[root]; - } - - private static bool IsCompactedFile(string filePath) - { - uint buf = 8; - _ = WofIsExternalFile(filePath, out int isExtFile, out uint _, out var info, ref buf); - if (isExtFile == 0) return false; - return info.Algorithm == CompressionAlgorithm.XPRESS8K; - } - - private void WOFCompressFile(string path) - { - var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo)); - Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: true); - ulong length = (ulong)Marshal.SizeOf(_efInfo); try { - using (var fs = new FileStream(path, FileMode.Open)) + var blockSize = GetBlockSizeForPath(fileInfo.FullName, _logger, _dalamudUtilService.IsWine); + var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); + var size = (long)hosize << 32 | losize; + return (flowControl: false, value: ((size + blockSize - 1) / blockSize) * blockSize); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); + } + + return (flowControl: true, value: default); + } + + /// + /// Compressing the given path with BTRFS or NTFS file system. + /// + /// Path of the decompressed/normal file + private void CompactFile(string filePath) + { + var fi = new FileInfo(filePath); + if (!fi.Exists) + { + _logger.LogTrace("Skip compact: missing {file}", filePath); + return; + } + + var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); + _logger.LogTrace("Detected filesystem {fs} for {file} (isWine={wine})", fsType, filePath, _dalamudUtilService.IsWine); + var oldSize = fi.Length; + + int blockSize = GetBlockSizeForPath(fi.FullName, _logger, _dalamudUtilService.IsWine); + if (oldSize < Math.Max(blockSize, 8 * 1024)) + { + _logger.LogTrace("Skip compact: {file} < block {block}", filePath, blockSize); + return; + } + + if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) + { + if (!IsWOFCompactedFile(filePath)) { -#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called - var hFile = fs.SafeFileHandle.DangerousGetHandle(); -#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called - if (fs.SafeFileHandle.IsInvalid) + _logger.LogDebug("NTFS compact XPRESS8K: {file}", filePath); + if (WOFCompressFile(filePath)) { - _logger.LogWarning("Invalid file handle to {file}", path); + var newSize = GetFileSizeOnDisk(fi); + _logger.LogDebug("NTFS compressed {file} {old} -> {new}", filePath, oldSize, newSize); } else { - var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length); - if (!(ret == 0 || ret == unchecked((int)0x80070158))) - { - _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); - } + _logger.LogWarning("NTFS compression failed or unavailable for {file}", filePath); } } + else + { + _logger.LogTrace("Already NTFS-compressed: {file}", filePath); + } + return; + } + + if (fsType == FilesystemType.Btrfs) + { + if (!IsBtrfsCompressedFile(filePath)) + { + _logger.LogDebug("Btrfs compress zstd: {file}", filePath); + if (BtrfsCompressFile(filePath)) + { + var newSize = GetFileSizeOnDisk(fi); + _logger.LogDebug("Btrfs compressed {file} {old} -> {new}", filePath, oldSize, newSize); + } + else + { + _logger.LogWarning("Btrfs compression failed or unavailable for {file}", filePath); + } + } + else + { + _logger.LogTrace("Already Btrfs-compressed: {file}", filePath); + } + return; + } + + _logger.LogTrace("Skip compact: unsupported FS for {file}", filePath); + } + + /// + /// Decompressing the given path with BTRFS file system or NTFS file system. + /// + /// Path of the compressed file + private void DecompressFile(string path) + { + _logger.LogDebug("Decompress request: {file}", path); + var fsType = GetFilesystemType(path, _dalamudUtilService.IsWine); + + if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) + { + try + { + bool flowControl = DecompressWOFFile(path, out FileStream fs); + if (!flowControl) + { + return; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "NTFS decompress error {file}", path); + } + } + + if (fsType == FilesystemType.Btrfs) + { + try + { + bool flowControl = DecompressBtrfsFile(path); + if (!flowControl) + { + return; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Btrfs decompress error {file}", path); + } + } + } + + /// + /// Decompress an BTRFS File + /// + /// Path of the compressed file + /// Decompressing state + private bool DecompressBtrfsFile(string path) + { + var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + + try + { + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; + + var mountOptions = GetMountOptionsForPath(realPath); + if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) + { + _logger.LogWarning( + "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " + + "Remount with 'compress=no' before running decompression.", + realPath, mountOptions); + return false; + } + + if (!IsBtrfsCompressedFile(realPath)) + { + _logger.LogTrace("File {file} is not compressed, skipping decompression.", realPath); + return true; + } + + (bool flowControl, bool value) = FileStreamOpening(realPath, ref fs); + + if (!flowControl) + { + return value; + } + + string fileName = isWine ? "/bin/bash" : "btrfs"; + string command = isWine ? $"-c \"filesystem defragment -- \"{realPath}\"\"" : $"filesystem defragment -- \"{realPath}\""; + + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); + if (!processControl && !success) + { + return value; + } + + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); + + _logger.LogInformation("Decompressed btrfs file successfully: {file}", realPath); + + return true; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path); + return false; + } + } + + /// + /// Decompress an NTFS File + /// + /// Path of the compressed file + /// Decompressing state + private bool DecompressWOFFile(string path, out FileStream fs) + { + fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + var handle = fs.SafeFileHandle; + + if (handle.IsInvalid) + { + _logger.LogWarning("Invalid handle: {file}", path); + return false; + } + + if (!DeviceIoControl(handle, FSCTL_DELETE_EXTERNAL_BACKING, + IntPtr.Zero, 0, IntPtr.Zero, 0, + out _, IntPtr.Zero)) + { + int err = Marshal.GetLastWin32Error(); + + if (err == 342) + { + _logger.LogTrace("File {file} not externally backed (already decompressed)", path); + } + else + { + _logger.LogWarning("DeviceIoControl failed for {file} with Win32 error {err}", path, err); + } + } + else + { + _logger.LogTrace("Successfully decompressed NTFS file {file}", path); + } + + return true; + } + + /// + /// Converts to Linux Path if its using Wine (diferent pathing system in Wine) + /// + /// Path that has to be converted + /// Extra check if using the wine enviroment + /// Converted path to be used in Linux + private string ToLinuxPathIfWine(string path, bool isWine) + { + if (!IsProbablyWine() && !isWine) + return path; + + string linuxPath = path; + if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + linuxPath = "/" + path[3..].Replace('\\', '/'); + else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + linuxPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/'); + + _logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", linuxPath); + return linuxPath; + } + + /// + /// Compress an WOF File + /// + /// Path of the decompressed/normal file + /// Compessing state + private bool WOFCompressFile(string path) + { + FileStream? fs = null; + int size = Marshal.SizeOf(); + IntPtr efInfoPtr = Marshal.AllocHGlobal(size); + + try + { + Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: false); + ulong length = (ulong)size; + + (bool flowControl, bool value) = FileStreamOpening(path, ref fs); + + if (!flowControl) + { + return value; + } + + if (fs == null) + { + _logger.LogWarning("Failed to open {file} for compression; skipping", path); + return false; + } + + var handle = fs.SafeFileHandle; + + if (handle.IsInvalid) + { + _logger.LogWarning("Invalid file handle for {file}", path); + return false; + } + + int ret = WofSetFileDataLocation(handle, WOF_PROVIDER_FILE, efInfoPtr, length); + + // 0x80070158 is WOF error whenever compression fails in an non-fatal way. + if (ret != 0 && ret != unchecked((int)0x80070158)) + { + _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); + return false; + } + + return true; + } + catch (DllNotFoundException ex) + { + _logger.LogTrace(ex, "WofUtil.dll not available, this DLL is needed for compression; skipping NTFS compaction for {file}", path); + return false; + } + catch (EntryPointNotFoundException ex) + { + _logger.LogTrace(ex, "WOF entrypoint missing (Wine/older OS); skipping NTFS compaction for {file}", path); + return false; } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting file {path}", path); + return false; } finally { - Marshal.FreeHGlobal(efInfoPtr); + fs?.Dispose(); + + if (efInfoPtr != IntPtr.Zero) + { + Marshal.FreeHGlobal(efInfoPtr); + } } } - private struct WOF_FILE_COMPRESSION_INFO_V1 + /// + /// Checks if an File is compacted with WOF compression + /// + /// Path of the file + /// State of the file + private static bool IsWOFCompactedFile(string filePath) { - public CompressionAlgorithm Algorithm; - public ulong Flags; + try + { + uint buf = (uint)Marshal.SizeOf(); + int result = WofIsExternalFile(filePath, out int isExternal, out _, out var info, ref buf); + if (result != 0 || isExternal == 0) + return false; + + return info.Algorithm == (int)CompressionAlgorithm.XPRESS8K + || info.Algorithm == (int)CompressionAlgorithm.XPRESS4K + || info.Algorithm == (int)CompressionAlgorithm.XPRESS16K + || info.Algorithm == (int)CompressionAlgorithm.LZX + || info.Algorithm == (int)CompressionAlgorithm.LZNT1 + || info.Algorithm == (int)CompressionAlgorithm.NO_COMPRESSION; + } + catch + { + return false; + } } + /// + /// Checks if an File is compacted with Btrfs compression + /// + /// Path of the file + /// State of the file + private bool IsBtrfsCompressedFile(string path) + { + try + { + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; + + var fi = new FileInfo(realPath); + + if (fi == null) + { + _logger.LogWarning("Failed to open {file} for checking on compression; skipping", realPath); + return false; + } + + string fileName = isWine ? "/bin/bash" : "filefrag"; + string command = isWine ? $"-c \"filefrag -v '{EscapeSingle(realPath)}'\"" : $"-v \"{realPath}\""; + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); + if (!processControl && !success) + { + return success; + } + + bool compressed = stdout.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); + _logger.LogTrace("Btrfs compression check for {file}: {compressed}", realPath, compressed); + return compressed; + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed to detect Btrfs compression for {file}", path); + return false; + } + } + + /// + /// Compress an Btrfs File + /// + /// Path of the decompressed/normal file + /// Compessing state + private bool BtrfsCompressFile(string path) + { + FileStream? fs = null; + + try + { + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; + + var fi = new FileInfo(realPath); + + if (fi == null) + { + _logger.LogWarning("Failed to open {file} for compression; skipping", realPath); + return false; + } + + if (IsBtrfsCompressedFile(realPath)) + { + _logger.LogTrace("File {file} already compressed (Btrfs), skipping file", realPath); + return true; + } + + (bool flowControl, bool value) = FileStreamOpening(realPath, ref fs); + + if (!flowControl) + { + return value; + } + + string fileName = isWine ? "/bin/bash" : "btrfs"; + string command = isWine ? $"-c \"btrfs filesystem defragment -czstd:1 -- \"{realPath}\"\"" : $"btrfs filesystem defragment -czstd:1 -- \"{realPath}\""; + + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); + if (!processControl && !success) + { + return value; + } + + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); + + _logger.LogInformation("Compressed btrfs file successfully: {file}", realPath); + + return true; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); + return false; + } + } + + + /// + /// Trying opening file stream in certain amount of tries. + /// + /// Path where the file is located + /// Filestream used for the function + /// State of the filestream opening + private (bool flowControl, bool value) FileStreamOpening(string path, ref FileStream? fs) + { + for (int attempt = 0; attempt < _maxRetries; attempt++) + { + try + { + fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + break; + } + catch (IOException) + { + if (attempt == _maxRetries - 1) + { + _logger.LogWarning("File still in use after {attempts} attempts, skipping compression for {file}", _maxRetries, path); + return (flowControl: false, value: false); + } + + int delay = 150 * (attempt + 1); + _logger.LogTrace("File in use, retrying in {delay}ms for {file}", delay, path); + Thread.Sleep(delay); + } + } + + return (flowControl: true, value: default); + } + + /// + /// Starts an process with given Filename and Arguments + /// + /// Path you want to use for the process (Compression is using these) + /// File of the command + /// Arguments used for the command + /// Returns process of the given command + /// Returns output of the given command + /// Returns if the process been done succesfully or not + private (bool processControl, bool success) StartProcessInfo(string path, string fileName, string arguments, out Process? proc, out string stdout) + { + var psi = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = "/" + }; + proc = Process.Start(psi); + + if (proc == null) + { + _logger.LogWarning("Failed to start {arguments} for {file}", arguments, path); + stdout = string.Empty; + return (processControl: false, success: false); + } + + stdout = proc.StandardOutput.ReadToEnd(); + string stderr = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + + if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) + { + _logger.LogTrace("{arguments} exited with code {code}: {stderr}", arguments, proc.ExitCode, stderr); + return (processControl: false, success: false); + } + + return (processControl: true, success: default); + } + + private static string EscapeSingle(string p) => p.Replace("'", "'\\'", StringComparison.Ordinal); + private void EnqueueCompaction(string filePath) { - if (!_pendingCompactions.TryAdd(filePath, 0)) - { + // Safe-checks + if (string.IsNullOrWhiteSpace(filePath)) return; - } - if (!_compactionQueue.Writer.TryWrite(filePath)) + if (!_lightlessConfigService.Current.UseCompactor) + return; + + if (!File.Exists(filePath)) + return; + + if (!_pendingCompactions.TryAdd(filePath, 0)) + return; + + bool enqueued = false; + try { - _pendingCompactions.TryRemove(filePath, out _); - _logger.LogDebug("Failed to enqueue compaction job for {file}", filePath); + bool isWine = _dalamudUtilService?.IsWine ?? false; + var fsType = GetFilesystemType(filePath, isWine); + + // If under Wine, we should skip NTFS because its not Windows but might return NTFS. + if (fsType == FilesystemType.NTFS && isWine) + { + _logger.LogTrace("Skip enqueue (NTFS under Wine) {file}", filePath); + return; + } + + // Unknown file system should be skipped. + if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) + { + _logger.LogTrace("Skip enqueue (unsupported fs) {fs} {file}", fsType, filePath); + return; + } + + if (!_compactionQueue.Writer.TryWrite(filePath)) + { + _logger.LogTrace("Skip enqueue: compaction channel is closed {file}", filePath); + return; + } + + enqueued = true; + _logger.LogTrace("Queued compaction for {file} (fs={fs})", filePath, fsType); + } + finally + { + if (!enqueued) + _pendingCompactions.TryRemove(filePath, out _); } } @@ -299,20 +790,20 @@ public sealed class FileCompactor : IDisposable return; } - if (_dalamudUtilService.IsWine || !_lightlessConfigService.Current.UseCompactor) + if (!_lightlessConfigService.Current.UseCompactor) { continue; } if (!File.Exists(filePath)) { - _logger.LogTrace("Skipping compaction for missing file {file}", filePath); + _logger.LogTrace("Skip compact (missing) {file}", filePath); continue; } CompactFile(filePath); } - catch (OperationCanceledException) + catch (OperationCanceledException) { return; } @@ -329,7 +820,39 @@ public sealed class FileCompactor : IDisposable } catch (OperationCanceledException) { - // expected during shutdown + _logger.LogDebug("Compaction queue cancelled"); } } + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped); + + [DllImport("kernel32.dll")] + private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh); + + [DllImport("WofUtil.dll")] + private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength); + + [DllImport("WofUtil.dll", SetLastError = true)] + private static extern int WofSetFileDataLocation(SafeFileHandle FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); + + public void Dispose() + { + _compactionQueue.Writer.TryComplete(); + _compactionCts.Cancel(); + try + { + _compactionWorker.Wait(TimeSpan.FromSeconds(5)); + } + catch + { + //ignore on catch ^^ + } + finally + { + _compactionCts.Dispose(); + } + + GC.SuppressFinalize(this); + } } diff --git a/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs b/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs index 2b003bb..20302fe 100644 --- a/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs +++ b/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs @@ -13,5 +13,4 @@ public class ServerStorage public bool UseOAuth2 { get; set; } = false; public string? OAuthToken { get; set; } = null; public HttpTransportType HttpTransportType { get; set; } = HttpTransportType.WebSockets; - public bool ForceWebSockets { get; set; } = false; } \ No newline at end of file diff --git a/LightlessSync/UI/SettingsUi.cs b/LightlessSync/UI/SettingsUi.cs index febc142..d686a75 100644 --- a/LightlessSync/UI/SettingsUi.cs +++ b/LightlessSync/UI/SettingsUi.cs @@ -1227,16 +1227,16 @@ public class SettingsUi : WindowMediatorSubscriberBase ImGui.TextUnformatted($"Currently utilized local storage: Calculating..."); ImGui.TextUnformatted( $"Remaining space free on drive: {UiSharedService.ByteToString(_cacheMonitor.FileCacheDriveFree)}"); + bool useFileCompactor = _configService.Current.UseCompactor; - bool isLinux = _dalamudUtilService.IsWine; - if (!useFileCompactor && !isLinux) + if (!useFileCompactor) { UiSharedService.ColorTextWrapped( "Hint: To free up space when using Lightless consider enabling the File Compactor", UIColors.Get("LightlessYellow")); } - if (isLinux || !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); + if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); if (ImGui.Checkbox("Use file compactor", ref useFileCompactor)) { _configService.Current.UseCompactor = useFileCompactor; @@ -1281,10 +1281,20 @@ public class SettingsUi : WindowMediatorSubscriberBase UIColors.Get("LightlessYellow")); } - if (isLinux || !_cacheMonitor.StorageisNTFS) + if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) { ImGui.EndDisabled(); - ImGui.TextUnformatted("The file compactor is only available on Windows and NTFS drives."); + ImGui.TextUnformatted("The file compactor is only available on BTRFS and NTFS drives."); + } + + if (_cacheMonitor.StorageisNTFS) + { + ImGui.TextUnformatted("The file compactor is running on NTFS Drive."); + } + + if (_cacheMonitor.StorageIsBtrfs) + { + ImGui.TextUnformatted("The file compactor is running on Btrfs Drive."); } ImGuiHelpers.ScaledDummy(new Vector2(10, 10)); @@ -3113,22 +3123,6 @@ public class SettingsUi : WindowMediatorSubscriberBase UiSharedService.TooltipSeparator + "Note: if the server does not support a specific Transport Type it will fall through to the next automatically: WebSockets > ServerSentEvents > LongPolling"); - if (_dalamudUtilService.IsWine) - { - bool forceWebSockets = selectedServer.ForceWebSockets; - if (ImGui.Checkbox("[wine only] Force WebSockets", ref forceWebSockets)) - { - selectedServer.ForceWebSockets = forceWebSockets; - _serverConfigurationManager.Save(); - } - - _uiShared.DrawHelpText( - "On wine, Lightless will automatically fall back to ServerSentEvents/LongPolling, even if WebSockets is selected. " - + "WebSockets are known to crash XIV entirely on wine 8.5 shipped with Dalamud. " - + "Only enable this if you are not running wine 8.5." + Environment.NewLine - + "Note: If the issue gets resolved at some point this option will be removed."); - } - ImGuiHelpers.ScaledDummy(5); if (ImGui.Checkbox("Use Discord OAuth2 Authentication", ref useOauth)) diff --git a/LightlessSync/Utils/Crypto.cs b/LightlessSync/Utils/Crypto.cs index de04d26..c31f82f 100644 --- a/LightlessSync/Utils/Crypto.cs +++ b/LightlessSync/Utils/Crypto.cs @@ -1,16 +1,15 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Security.Cryptography; +using System.Security.Cryptography; using System.Text; namespace LightlessSync.Utils; public static class Crypto { + //This buffersize seems to be the best sweetpoint for Linux and Windows + private const int _bufferSize = 65536; #pragma warning disable SYSLIB0021 // Type or member is obsolete - private static readonly Dictionary<(string, ushort), string> _hashListPlayersSHA256 = new(); + private static readonly Dictionary<(string, ushort), string> _hashListPlayersSHA256 = []; private static readonly Dictionary _hashListSHA256 = new(StringComparer.Ordinal); private static readonly SHA256CryptoServiceProvider _sha256CryptoProvider = new(); @@ -21,6 +20,26 @@ public static class Crypto return BitConverter.ToString(sha1.ComputeHash(stream)).Replace("-", "", StringComparison.Ordinal); } + public static async Task GetFileHashAsync(string filePath, CancellationToken cancellationToken = default) + { + var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, bufferSize: _bufferSize, options: FileOptions.Asynchronous); + await using (stream.ConfigureAwait(false)) + { + using var sha1 = SHA1.Create(); + + var buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0) + { + sha1.TransformBlock(buffer, 0, bytesRead, outputBuffer: null, 0); + } + + sha1.TransformFinalBlock([], 0, 0); + + return Convert.ToHexString(sha1.Hash!); + } + } + public static string GetHash256(this (string, ushort) playerToHash) { if (_hashListPlayersSHA256.TryGetValue(playerToHash, out var hash)) diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs new file mode 100644 index 0000000..af4c98b --- /dev/null +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -0,0 +1,282 @@ +using Microsoft.Extensions.Logging; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace LightlessSync.Utils +{ + public static class FileSystemHelper + { + public enum FilesystemType + { + Unknown = 0, + NTFS, // Compressable on file level + Btrfs, // Compressable on file level + Ext4, // Uncompressable + Xfs, // Uncompressable + Apfs, // Compressable on OS + HfsPlus, // Compressable on OS + Fat, // Uncompressable + Exfat, // Uncompressable + Zfs // Compressable, not on file level + } + + private const string _mountPath = "/proc/mounts"; + private const int _defaultBlockSize = 4096; + private static readonly Dictionary _blockSizeCache = new(StringComparer.OrdinalIgnoreCase); + private static readonly ConcurrentDictionary _filesystemTypeCache = new(StringComparer.OrdinalIgnoreCase); + + public static FilesystemType GetFilesystemType(string filePath, bool isWine = false) + { + try + { + string rootPath; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (!IsProbablyWine() || !isWine)) + { + var info = new FileInfo(filePath); + var dir = info.Directory ?? new DirectoryInfo(filePath); + rootPath = dir.Root.FullName; + } + else + { + rootPath = GetMountPoint(filePath); + if (string.IsNullOrEmpty(rootPath)) + rootPath = "/"; + } + + if (_filesystemTypeCache.TryGetValue(rootPath, out var cachedType)) + return cachedType; + + FilesystemType detected; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (!IsProbablyWine() || !isWine)) + { + var root = new DriveInfo(rootPath); + var format = root.DriveFormat?.ToUpperInvariant() ?? string.Empty; + + detected = format switch + { + "NTFS" => FilesystemType.NTFS, + "FAT32" => FilesystemType.Fat, + "EXFAT" => FilesystemType.Exfat, + _ => FilesystemType.Unknown + }; + } + else + { + detected = GetLinuxFilesystemType(filePath); + } + + if (isWine || IsProbablyWine()) + { + switch (detected) + { + case FilesystemType.NTFS: + case FilesystemType.Unknown: + { + var linuxDetected = GetLinuxFilesystemType(filePath); + if (linuxDetected != FilesystemType.Unknown) + { + detected = linuxDetected; + } + + break; + } + } + } + + _filesystemTypeCache[rootPath] = detected; + return detected; + } + catch + { + return FilesystemType.Unknown; + } + } + + private static string GetMountPoint(string filePath) + { + try + { + var path = Path.GetFullPath(filePath); + if (!File.Exists(_mountPath)) return "/"; + var mounts = File.ReadAllLines(_mountPath); + + string bestMount = "/"; + foreach (var line in mounts) + { + var parts = line.Split(' '); + if (parts.Length < 3) continue; + var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); + + string normalizedMount; + try { normalizedMount = Path.GetFullPath(mountPoint); } + catch { normalizedMount = mountPoint; } + + if (path.StartsWith(normalizedMount, StringComparison.Ordinal) && + normalizedMount.Length > bestMount.Length) + { + bestMount = normalizedMount; + } + } + + return bestMount; + } + catch + { + return "/"; + } + } + + public static string GetMountOptionsForPath(string path) + { + try + { + var fullPath = Path.GetFullPath(path); + var mounts = File.ReadAllLines("/proc/mounts"); + string bestMount = string.Empty; + string mountOptions = string.Empty; + + foreach (var line in mounts) + { + var parts = line.Split(' '); + if (parts.Length < 4) continue; + var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); + string normalized; + try { normalized = Path.GetFullPath(mountPoint); } + catch { normalized = mountPoint; } + + if (fullPath.StartsWith(normalized, StringComparison.Ordinal) && + normalized.Length > bestMount.Length) + { + bestMount = normalized; + mountOptions = parts[3]; + } + } + + return mountOptions; + } + catch (Exception ex) + { + return string.Empty; + } + } + + private static FilesystemType GetLinuxFilesystemType(string filePath) + { + try + { + var mountPoint = GetMountPoint(filePath); + var mounts = File.ReadAllLines(_mountPath); + + foreach (var line in mounts) + { + var parts = line.Split(' '); + if (parts.Length < 3) continue; + var mount = parts[1].Replace("\\040", " ", StringComparison.Ordinal); + if (string.Equals(mount, mountPoint, StringComparison.Ordinal)) + { + var fstype = parts[2].ToLowerInvariant(); + return fstype switch + { + "btrfs" => FilesystemType.Btrfs, + "ext4" => FilesystemType.Ext4, + "xfs" => FilesystemType.Xfs, + "zfs" => FilesystemType.Zfs, + "apfs" => FilesystemType.Apfs, + "hfsplus" => FilesystemType.HfsPlus, + _ => FilesystemType.Unknown + }; + } + } + + return FilesystemType.Unknown; + } + catch + { + return FilesystemType.Unknown; + } + } + + public static int GetBlockSizeForPath(string path, ILogger? logger = null, bool isWine = false) + { + try + { + if (string.IsNullOrWhiteSpace(path)) + return _defaultBlockSize; + + var fi = new FileInfo(path); + if (!fi.Exists) + return _defaultBlockSize; + + var root = fi.Directory?.Root.FullName.ToLowerInvariant() ?? "/"; + if (_blockSizeCache.TryGetValue(root, out int cached)) + return cached; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !isWine) + { + int result = GetDiskFreeSpaceW(root, + out uint sectorsPerCluster, + out uint bytesPerSector, + out _, + out _); + + if (result == 0) + { + logger?.LogWarning("Failed to determine block size for {root}", root); + return _defaultBlockSize; + } + + int clusterSize = (int)(sectorsPerCluster * bytesPerSector); + _blockSizeCache[root] = clusterSize; + logger?.LogTrace("NTFS cluster size for {root}: {cluster}", root, clusterSize); + return clusterSize; + } + + string realPath = fi.FullName; + if (isWine && realPath.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + { + realPath = "/" + realPath.Substring(3).Replace('\\', '/'); + } + + var psi = new ProcessStartInfo + { + FileName = "/bin/bash", + Arguments = $"-c \"stat -f -c %s '{realPath.Replace("'", "'\\''")}'\"", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = "/" + }; + + using var proc = Process.Start(psi); + string stdout = proc?.StandardOutput.ReadToEnd().Trim() ?? ""; + proc?.WaitForExit(); + + if (int.TryParse(stdout, out int blockSize) && blockSize > 0) + { + _blockSizeCache[root] = blockSize; + logger?.LogTrace("Filesystem block size via stat for {root}: {block}", root, blockSize); + return blockSize; + } + + logger?.LogTrace("stat did not return valid block size for {file}, output: {out}", fi.FullName, stdout); + _blockSizeCache[root] = _defaultBlockSize; + return _defaultBlockSize; + } + catch (Exception ex) + { + logger?.LogTrace(ex, "Error determining block size for {path}", path); + return _defaultBlockSize; + } + } + + [DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)] + private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters, out uint lpTotalNumberOfClusters); + + //Extra check on + public static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); + } +} diff --git a/LightlessSync/WebAPI/SignalR/HubFactory.cs b/LightlessSync/WebAPI/SignalR/HubFactory.cs index ef9f919..1d5a0c8 100644 --- a/LightlessSync/WebAPI/SignalR/HubFactory.cs +++ b/LightlessSync/WebAPI/SignalR/HubFactory.cs @@ -70,13 +70,6 @@ public class HubFactory : MediatorSubscriberBase _ => HttpTransportType.WebSockets | HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling }; - if (_isWine && !_serverConfigurationManager.CurrentServer.ForceWebSockets - && transportType.HasFlag(HttpTransportType.WebSockets)) - { - Logger.LogDebug("Wine detected, falling back to ServerSentEvents / LongPolling"); - transportType = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; - } - Logger.LogDebug("Building new HubConnection using transport {transport}", transportType); _instance = new HubConnectionBuilder()