diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 3b41d85..486e11e 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -115,6 +115,8 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase public bool StorageisNTFS { get; private set; } = false; + public bool StorageIsBtrfs { get ; private set; } = false; + public void StartLightlessWatcher(string? lightlessPath) { LightlessWatcher?.Dispose(); @@ -124,10 +126,19 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase Logger.LogWarning("Lightless file path is not set, cannot start the FSW for Lightless."); return; } + var fsType = FileSystemHelper.GetFilesystemType(_configService.Current.CacheFolder, _dalamudUtil.IsWine); - DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName); - StorageisNTFS = string.Equals("NTFS", di.DriveFormat, StringComparison.OrdinalIgnoreCase); - Logger.LogInformation("Lightless Storage is on NTFS drive: {isNtfs}", StorageisNTFS); + if (fsType == FileSystemHelper.FilesystemType.NTFS) + { + StorageisNTFS = true; + Logger.LogInformation("Lightless Storage is on NTFS drive: {isNtfs}", StorageisNTFS); + } + + if (fsType == FileSystemHelper.FilesystemType.Btrfs) + { + StorageIsBtrfs = true; + Logger.LogInformation("Lightless Storage is on BTRFS drive: {isBtrfs}", StorageIsBtrfs); + } Logger.LogDebug("Initializing Lightless FSW on {path}", lightlessPath); LightlessWatcher = new() @@ -392,51 +403,94 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase public void RecalculateFileCacheSize(CancellationToken token) { - if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder)) + if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || + !Directory.Exists(_configService.Current.CacheFolder)) { FileCacheSize = 0; return; } FileCacheSize = -1; - DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName); + bool isWine = _dalamudUtil?.IsWine ?? false; + try { - FileCacheDriveFree = di.AvailableFreeSpace; + var drive = DriveInfo.GetDrives() + .FirstOrDefault(d => _configService.Current.CacheFolder + .StartsWith(d.Name, StringComparison.OrdinalIgnoreCase)); + + if (drive != null) + FileCacheDriveFree = drive.AvailableFreeSpace; } catch (Exception ex) { - Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder); + Logger.LogWarning(ex, "Could not determine drive size for storage folder {folder}", _configService.Current.CacheFolder); } - var files = Directory.EnumerateFiles(_configService.Current.CacheFolder).Select(f => new FileInfo(f)) - .OrderBy(f => f.LastAccessTime).ToList(); - FileCacheSize = files - .Sum(f => - { - token.ThrowIfCancellationRequested(); + var files = Directory.EnumerateFiles(_configService.Current.CacheFolder) + .Select(f => new FileInfo(f)) + .OrderBy(f => f.LastAccessTime) + .ToList(); - try + long totalSize = 0; + + foreach (var f in files) + { + token.ThrowIfCancellationRequested(); + + try + { + long size = 0; + + if (!isWine) { - return _fileCompactor.GetFileSizeOnDisk(f, StorageisNTFS); + try + { + size = _fileCompactor.GetFileSizeOnDisk(f); + } + catch (Exception ex) + { + Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName); + size = f.Length; + } } - catch + else { - return 0; + size = f.Length; } - }); + + totalSize += size; + } + catch (Exception ex) + { + Logger.LogTrace(ex, "Error getting size for {file}", f.FullName); + } + } + + FileCacheSize = totalSize; var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d); - - if (FileCacheSize < maxCacheInBytes) return; + if (FileCacheSize < maxCacheInBytes) + return; var maxCacheBuffer = maxCacheInBytes * 0.05d; - while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer) + + while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer && files.Count > 0) { var oldestFile = files[0]; - FileCacheSize -= _fileCompactor.GetFileSizeOnDisk(oldestFile); - File.Delete(oldestFile.FullName); - files.Remove(oldestFile); + + try + { + long fileSize = oldestFile.Length; + File.Delete(oldestFile.FullName); + FileCacheSize -= fileSize; + } + catch (Exception ex) + { + Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullName); + } + + files.RemoveAt(0); } } @@ -644,44 +698,44 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase if (ct.IsCancellationRequested) return; - // scan new files - if (allScannedFiles.Any(c => !c.Value)) + var newFiles = allScannedFiles.Where(c => !c.Value).Select(c => c.Key).ToList(); + foreach (var cachePath in newFiles) { - Parallel.ForEach(allScannedFiles.Where(c => !c.Value).Select(c => c.Key), - new ParallelOptions() - { - MaxDegreeOfParallelism = threadCount, - CancellationToken = ct - }, (cachePath) => - { - if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null) - { - Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}", _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null); - return; - } + if (ct.IsCancellationRequested) break; + ProcessOne(cachePath); + Interlocked.Increment(ref _currentFileProgress); + } - if (ct.IsCancellationRequested) return; + Logger.LogTrace("Scanner added {count} new files to db", newFiles.Count); - if (!_ipcManager.Penumbra.APIAvailable) - { - Logger.LogWarning("Penumbra not available"); - return; - } + void ProcessOne(string? cachePath) + { + if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null) + { + Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}", + _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null); + return; + } - try - { - var entry = _fileDbManager.CreateFileEntry(cachePath); - if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath); - } - catch (Exception ex) - { - Logger.LogWarning(ex, "Failed adding {file}", cachePath); - } + if (!_ipcManager.Penumbra.APIAvailable) + { + Logger.LogWarning("Penumbra not available"); + return; + } - Interlocked.Increment(ref _currentFileProgress); - }); - - Logger.LogTrace("Scanner added {notScanned} new files to db", allScannedFiles.Count(c => !c.Value)); + try + { + var entry = _fileDbManager.CreateFileEntry(cachePath); + if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath); + } + catch (IOException ioex) + { + Logger.LogDebug(ioex, "File busy or locked: {file}", cachePath); + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Failed adding {file}", cachePath); + } } Logger.LogDebug("Scan complete"); diff --git a/LightlessSync/FileCache/FileCacheManager.cs b/LightlessSync/FileCache/FileCacheManager.cs index 972c4d9..7ee6c99 100644 --- a/LightlessSync/FileCache/FileCacheManager.cs +++ b/LightlessSync/FileCache/FileCacheManager.cs @@ -203,42 +203,72 @@ public sealed class FileCacheManager : IHostedService return output; } - public Task> ValidateLocalIntegrity(IProgress<(int, int, FileCacheEntity)> progress, CancellationToken cancellationToken) + public async Task> ValidateLocalIntegrity(IProgress<(int completed, int total, FileCacheEntity current)> progress, CancellationToken cancellationToken) { _lightlessMediator.Publish(new HaltScanMessage(nameof(ValidateLocalIntegrity))); _logger.LogInformation("Validating local storage"); - var cacheEntries = _fileCaches.Values.SelectMany(v => v.Values.Where(e => e != null)).Where(v => v.IsCacheEntry).ToList(); - List brokenEntities = []; - int i = 0; - foreach (var fileCache in cacheEntries) + + var cacheEntries = _fileCaches.Values + .SelectMany(v => v.Values) + .Where(v => v.IsCacheEntry) + .ToList(); + + int total = cacheEntries.Count; + int processed = 0; + var brokenEntities = new ConcurrentBag(); + + _logger.LogInformation("Checking {count} cache entries...", total); + + await Parallel.ForEachAsync(cacheEntries, new ParallelOptions + { + MaxDegreeOfParallelism = Environment.ProcessorCount, + CancellationToken = cancellationToken + }, + async (fileCache, token) => { - if (cancellationToken.IsCancellationRequested) break; - - _logger.LogInformation("Validating {file}", fileCache.ResolvedFilepath); - - progress.Report((i, cacheEntries.Count, fileCache)); - i++; - if (!File.Exists(fileCache.ResolvedFilepath)) - { - brokenEntities.Add(fileCache); - continue; - } - try { - var computedHash = Crypto.GetFileHash(fileCache.ResolvedFilepath); + int current = Interlocked.Increment(ref processed); + if (current % 10 == 0) + progress.Report((current, total, fileCache)); + + if (!File.Exists(fileCache.ResolvedFilepath)) + { + brokenEntities.Add(fileCache); + return; + } + + string computedHash; + try + { + computedHash = await Crypto.GetFileHashAsync(fileCache.ResolvedFilepath, token).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error hashing {file}", fileCache.ResolvedFilepath); + brokenEntities.Add(fileCache); + return; + } + if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal)) { - _logger.LogInformation("Failed to validate {file}, got hash {computedHash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash); + _logger.LogInformation( + "Hash mismatch: {file} (got {computedHash}, expected {expected})", + fileCache.ResolvedFilepath, computedHash, fileCache.Hash); + brokenEntities.Add(fileCache); } } - catch (Exception e) + catch (OperationCanceledException) { - _logger.LogWarning(e, "Error during validation of {file}", fileCache.ResolvedFilepath); + _logger.LogError("Validation got cancelled for {file}", fileCache.ResolvedFilepath); + } + catch (Exception ex) + { + _logger.LogError(ex, "Unexpected error validating {file}", fileCache.ResolvedFilepath); brokenEntities.Add(fileCache); } - } + }).ConfigureAwait(false); foreach (var brokenEntity in brokenEntities) { @@ -250,12 +280,14 @@ public sealed class FileCacheManager : IHostedService } catch (Exception ex) { - _logger.LogWarning(ex, "Could not delete {file}", brokenEntity.ResolvedFilepath); + _logger.LogWarning(ex, "Failed to delete invalid cache file {file}", brokenEntity.ResolvedFilepath); } } _lightlessMediator.Publish(new ResumeScanMessage(nameof(ValidateLocalIntegrity))); - return Task.FromResult(brokenEntities); + _logger.LogInformation("Validation complete. Found {count} invalid entries.", brokenEntities.Count); + + return [.. brokenEntities]; } public string GetCacheFilePath(string hash, string extension) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 1a35ad6..be89c1f 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,11 +1,13 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; +using LightlessSync.Services.Compression; using Microsoft.Extensions.Logging; -using System.Runtime.InteropServices; +using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; -using System.Threading; +using System.Diagnostics; +using System.Runtime.InteropServices; using System.Threading.Channels; -using System.Threading.Tasks; +using static LightlessSync.Utils.FileSystemHelper; namespace LightlessSync.FileCache; @@ -13,45 +15,35 @@ public sealed class FileCompactor : IDisposable { public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U; public const ulong WOF_PROVIDER_FILE = 2UL; + public const int _maxRetries = 3; - private readonly Dictionary _clusterSizes; private readonly ConcurrentDictionary _pendingCompactions; - private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo; private readonly ILogger _logger; - private readonly LightlessConfigService _lightlessConfigService; private readonly DalamudUtilService _dalamudUtilService; + private readonly Channel _compactionQueue; private readonly CancellationTokenSource _compactionCts = new(); - private readonly Task _compactionWorker; - public FileCompactor(ILogger logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService) + private readonly List _workers = []; + private readonly SemaphoreSlim _globalGate; + private static readonly SemaphoreSlim _btrfsGate = new(4, 4); + private readonly BatchFilefragService _fragBatch; + + private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo = new() { - _clusterSizes = new(StringComparer.Ordinal); - _pendingCompactions = new(StringComparer.OrdinalIgnoreCase); - _logger = logger; - _lightlessConfigService = lightlessConfigService; - _dalamudUtilService = dalamudUtilService; - _efInfo = new WOF_FILE_COMPRESSION_INFO_V1 - { - Algorithm = CompressionAlgorithm.XPRESS8K, - Flags = 0 - }; + Algorithm = (int)CompressionAlgorithm.XPRESS8K, + Flags = 0 + }; - _compactionQueue = Channel.CreateUnbounded(new UnboundedChannelOptions - { - SingleReader = true, - SingleWriter = false - }); - _compactionWorker = Task.Factory.StartNew( - () => ProcessQueueAsync(_compactionCts.Token), - _compactionCts.Token, - TaskCreationOptions.LongRunning, - TaskScheduler.Default) - .Unwrap(); + [StructLayout(LayoutKind.Sequential, Pack = 1)] + private struct WOF_FILE_COMPRESSION_INFO_V1 + { + public int Algorithm; + public ulong Flags; } - private enum CompressionAlgorithm + private enum CompressionAlgorithm { NO_COMPRESSION = -2, LZNT1 = -1, @@ -61,230 +53,844 @@ public sealed class FileCompactor : IDisposable XPRESS16K = 3 } - public bool MassCompactRunning { get; private set; } = false; + public FileCompactor(ILogger logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService) + { + _pendingCompactions = new(StringComparer.OrdinalIgnoreCase); + _logger = logger; + _lightlessConfigService = lightlessConfigService; + _dalamudUtilService = dalamudUtilService; + _compactionQueue = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = false, + SingleWriter = false + }); + + int workers = Math.Clamp(Math.Min(Environment.ProcessorCount / 2, 4), 1, 8); + _globalGate = new SemaphoreSlim(workers, workers); + int workerCount = Math.Max(workers * 2, workers); + + for (int i = 0; i < workerCount; i++) + { + _workers.Add(Task.Factory.StartNew( + () => ProcessQueueWorkerAsync(_compactionCts.Token), + _compactionCts.Token, + TaskCreationOptions.LongRunning, + TaskScheduler.Default).Unwrap()); + } + + _fragBatch = new BatchFilefragService( + useShell: _dalamudUtilService.IsWine, + log: _logger, + batchSize: 128, + flushMs: 25); + + _logger.LogInformation("FileCompactor started with {workers} workers", workerCount); + } + + public bool MassCompactRunning { get; private set; } public string Progress { get; private set; } = string.Empty; + /// + /// Compact the storage of the Cache Folder + /// + /// Used to check if files needs to be compressed public void CompactStorage(bool compress) { MassCompactRunning = true; - - int currentFile = 1; - var allFiles = Directory.EnumerateFiles(_lightlessConfigService.Current.CacheFolder).ToList(); - int allFilesCount = allFiles.Count; - foreach (var file in allFiles) - { - Progress = $"{currentFile}/{allFilesCount}"; - if (compress) - CompactFile(file); - else - DecompressFile(file); - currentFile++; - } - - MassCompactRunning = false; - } - - public long GetFileSizeOnDisk(FileInfo fileInfo, bool? isNTFS = null) - { - bool ntfs = isNTFS ?? string.Equals(new DriveInfo(fileInfo.Directory!.Root.FullName).DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase); - - if (_dalamudUtilService.IsWine || !ntfs) return fileInfo.Length; - - var clusterSize = GetClusterSize(fileInfo); - if (clusterSize == -1) return fileInfo.Length; - var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); - var size = (long)hosize << 32 | losize; - return ((size + clusterSize - 1) / clusterSize) * clusterSize; - } - - public async Task WriteAllBytesAsync(string filePath, byte[] decompressedFile, CancellationToken token) - { - await File.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false); - - if (_dalamudUtilService.IsWine || !_lightlessConfigService.Current.UseCompactor) - { - return; - } - - EnqueueCompaction(filePath); - } - - public void Dispose() - { - _compactionQueue.Writer.TryComplete(); - _compactionCts.Cancel(); try { - if (!_compactionWorker.Wait(TimeSpan.FromSeconds(5))) + var allFiles = Directory.EnumerateFiles(_lightlessConfigService.Current.CacheFolder).ToList(); + int total = allFiles.Count; + int current = 0; + + foreach (var file in allFiles) { - _logger.LogDebug("Compaction worker did not shut down within timeout"); + current++; + Progress = $"{current}/{total}"; + + try + { + // Compress or decompress files + if (compress) + CompactFile(file); + else + DecompressFile(file); + } + catch (IOException ioEx) + { + _logger.LogDebug(ioEx, "File {file} locked or busy, skipping", file); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error compacting/decompressing file {file}", file); + } } } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogDebug(ex, "Error shutting down compaction worker"); - } finally { - _compactionCts.Dispose(); - } - - GC.SuppressFinalize(this); - } - - [DllImport("kernel32.dll")] - private static extern int DeviceIoControl(IntPtr hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out IntPtr lpBytesReturned, out IntPtr lpOverlapped); - - [DllImport("kernel32.dll")] - private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, - [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh); - - [DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)] - private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, - out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters, - out uint lpTotalNumberOfClusters); - - [DllImport("WoFUtil.dll")] - private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength); - - [DllImport("WofUtil.dll")] - private static extern int WofSetFileDataLocation(IntPtr FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); - - private void CompactFile(string filePath) - { - var fs = new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName); - bool isNTFS = string.Equals(fs.DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase); - if (!isNTFS) - { - _logger.LogWarning("Drive for file {file} is not NTFS", filePath); - return; - } - - var fi = new FileInfo(filePath); - var oldSize = fi.Length; - var clusterSize = GetClusterSize(fi); - - if (oldSize < Math.Max(clusterSize, 8 * 1024)) - { - _logger.LogDebug("File {file} is smaller than cluster size ({size}), ignoring", filePath, clusterSize); - return; - } - - if (!IsCompactedFile(filePath)) - { - _logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath); - - WOFCompressFile(filePath); - - var newSize = GetFileSizeOnDisk(fi); - - _logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); - } - else - { - _logger.LogDebug("File {file} already compressed", filePath); + MassCompactRunning = false; + Progress = string.Empty; } } - private void DecompressFile(string path) + /// + /// Write all bytes into a directory async + /// + /// Bytes will be writen to this filepath + /// Bytes that have to be written + /// Cancellation Token for interupts + /// Writing Task + public async Task WriteAllBytesAsync(string filePath, byte[] bytes, CancellationToken token) + { + var dir = Path.GetDirectoryName(filePath); + if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) + Directory.CreateDirectory(dir); + + await File.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false); + + if (_lightlessConfigService.Current.UseCompactor) + EnqueueCompaction(filePath); + } + + /// + /// Gets the File size for an BTRFS or NTFS file system for the given FileInfo + /// + /// Amount of blocks used in the disk + public long GetFileSizeOnDisk(FileInfo fileInfo) + { + var fsType = GetFilesystemType(fileInfo.FullName, _dalamudUtilService.IsWine); + + if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) + { + (bool flowControl, long value) = GetFileSizeNTFS(fileInfo); + if (!flowControl) + { + return value; + } + } + + if (fsType == FilesystemType.Btrfs) + { + (bool flowControl, long value) = GetFileSizeBtrfs(fileInfo); + if (!flowControl) + { + return value; + } + } + + return fileInfo.Length; + } + + /// + /// Get File Size in an Btrfs file system (Linux/Wine). + /// + /// File that you want the size from. + /// Succesful check and value of the filesize. + /// Fails on the Process in StartProcessInfo + private (bool flowControl, long value) GetFileSizeBtrfs(FileInfo fileInfo) { - _logger.LogDebug("Removing compression from {file}", path); try { - using (var fs = new FileStream(path, FileMode.Open)) - { -#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called - var hDevice = fs.SafeFileHandle.DangerousGetHandle(); -#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called - _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _); - } + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(fileInfo.FullName, isWine) : fileInfo.FullName; + + (bool ok, string stdout, string stderr, int code) = + RunProcessDirect("stat", ["-c", "%b", realPath]); + + if (!ok || !long.TryParse(stdout.Trim(), out var blocks)) + throw new InvalidOperationException($"stat failed (exit {code}): {stderr}"); + + return (flowControl: false, value: blocks * 512L); } catch (Exception ex) { - _logger.LogWarning(ex, "Error decompressing file {path}", path); + _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); } + + return (flowControl: true, value: default); } - private int GetClusterSize(FileInfo fi) + /// + /// Get File Size in an NTFS file system (Windows). + /// + /// File that you want the size from. + /// Succesful check and value of the filesize. + private (bool flowControl, long value) GetFileSizeNTFS(FileInfo fileInfo) { - if (!fi.Exists) return -1; - var root = fi.Directory?.Root.FullName.ToLower() ?? string.Empty; - if (string.IsNullOrEmpty(root)) return -1; - if (_clusterSizes.TryGetValue(root, out int value)) return value; - _logger.LogDebug("Getting Cluster Size for {path}, root {root}", fi.FullName, root); - int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _); - if (result == 0) return -1; - _clusterSizes[root] = (int)(sectorsPerCluster * bytesPerSector); - _logger.LogDebug("Determined Cluster Size for root {root}: {cluster}", root, _clusterSizes[root]); - return _clusterSizes[root]; - } - - private static bool IsCompactedFile(string filePath) - { - uint buf = 8; - _ = WofIsExternalFile(filePath, out int isExtFile, out uint _, out var info, ref buf); - if (isExtFile == 0) return false; - return info.Algorithm == CompressionAlgorithm.XPRESS8K; - } - - private void WOFCompressFile(string path) - { - var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo)); - Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: true); - ulong length = (ulong)Marshal.SizeOf(_efInfo); try { - using (var fs = new FileStream(path, FileMode.Open)) + var blockSize = GetBlockSizeForPath(fileInfo.FullName, _logger, _dalamudUtilService.IsWine); + var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); + var size = (long)hosize << 32 | losize; + return (flowControl: false, value: ((size + blockSize - 1) / blockSize) * blockSize); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); + } + + return (flowControl: true, value: default); + } + + /// + /// Compressing the given path with BTRFS or NTFS file system. + /// + /// Path of the decompressed/normal file + private void CompactFile(string filePath) + { + var fi = new FileInfo(filePath); + if (!fi.Exists) + { + _logger.LogTrace("Skip compaction: missing {file}", filePath); + return; + } + + var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); + var oldSize = fi.Length; + int blockSize = GetBlockSizeForPath(fi.FullName, _logger, _dalamudUtilService.IsWine); + + // We skipping small files (128KiB) as they slow down the system a lot for BTRFS. as BTRFS has a different blocksize it requires an different calculation. + long minSizeBytes = fsType == FilesystemType.Btrfs + ? Math.Max(blockSize * 2L, 128 * 1024L) + : Math.Max(blockSize, 8 * 1024L); + + if (oldSize < minSizeBytes) + { + _logger.LogTrace("Skip compaction: {file} ({size} B) < threshold ({th} B)", filePath, oldSize, minSizeBytes); + return; + } + + if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) + { + if (!IsWOFCompactedFile(filePath)) { -#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called - var hFile = fs.SafeFileHandle.DangerousGetHandle(); -#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called - if (fs.SafeFileHandle.IsInvalid) + _logger.LogDebug("NTFS compaction XPRESS8K: {file}", filePath); + if (WOFCompressFile(filePath)) { - _logger.LogWarning("Invalid file handle to {file}", path); + var newSize = GetFileSizeOnDisk(fi); + _logger.LogDebug("NTFS compressed {file} {old} -> {new}", filePath, oldSize, newSize); } else { - var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length); - if (!(ret == 0 || ret == unchecked((int)0x80070158))) - { - _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); - } + _logger.LogWarning("NTFS compression failed or unavailable for {file}", filePath); } } + else + { + _logger.LogTrace("Already NTFS-compressed: {file}", filePath); + } + return; + } + + if (fsType == FilesystemType.Btrfs) + { + if (!IsBtrfsCompressedFile(filePath)) + { + _logger.LogDebug("Btrfs compression zstd: {file}", filePath); + if (BtrfsCompressFile(filePath)) + { + var newSize = GetFileSizeOnDisk(fi); + _logger.LogDebug("Btrfs compressed {file} {old} -> {new}", filePath, oldSize, newSize); + } + else + { + _logger.LogWarning("Btrfs compression failed or unavailable for {file}", filePath); + } + } + else + { + _logger.LogTrace("Already Btrfs-compressed: {file}", filePath); + } + return; + } + + _logger.LogTrace("Skip compact: unsupported FS for {file}", filePath); + } + + /// + /// Decompressing the given path with BTRFS file system or NTFS file system. + /// + /// Path of the compressed file + private void DecompressFile(string path) + { + _logger.LogDebug("Decompress request: {file}", path); + var fsType = GetFilesystemType(path, _dalamudUtilService.IsWine); + + if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) + { + try + { + bool flowControl = DecompressWOFFile(path); + if (!flowControl) + { + return; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "NTFS decompress error {file}", path); + } + } + + if (fsType == FilesystemType.Btrfs) + { + try + { + bool flowControl = DecompressBtrfsFile(path); + if (!flowControl) + { + return; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Btrfs decompress error {file}", path); + } + } + } + + /// + /// Decompress an BTRFS File + /// + /// Path of the compressed file + /// Decompressing state + private bool DecompressBtrfsFile(string path) + { + try + { + _btrfsGate.Wait(_compactionCts.Token); + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; + + var mountOptions = GetMountOptionsForPath(realPath); + if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) + { + _logger.LogWarning( + "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " + + "Remount with 'compress=no' before running decompression.", + realPath, mountOptions); + return false; + } + + if (!IsBtrfsCompressedFile(realPath)) + { + _logger.LogTrace("File {file} is not compressed, skipping decompression.", realPath); + return true; + } + + if (!ProbeFileReadable(realPath)) + return false; + + (bool ok, string stdout, string stderr, int code) = + isWine + ? RunProcessShell($"btrfs filesystem defragment -- {QuoteSingle(realPath)}") + : RunProcessDirect("btrfs", ["filesystem", "defragment", "--", realPath]); + + if (!ok) + { + _logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit {code}): {stderr}", + realPath, code, stderr); + return false; + } + + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); + + _logger.LogInformation("Decompressed (rewritten) Btrfs file: {file}", realPath); + return true; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path); + return false; + } + finally + { + if (_btrfsGate.CurrentCount < 4) + _btrfsGate.Release(); + } + } + + /// + /// Decompress an NTFS File + /// + /// Path of the compressed file + /// Decompressing state + private bool DecompressWOFFile(string path) + { + if (TryIsWofExternal(path, out bool isExternal, out int algo)) + { + if (!isExternal) + { + _logger.LogTrace("Already decompressed file: {file}", path); + return true; + } + var compressString = ((CompressionAlgorithm)algo).ToString(); + _logger.LogTrace("WOF compression (algo={algo}) detected for {file}", compressString, path); + } + + return WithFileHandleForWOF(path, FileAccess.ReadWrite, h => + { + if (!DeviceIoControl(h, FSCTL_DELETE_EXTERNAL_BACKING, + IntPtr.Zero, 0, IntPtr.Zero, 0, + out uint _, IntPtr.Zero)) + { + int err = Marshal.GetLastWin32Error(); + // 342 error code means its been decompressed after the control, we handle it as it succesfully been decompressed. + if (err == 342) + { + _logger.LogTrace("Successfully decompressed NTFS file {file}", path); + return true; + } + + _logger.LogWarning("DeviceIoControl failed for {file} with Win32 error {err}", path, err); + return false; + } + + _logger.LogTrace("Successfully decompressed NTFS file {file}", path); + return true; + }); + } + + /// + /// Converts to Linux Path if its using Wine (diferent pathing system in Wine) + /// + /// Path that has to be converted + /// Extra check if using the wine enviroment + /// Converted path to be used in Linux + private string ToLinuxPathIfWine(string path, bool isWine) + { + if (!IsProbablyWine() && !isWine) + return path; + + string linuxPath = path; + if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + linuxPath = "/" + path[3..].Replace('\\', '/'); + else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + linuxPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/'); + + _logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", linuxPath); + return linuxPath; + } + + /// + /// Compress an WOF File + /// + /// Path of the decompressed/normal file + /// Compessing state + private bool WOFCompressFile(string path) + { + int size = Marshal.SizeOf(); + IntPtr efInfoPtr = Marshal.AllocHGlobal(size); + + try + { + Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: false); + ulong length = (ulong)size; + + return WithFileHandleForWOF(path, FileAccess.ReadWrite, h => + { + int ret = WofSetFileDataLocation(h, WOF_PROVIDER_FILE, efInfoPtr, length); + + // 0x80070158 is the benign "already compressed/unsupported" style return + if (ret != 0 && ret != unchecked((int)0x80070158)) + { + _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); + return false; + } + + return true; + }); + } + catch (DllNotFoundException ex) + { + _logger.LogTrace(ex, "WofUtil not available; skipping NTFS compaction for {file}", path); + return false; + } + catch (EntryPointNotFoundException ex) + { + _logger.LogTrace(ex, "WOF entrypoint missing on this system (Wine/older OS); skipping NTFS compaction for {file}", path); + return false; } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting file {path}", path); + return false; } finally { - Marshal.FreeHGlobal(efInfoPtr); + if (efInfoPtr != IntPtr.Zero) + Marshal.FreeHGlobal(efInfoPtr); } } - private struct WOF_FILE_COMPRESSION_INFO_V1 + /// + /// Checks if an File is compacted with WOF compression + /// + /// Path of the file + /// State of the file + private static bool IsWOFCompactedFile(string filePath) { - public CompressionAlgorithm Algorithm; - public ulong Flags; + try + { + uint buf = (uint)Marshal.SizeOf(); + int result = WofIsExternalFile(filePath, out int isExternal, out _, out var info, ref buf); + if (result != 0 || isExternal == 0) + return false; + + return info.Algorithm == (int)CompressionAlgorithm.XPRESS8K + || info.Algorithm == (int)CompressionAlgorithm.XPRESS4K + || info.Algorithm == (int)CompressionAlgorithm.XPRESS16K + || info.Algorithm == (int)CompressionAlgorithm.LZX + || info.Algorithm == (int)CompressionAlgorithm.LZNT1 + || info.Algorithm == (int)CompressionAlgorithm.NO_COMPRESSION; + } + catch + { + return false; + } } + /// + /// Checks if an File is compacted any WOF compression with an WOF backing + /// + /// Path of the file + /// State of the file, if its an external (no backing) and which algorithm if detected + private static bool TryIsWofExternal(string path, out bool isExternal, out int algorithm) + { + isExternal = false; + algorithm = 0; + try + { + uint buf = (uint)Marshal.SizeOf(); + int hr = WofIsExternalFile(path, out int ext, out _, out var info, ref buf); + if (hr == 0 && ext != 0) + { + isExternal = true; + algorithm = info.Algorithm; + } + return true; + } + catch (DllNotFoundException) + { + return false; + } + catch (EntryPointNotFoundException) + { + return false; + } + } + + /// + /// Checks if an File is compacted with Btrfs compression + /// + /// Path of the file + /// State of the file + private bool IsBtrfsCompressedFile(string path) + { + try + { + _btrfsGate.Wait(_compactionCts.Token); + + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; + + return _fragBatch.IsCompressedAsync(realPath, _compactionCts.Token).GetAwaiter().GetResult(); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed to detect Btrfs compression for {file}", path); + return false; + } + finally + { + if (_btrfsGate.CurrentCount < 4) + _btrfsGate.Release(); + } + } + + /// + /// Compress an Btrfs File + /// + /// Path of the decompressed/normal file + /// Compessing state + private bool BtrfsCompressFile(string path) + { + try + { + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; + + var fi = new FileInfo(realPath); + + if (fi == null) + { + _logger.LogWarning("Failed to open {file} for compression; skipping", realPath); + return false; + } + + if (IsBtrfsCompressedFile(realPath)) + { + _logger.LogTrace("File {file} already compressed (Btrfs), skipping file", realPath); + return true; + } + + if (!ProbeFileReadable(realPath)) + return false; + + (bool ok, string stdout, string stderr, int code) = + isWine + ? RunProcessShell($"btrfs filesystem defragment -clzo -- {QuoteSingle(realPath)}") + : RunProcessDirect("btrfs", ["filesystem", "defragment", "-clzo", "--", realPath]); + + if (!ok) + { + _logger.LogWarning("btrfs defragment failed for {file} (exit {code}): {stderr}", realPath, code, stderr); + return false; + } + + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs output for {file}: {stdout}", realPath, stdout.Trim()); + + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); + + _logger.LogInformation("Compressed btrfs file successfully: {file}", realPath); + + return true; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); + return false; + } + } + + + /// + /// Probe file if its readable for certain amount of tries. + /// + /// Path where the file is located + /// Filestream used for the function + /// State of the filestream opening + private bool ProbeFileReadable(string path) + { + for (int attempt = 0; attempt < _maxRetries; attempt++) + { + try + { + using var _ = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + return true; + } + catch (IOException ex) + { + if (attempt == _maxRetries - 1) + { + _logger.LogWarning(ex, "File still in use after {attempts} attempts, skipping {file}", _maxRetries, path); + return false; + } + int delay = 150 * (attempt + 1); + _logger.LogTrace(ex, "File busy, retrying in {delay}ms for {file}", delay, path); + Thread.Sleep(delay); + } + } + return false; + } + + /// + /// Attempt opening file stream for WOF functions + /// + /// File that has to be accessed + /// Permissions for the file + /// Access of the file stream for the WOF function to handle. + /// State of the attempt for the file + private bool WithFileHandleForWOF(string path, FileAccess access, Func body) + { + const FileShare share = FileShare.ReadWrite | FileShare.Delete; + + for (int attempt = 0; attempt < _maxRetries; attempt++) + { + try + { + using var fs = new FileStream(path, FileMode.Open, access, share); + + var handle = fs.SafeFileHandle; + if (handle.IsInvalid) + { + _logger.LogWarning("Invalid file handle for {file}", path); + return false; + } + + return body(handle); + } + catch (IOException ex) + { + if (attempt == _maxRetries - 1) + { + _logger.LogWarning(ex, "File still in use after {attempts} attempts, skipping {file}", _maxRetries, path); + return false; + } + + int delay = 150 * (attempt + 1); + _logger.LogTrace(ex, "File busy, retrying in {delay}ms for {file}", delay, path); + Thread.Sleep(delay); + } + } + + return false; + } + + /// + /// Runs an nonshell process meant for Linux/Wine enviroments + /// + /// File that has to be excuted + /// Arguments meant for the file/command + /// Working directory used to execute the file with/without arguments + /// Timeout timer for the process + /// State of the process, output of the process and error with exit code + private (bool ok, string stdout, string stderr, int exitCode) RunProcessDirect(string fileName, IEnumerable args, string? workingDir = null, int timeoutMs = 60000) + { + var psi = new ProcessStartInfo(fileName) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir; + + foreach (var a in args) psi.ArgumentList.Add(a); + + using var proc = Process.Start(psi); + if (proc is null) return (false, "", "failed to start process", -1); + + var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token); + var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token); + + if (!proc.WaitForExit(timeoutMs)) + { + try + { + proc.Kill(entireProcessTree: true); + } + catch + { + // Ignore this catch on the dispose + } + + Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); + return (false, outTask.Result, "timeout", -1); + } + + Task.WaitAll(outTask, errTask); + return (proc.ExitCode == 0, outTask.Result, errTask.Result, proc.ExitCode); + } + + /// + /// Runs an shell using '/bin/bash'/ command meant for Linux/Wine enviroments + /// + /// Command that has to be excuted + /// Timeout timer for the process + /// State of the process, output of the process and error with exit code + private (bool ok, string stdout, string stderr, int exitCode) RunProcessShell(string command, int timeoutMs = 60000) + { + var psi = new ProcessStartInfo("/bin/bash") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + psi.ArgumentList.Add("-c"); + psi.ArgumentList.Add(command); + + using var proc = Process.Start(psi); + if (proc is null) return (false, "", "failed to start /bin/bash", -1); + + var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token); + var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token); + + if (!proc.WaitForExit(timeoutMs)) + { + try + { + proc.Kill(entireProcessTree: true); + } + catch + { + // Ignore this catch on the dispose + } + + Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); + return (false, outTask.Result, "timeout", -1); + } + + Task.WaitAll(outTask, errTask); + return (proc.ExitCode == 0, outTask.Result, errTask.Result, proc.ExitCode); + } + + /// + /// Enqueues the compaction/decompation of an filepath. + /// + /// Filepath that will be enqueued private void EnqueueCompaction(string filePath) { - if (!_pendingCompactions.TryAdd(filePath, 0)) - { + // Safe-checks + if (string.IsNullOrWhiteSpace(filePath)) return; - } - if (!_compactionQueue.Writer.TryWrite(filePath)) + if (!_lightlessConfigService.Current.UseCompactor) + return; + + if (!File.Exists(filePath)) + return; + + if (!_pendingCompactions.TryAdd(filePath, 0)) + return; + + bool enqueued = false; + try { - _pendingCompactions.TryRemove(filePath, out _); - _logger.LogDebug("Failed to enqueue compaction job for {file}", filePath); + bool isWine = _dalamudUtilService?.IsWine ?? false; + var fsType = GetFilesystemType(filePath, isWine); + + // If under Wine, we should skip NTFS because its not Windows but might return NTFS. + if (fsType == FilesystemType.NTFS && isWine) + { + _logger.LogTrace("Skip enqueue (NTFS under Wine) {file}", filePath); + return; + } + + // Unknown file system should be skipped. + if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) + { + _logger.LogTrace("Skip enqueue (unsupported fs) {fs} {file}", fsType, filePath); + return; + } + + // Channel got closed, skip enqueue on file + if (!_compactionQueue.Writer.TryWrite(filePath)) + { + _logger.LogTrace("Skip enqueue: compaction channel is/got closed {file}", filePath); + return; + } + + enqueued = true; + _logger.LogTrace("Queued compaction for {file} (fs={fs})", filePath, fsType); + } + finally + { + if (!enqueued) + _pendingCompactions.TryRemove(filePath, out _); } } - private async Task ProcessQueueAsync(CancellationToken token) + /// + /// Process the queue with, meant for a worker/thread + /// + /// Cancellation token for the worker whenever it needs to be stopped + private async Task ProcessQueueWorkerAsync(CancellationToken token) { try { @@ -294,28 +900,20 @@ public sealed class FileCompactor : IDisposable { try { - if (token.IsCancellationRequested) - { - return; - } + token.ThrowIfCancellationRequested(); + await _globalGate.WaitAsync(token).ConfigureAwait(false); - if (_dalamudUtilService.IsWine || !_lightlessConfigService.Current.UseCompactor) + try { - continue; + if (_lightlessConfigService.Current.UseCompactor && File.Exists(filePath)) + CompactFile(filePath); } - - if (!File.Exists(filePath)) + finally { - _logger.LogTrace("Skipping compaction for missing file {file}", filePath); - continue; + _globalGate.Release(); } - - CompactFile(filePath); - } - catch (OperationCanceledException) - { - return; } + catch (OperationCanceledException) { return; } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting file {file}", filePath); @@ -327,9 +925,45 @@ public sealed class FileCompactor : IDisposable } } } - catch (OperationCanceledException) - { - // expected during shutdown + catch (OperationCanceledException) + { + // Shutting down worker, this exception is expected } } + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped); + + [DllImport("kernel32.dll")] + private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh); + + [DllImport("WofUtil.dll")] + private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength); + + [DllImport("WofUtil.dll", SetLastError = true)] + private static extern int WofSetFileDataLocation(SafeFileHandle FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); + + private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; + + public void Dispose() + { + _fragBatch?.Dispose(); + _compactionQueue.Writer.TryComplete(); + _compactionCts.Cancel(); + + try + { + Task.WaitAll([.. _workers.Where(t => t != null)], TimeSpan.FromSeconds(5)); + } + catch + { + // Ignore this catch on the dispose + } + finally + { + _compactionCts.Dispose(); + } + + GC.SuppressFinalize(this); + } } diff --git a/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs b/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs index 2b003bb..20302fe 100644 --- a/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs +++ b/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs @@ -13,5 +13,4 @@ public class ServerStorage public bool UseOAuth2 { get; set; } = false; public string? OAuthToken { get; set; } = null; public HttpTransportType HttpTransportType { get; set; } = HttpTransportType.WebSockets; - public bool ForceWebSockets { get; set; } = false; } \ No newline at end of file diff --git a/LightlessSync/LightlessSync.csproj b/LightlessSync/LightlessSync.csproj index b4b5288..726f2ef 100644 --- a/LightlessSync/LightlessSync.csproj +++ b/LightlessSync/LightlessSync.csproj @@ -3,7 +3,7 @@ - 1.12.3 + 1.12.4 https://github.com/Light-Public-Syncshells/LightlessClient diff --git a/LightlessSync/Services/BroadcastScanningService.cs b/LightlessSync/Services/BroadcastScanningService.cs index 79fe984..95abdae 100644 --- a/LightlessSync/Services/BroadcastScanningService.cs +++ b/LightlessSync/Services/BroadcastScanningService.cs @@ -28,7 +28,7 @@ public class BroadcastScannerService : DisposableMediatorSubscriberBase, IDispos private readonly CancellationTokenSource _cleanupCts = new(); private Task? _cleanupTask; - private int _checkEveryFrames = 20; + private readonly int _checkEveryFrames = 20; private int _frameCounter = 0; private int _lookupsThisFrame = 0; private const int MaxLookupsPerFrame = 30; @@ -221,6 +221,16 @@ public class BroadcastScannerService : DisposableMediatorSubscriberBase, IDispos (excludeHashedCid is null || !comparer.Equals(entry.Key, excludeHashedCid))); } + public List> GetActiveBroadcasts(string? excludeHashedCid = null) + { + var now = DateTime.UtcNow; + var comparer = StringComparer.Ordinal; + return [.. _broadcastCache.Where(entry => + entry.Value.IsBroadcasting && + entry.Value.ExpiryTime > now && + (excludeHashedCid is null || !comparer.Equals(entry.Key, excludeHashedCid)))]; + } + protected override void Dispose(bool disposing) { base.Dispose(disposing); diff --git a/LightlessSync/Services/BroadcastService.cs b/LightlessSync/Services/BroadcastService.cs index 81e54d5..cca9af6 100644 --- a/LightlessSync/Services/BroadcastService.cs +++ b/LightlessSync/Services/BroadcastService.cs @@ -144,11 +144,7 @@ public class BroadcastService : IHostedService, IMediatorSubscriber IsLightFinderAvailable = false; ApplyBroadcastDisabled(forcePublish: true); _logger.LogDebug("Cleared Lightfinder state due to disconnect."); - - _mediator.Publish(new NotificationMessage( - "Disconnected from Server", - "Your Lightfinder broadcast has been disabled due to disconnection.", - NotificationType.Warning)); + } public Task StartAsync(CancellationToken cancellationToken) diff --git a/LightlessSync/Services/CommandManagerService.cs b/LightlessSync/Services/CommandManagerService.cs index 7aedc7b..88f8780 100644 --- a/LightlessSync/Services/CommandManagerService.cs +++ b/LightlessSync/Services/CommandManagerService.cs @@ -13,7 +13,8 @@ namespace LightlessSync.Services; public sealed class CommandManagerService : IDisposable { - private const string _commandName = "/light"; + private const string _longName = "/lightless"; + private const string _shortName = "/light"; private readonly ApiController _apiController; private readonly ICommandManager _commandManager; @@ -34,7 +35,11 @@ public sealed class CommandManagerService : IDisposable _apiController = apiController; _mediator = mediator; _lightlessConfigService = lightlessConfigService; - _commandManager.AddHandler(_commandName, new CommandInfo(OnCommand) + _commandManager.AddHandler(_longName, new CommandInfo(OnCommand) + { + HelpMessage = $"\u2191;" + }); + _commandManager.AddHandler(_shortName, new CommandInfo(OnCommand) { HelpMessage = "Opens the Lightless Sync UI" + Environment.NewLine + Environment.NewLine + "Additionally possible commands:" + Environment.NewLine + @@ -49,7 +54,8 @@ public sealed class CommandManagerService : IDisposable public void Dispose() { - _commandManager.RemoveHandler(_commandName); + _commandManager.RemoveHandler(_longName); + _commandManager.RemoveHandler(_shortName); } private void OnCommand(string command, string args) diff --git a/LightlessSync/Services/Compression/BatchFileFragService.cs b/LightlessSync/Services/Compression/BatchFileFragService.cs new file mode 100644 index 0000000..ae5bb71 --- /dev/null +++ b/LightlessSync/Services/Compression/BatchFileFragService.cs @@ -0,0 +1,245 @@ +using Microsoft.Extensions.Logging; +using System.Diagnostics; +using System.Text.RegularExpressions; +using System.Threading.Channels; + +namespace LightlessSync.Services.Compression +{ + /// + /// This batch service is made for the File Frag command, because of each file needing to use this command. + /// It's better to combine into one big command in batches then doing each command on each compressed call. + /// + public sealed partial class BatchFilefragService : IDisposable + { + private readonly Channel<(string path, TaskCompletionSource tcs)> _ch; + private readonly Task _worker; + private readonly bool _useShell; + private readonly ILogger _log; + private readonly int _batchSize; + private readonly TimeSpan _flushDelay; + private readonly CancellationTokenSource _cts = new(); + + public BatchFilefragService(bool useShell, ILogger log, int batchSize = 128, int flushMs = 25) + { + _useShell = useShell; + _log = log; + _batchSize = Math.Max(8, batchSize); + _flushDelay = TimeSpan.FromMilliseconds(Math.Max(5, flushMs)); + _ch = Channel.CreateUnbounded<(string, TaskCompletionSource)>(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); + _worker = Task.Run(ProcessAsync, _cts.Token); + } + + /// + /// Checks if the file is compressed using Btrfs using tasks + /// + /// Linux/Wine path given for the file. + /// Cancellation Token + /// If it was compressed or not + public Task IsCompressedAsync(string linuxPath, CancellationToken ct = default) + { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + if (!_ch.Writer.TryWrite((linuxPath, tcs))) + { + tcs.TrySetResult(false); + return tcs.Task; + } + + if (ct.CanBeCanceled) + { + var reg = ct.Register(() => tcs.TrySetCanceled(ct)); + _ = tcs.Task.ContinueWith(_ => reg.Dispose(), CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + } + + return tcs.Task; + } + + /// + /// Process the pending compression tasks asynchronously + /// + /// Task + private async Task ProcessAsync() + { + var reader = _ch.Reader; + var pending = new List<(string path, TaskCompletionSource tcs)>(_batchSize); + + try + { + while (await reader.WaitToReadAsync(_cts.Token).ConfigureAwait(false)) + { + if (!reader.TryRead(out var first)) continue; + pending.Add(first); + + var flushAt = DateTime.UtcNow + _flushDelay; + while (pending.Count < _batchSize && DateTime.UtcNow < flushAt) + { + if (reader.TryRead(out var item)) + { + pending.Add(item); + continue; + } + + if ((flushAt - DateTime.UtcNow) <= TimeSpan.Zero) break; + try + { + await Task.Delay(TimeSpan.FromMilliseconds(5), _cts.Token).ConfigureAwait(false); + } + catch + { + break; + } + } + + try + { + var map = await RunBatchAsync(pending.Select(p => p.path)).ConfigureAwait(false); + foreach (var (path, tcs) in pending) + { + tcs.TrySetResult(map.TryGetValue(path, out var c) && c); + } + } + catch (Exception ex) + { + _log.LogDebug(ex, "filefrag batch failed. falling back to false"); + foreach (var (_, tcs) in pending) + { + tcs.TrySetResult(false); + } + } + finally + { + pending.Clear(); + } + } + } + catch (OperationCanceledException) + { + //Shutting down worker, exception called + } + } + + /// + /// Running the batch of each file in the queue in one file frag command. + /// + /// Paths that are needed for the command building for the batch return + /// Path of the file and if it went correctly + /// Failing to start filefrag on the system if this exception is found + private async Task> RunBatchAsync(IEnumerable paths) + { + var list = paths.Distinct(StringComparer.Ordinal).ToList(); + var result = list.ToDictionary(p => p, _ => false, StringComparer.Ordinal); + + ProcessStartInfo psi; + if (_useShell) + { + var inner = "filefrag -v -- " + string.Join(' ', list.Select(QuoteSingle)); + psi = new ProcessStartInfo + { + FileName = "/bin/bash", + Arguments = "-c " + QuoteDouble(inner), + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = "/" + }; + } + else + { + psi = new ProcessStartInfo + { + FileName = "filefrag", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + psi.ArgumentList.Add("-v"); + psi.ArgumentList.Add("--"); + foreach (var p in list) psi.ArgumentList.Add(p); + } + + using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Failed to start filefrag"); + var stdoutTask = proc.StandardOutput.ReadToEndAsync(_cts.Token); + var stderrTask = proc.StandardError.ReadToEndAsync(_cts.Token); + await Task.WhenAll(stdoutTask, stderrTask).ConfigureAwait(false); + try + { + await proc.WaitForExitAsync(_cts.Token).ConfigureAwait(false); + } + catch (Exception ex) + { + _log.LogWarning(ex, "Error in the batch frag service. proc = {proc}", proc); + } + + var stdout = await stdoutTask.ConfigureAwait(false); + var stderr = await stderrTask.ConfigureAwait(false); + + if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) + _log.LogTrace("filefrag exited {code}: {err}", proc.ExitCode, stderr.Trim()); + + ParseFilefrag(stdout, result); + return result; + } + + /// + /// Parsing the string given from the File Frag command into mapping + /// + /// Output of the process from the File Frag + /// Mapping of the processed files + private static void ParseFilefrag(string output, Dictionary map) + { + var reHeaderColon = ColonRegex(); + var reHeaderSize = SizeRegex(); + + string? current = null; + using var sr = new StringReader(output); + for (string? line = sr.ReadLine(); line != null; line = sr.ReadLine()) + { + var m1 = reHeaderColon.Match(line); + if (m1.Success) { current = m1.Groups[1].Value; continue; } + + var m2 = reHeaderSize.Match(line); + if (m2.Success) { current = m2.Groups[1].Value; continue; } + + if (current is not null && line.Contains("flags:", StringComparison.OrdinalIgnoreCase) && + line.Contains("compressed", StringComparison.OrdinalIgnoreCase) && map.ContainsKey(current)) + { + map[current] = true; + } + } + } + + private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; + private static string QuoteDouble(string s) => "\"" + s.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal).Replace("$", "\\$", StringComparison.Ordinal).Replace("`", "\\`", StringComparison.Ordinal) + "\""; + + /// + /// Regex of the File Size return on the Linux/Wine systems, giving back the amount + /// + /// Regex of the File Size + [GeneratedRegex(@"^File size of (/.+?) is ", RegexOptions.ExplicitCapture | RegexOptions.CultureInvariant,matchTimeoutMilliseconds: 500)] + private static partial Regex SizeRegex(); + + /// + /// Regex on colons return on the Linux/Wine systems + /// + /// Regex of the colons in the given path + [GeneratedRegex(@"^(/.+?):\s", RegexOptions.ExplicitCapture | RegexOptions.CultureInvariant, matchTimeoutMilliseconds: 500)] + private static partial Regex ColonRegex(); + + public void Dispose() + { + _ch.Writer.TryComplete(); + _cts.Cancel(); + try + { + _worker.Wait(TimeSpan.FromSeconds(2), _cts.Token); + } + catch + { + // Ignore the catch in dispose + } + _cts.Dispose(); + } + } +} \ No newline at end of file diff --git a/LightlessSync/Services/ContextMenuService.cs b/LightlessSync/Services/ContextMenuService.cs index 97bfc17..464fee1 100644 --- a/LightlessSync/Services/ContextMenuService.cs +++ b/LightlessSync/Services/ContextMenuService.cs @@ -98,7 +98,7 @@ internal class ContextMenuService : IHostedService if (targetData == null || targetData.Address == nint.Zero) return; - //Check if user is paired or is own. + //Check if user is directly paired or is own. if (VisibleUserIds.Any(u => u == target.TargetObjectId) || _clientState.LocalPlayer.GameObjectId == target.TargetObjectId) return; @@ -116,7 +116,7 @@ internal class ContextMenuService : IHostedService args.AddMenuItem(new MenuItem { - Name = "Send Pair Request", + Name = "Send Direct Pair Request", PrefixChar = 'L', UseDefaultPrefix = false, PrefixColor = 708, @@ -159,7 +159,7 @@ internal class ContextMenuService : IHostedService } } - private HashSet VisibleUserIds => [.. _pairManager.GetOnlineUserPairs() + private HashSet VisibleUserIds => [.. _pairManager.DirectPairs .Where(u => u.IsVisible && u.PlayerCharacterId != uint.MaxValue) .Select(u => (ulong)u.PlayerCharacterId)]; diff --git a/LightlessSync/Services/NameplateHandler.cs b/LightlessSync/Services/NameplateHandler.cs index a28be5f..11af974 100644 --- a/LightlessSync/Services/NameplateHandler.cs +++ b/LightlessSync/Services/NameplateHandler.cs @@ -1,5 +1,6 @@ using Dalamud.Game.Addon.Lifecycle; using Dalamud.Game.Addon.Lifecycle.AddonArgTypes; +using Dalamud.Game.ClientState.Objects.Enums; using Dalamud.Game.Text; using Dalamud.Plugin.Services; using FFXIVClientStructs.FFXIV.Client.System.Framework; @@ -15,8 +16,9 @@ using LightlessSync.UtilsEnum.Enum; // Created using https://github.com/PunishedPineapple/Distance as a reference, thank you! using Microsoft.Extensions.Logging; +using System.Collections.Generic; +using System.Collections.Immutable; using System.Globalization; -using System.Text; namespace LightlessSync.Services; @@ -32,10 +34,10 @@ public unsafe class NameplateHandler : IMediatorSubscriber private readonly LightlessMediator _mediator; public LightlessMediator Mediator => _mediator; - private bool mEnabled = false; + private bool _mEnabled = false; private bool _needsLabelRefresh = false; - private AddonNamePlate* mpNameplateAddon = null; - private readonly AtkTextNode*[] mTextNodes = new AtkTextNode*[AddonNamePlate.NumNamePlateObjects]; + private AddonNamePlate* _mpNameplateAddon = null; + private readonly AtkTextNode*[] _mTextNodes = new AtkTextNode*[AddonNamePlate.NumNamePlateObjects]; private readonly int[] _cachedNameplateTextWidths = new int[AddonNamePlate.NumNamePlateObjects]; private readonly int[] _cachedNameplateTextHeights = new int[AddonNamePlate.NumNamePlateObjects]; private readonly int[] _cachedNameplateContainerHeights = new int[AddonNamePlate.NumNamePlateObjects]; @@ -44,10 +46,10 @@ public unsafe class NameplateHandler : IMediatorSubscriber internal const uint mNameplateNodeIDBase = 0x7D99D500; private const string DefaultLabelText = "LightFinder"; private const SeIconChar DefaultIcon = SeIconChar.Hyadelyn; - private const int ContainerOffsetX = 50; + private const int _containerOffsetX = 50; private static readonly string DefaultIconGlyph = SeIconCharExtensions.ToIconString(DefaultIcon); - private volatile HashSet _activeBroadcastingCids = []; + private ImmutableHashSet _activeBroadcastingCids = []; public NameplateHandler(ILogger logger, IAddonLifecycle addonLifecycle, IGameGui gameGui, DalamudUtilService dalamudUtil, LightlessConfigService configService, LightlessMediator mediator, IClientState clientState, PairManager pairManager) { @@ -74,17 +76,17 @@ public unsafe class NameplateHandler : IMediatorSubscriber DisableNameplate(); DestroyNameplateNodes(); _mediator.Unsubscribe(this); - mpNameplateAddon = null; + _mpNameplateAddon = null; } internal void EnableNameplate() { - if (!mEnabled) + if (!_mEnabled) { try { _addonLifecycle.RegisterListener(AddonEvent.PostDraw, "NamePlate", NameplateDrawDetour); - mEnabled = true; + _mEnabled = true; } catch (Exception e) { @@ -96,7 +98,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber internal void DisableNameplate() { - if (mEnabled) + if (_mEnabled) { try { @@ -107,24 +109,30 @@ public unsafe class NameplateHandler : IMediatorSubscriber _logger.LogError($"Unknown error while unregistering nameplate listener:\n{e}"); } - mEnabled = false; + _mEnabled = false; HideAllNameplateNodes(); } } private void NameplateDrawDetour(AddonEvent type, AddonArgs args) { + if (args.Addon.Address == nint.Zero) + { + _logger.LogWarning("Nameplate draw detour received a null addon address, skipping update."); + return; + } + var pNameplateAddon = (AddonNamePlate*)args.Addon.Address; - if (mpNameplateAddon != pNameplateAddon) + if (_mpNameplateAddon != pNameplateAddon) { - for (int i = 0; i < mTextNodes.Length; ++i) mTextNodes[i] = null; + for (int i = 0; i < _mTextNodes.Length; ++i) _mTextNodes[i] = null; System.Array.Clear(_cachedNameplateTextWidths, 0, _cachedNameplateTextWidths.Length); System.Array.Clear(_cachedNameplateTextHeights, 0, _cachedNameplateTextHeights.Length); System.Array.Clear(_cachedNameplateContainerHeights, 0, _cachedNameplateContainerHeights.Length); System.Array.Fill(_cachedNameplateTextOffsets, int.MinValue); - mpNameplateAddon = pNameplateAddon; - if (mpNameplateAddon != null) CreateNameplateNodes(); + _mpNameplateAddon = pNameplateAddon; + if (_mpNameplateAddon != null) CreateNameplateNodes(); } UpdateNameplateNodes(); @@ -138,7 +146,16 @@ public unsafe class NameplateHandler : IMediatorSubscriber if (nameplateObject == null) continue; + var rootNode = nameplateObject.Value.RootComponentNode; + if (rootNode == null || rootNode->Component == null) + continue; + var pNameplateResNode = nameplateObject.Value.NameContainer; + if (pNameplateResNode == null) + continue; + if (pNameplateResNode->ChildNode == null) + continue; + var pNewNode = AtkNodeHelpers.CreateOrphanTextNode(mNameplateNodeIDBase + (uint)i, TextFlags.Edge | TextFlags.Glare); if (pNewNode != null) @@ -148,24 +165,43 @@ public unsafe class NameplateHandler : IMediatorSubscriber pNewNode->AtkResNode.NextSiblingNode = pLastChild; pNewNode->AtkResNode.ParentNode = pNameplateResNode; pLastChild->PrevSiblingNode = (AtkResNode*)pNewNode; - nameplateObject.Value.RootComponentNode->Component->UldManager.UpdateDrawNodeList(); + rootNode->Component->UldManager.UpdateDrawNodeList(); pNewNode->AtkResNode.SetUseDepthBasedPriority(true); - mTextNodes[i] = pNewNode; + _mTextNodes[i] = pNewNode; } } } private void DestroyNameplateNodes() { - var pCurrentNameplateAddon = (AddonNamePlate*)_gameGui.GetAddonByName("NamePlate", 1).Address; - if (mpNameplateAddon == null || mpNameplateAddon != pCurrentNameplateAddon) + var currentHandle = _gameGui.GetAddonByName("NamePlate", 1); + if (currentHandle.Address == nint.Zero) + { + _logger.LogWarning("Unable to destroy nameplate nodes because the NamePlate addon is not available."); return; + } + + var pCurrentNameplateAddon = (AddonNamePlate*)currentHandle.Address; + if (_mpNameplateAddon == null) + return; + + if (_mpNameplateAddon != pCurrentNameplateAddon) + { + _logger.LogWarning("Skipping nameplate node destroy due to addon address mismatch (cached {Cached:X}, current {Current:X}).", (IntPtr)_mpNameplateAddon, (IntPtr)pCurrentNameplateAddon); + return; + } for (int i = 0; i < AddonNamePlate.NumNamePlateObjects; ++i) { - var pTextNode = mTextNodes[i]; + var pTextNode = _mTextNodes[i]; var pNameplateNode = GetNameplateComponentNode(i); - if (pTextNode != null && pNameplateNode != null) + if (pTextNode != null && (pNameplateNode == null || pNameplateNode->Component == null)) + { + _logger.LogDebug("Skipping destroy for nameplate {Index} because its component node is unavailable.", i); + continue; + } + + if (pTextNode != null && pNameplateNode != null && pNameplateNode->Component != null) { try { @@ -175,7 +211,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber pTextNode->AtkResNode.NextSiblingNode->PrevSiblingNode = pTextNode->AtkResNode.PrevSiblingNode; pNameplateNode->Component->UldManager.UpdateDrawNodeList(); pTextNode->AtkResNode.Destroy(true); - mTextNodes[i] = null; + _mTextNodes[i] = null; } catch (Exception e) { @@ -192,7 +228,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber private void HideAllNameplateNodes() { - for (int i = 0; i < mTextNodes.Length; ++i) + for (int i = 0; i < _mTextNodes.Length; ++i) { HideNameplateTextNode(i); } @@ -200,22 +236,62 @@ public unsafe class NameplateHandler : IMediatorSubscriber private void UpdateNameplateNodes() { - var framework = Framework.Instance(); - var ui3DModule = framework->GetUIModule()->GetUI3DModule(); + var currentHandle = _gameGui.GetAddonByName("NamePlate"); + if (currentHandle.Address == nint.Zero) + { + _logger.LogDebug("NamePlate addon unavailable during update, skipping label refresh."); + return; + } + var currentAddon = (AddonNamePlate*)currentHandle.Address; + if (_mpNameplateAddon == null || currentAddon == null || currentAddon != _mpNameplateAddon) + { + if (_mpNameplateAddon != null) + _logger.LogDebug("Cached NamePlate addon pointer differs from current: waiting for new hook (cached {Cached:X}, current {Current:X}).", (IntPtr)_mpNameplateAddon, (IntPtr)currentAddon); + return; + } + + var framework = Framework.Instance(); + if (framework == null) + { + _logger.LogDebug("Framework instance unavailable during nameplate update, skipping."); + return; + } + + var uiModule = framework->GetUIModule(); + if (uiModule == null) + { + _logger.LogDebug("UI module unavailable during nameplate update, skipping."); + return; + } + + var ui3DModule = uiModule->GetUI3DModule(); if (ui3DModule == null) + { + _logger.LogDebug("UI3D module unavailable during nameplate update, skipping."); + return; + } + + var vec = ui3DModule->NamePlateObjectInfoPointers; + if (vec.IsEmpty) return; - for (int i = 0; i < ui3DModule->NamePlateObjectInfoCount; ++i) + var visibleUserIdsSnapshot = VisibleUserIds; + + var safeCount = System.Math.Min( + ui3DModule->NamePlateObjectInfoCount, + vec.Length + ); + + for (int i = 0; i < safeCount; ++i) { - if (ui3DModule->NamePlateObjectInfoPointers.IsEmpty) continue; + var config = _configService.Current; - var objectInfoPtr = ui3DModule->NamePlateObjectInfoPointers[i]; - - if (objectInfoPtr == null) continue; + var objectInfoPtr = vec[i]; + if (objectInfoPtr == null) + continue; var objectInfo = objectInfoPtr.Value; - if (objectInfo == null || objectInfo->GameObject == null) continue; @@ -223,62 +299,68 @@ public unsafe class NameplateHandler : IMediatorSubscriber if (nameplateIndex < 0 || nameplateIndex >= AddonNamePlate.NumNamePlateObjects) continue; - var pNode = mTextNodes[nameplateIndex]; + var pNode = _mTextNodes[nameplateIndex]; if (pNode == null) continue; - if (mpNameplateAddon == null) + var gameObject = objectInfo->GameObject; + if ((ObjectKind)gameObject->ObjectKind != ObjectKind.Player) + { + pNode->AtkResNode.ToggleVisibility(enable: false); continue; - - var cid = DalamudUtilService.GetHashedCIDFromPlayerPointer((nint)objectInfo->GameObject); + } + // CID gating + var cid = DalamudUtilService.GetHashedCIDFromPlayerPointer((nint)gameObject); if (cid == null || !_activeBroadcastingCids.Contains(cid)) { - pNode->AtkResNode.ToggleVisibility(false); + pNode->AtkResNode.ToggleVisibility(enable: false); continue; } - if (!_configService.Current.LightfinderLabelShowOwn && (objectInfo->GameObject->GetGameObjectId() == _clientState.LocalPlayer.GameObjectId)) + var local = _clientState.LocalPlayer; + if (!config.LightfinderLabelShowOwn && local != null && + objectInfo->GameObject->GetGameObjectId() == local.GameObjectId) { - pNode->AtkResNode.ToggleVisibility(false); + pNode->AtkResNode.ToggleVisibility(enable: false); continue; } - if (!_configService.Current.LightfinderLabelShowPaired && VisibleUserIds.Any(u => u == objectInfo->GameObject->GetGameObjectId())) + var hidePaired = !config.LightfinderLabelShowPaired; + + var goId = (ulong)gameObject->GetGameObjectId(); + if (hidePaired && visibleUserIdsSnapshot.Contains(goId)) { - pNode->AtkResNode.ToggleVisibility(false); - continue; - } - - var nameplateObject = mpNameplateAddon->NamePlateObjectArray[nameplateIndex]; - nameplateObject.RootComponentNode->Component->UldManager.UpdateDrawNodeList(); - - var pNameplateIconNode = nameplateObject.MarkerIcon; - var pNameplateResNode = nameplateObject.NameContainer; - var pNameplateTextNode = nameplateObject.NameText; - bool IsVisible = pNameplateIconNode->AtkResNode.IsVisible() || (pNameplateResNode->IsVisible() && pNameplateTextNode->AtkResNode.IsVisible()) || _configService.Current.LightfinderLabelShowHidden; - pNode->AtkResNode.ToggleVisibility(IsVisible); - - if (nameplateObject.RootComponentNode == null || - nameplateObject.NameContainer == null || - nameplateObject.NameText == null) - { - pNode->AtkResNode.ToggleVisibility(false); + pNode->AtkResNode.ToggleVisibility(enable: false); continue; } + var nameplateObject = _mpNameplateAddon->NamePlateObjectArray[nameplateIndex]; + var root = nameplateObject.RootComponentNode; var nameContainer = nameplateObject.NameContainer; var nameText = nameplateObject.NameText; + var marker = nameplateObject.MarkerIcon; - if (nameContainer == null || nameText == null) + if (root == null || root->Component == null || nameContainer == null || nameText == null) { - pNode->AtkResNode.ToggleVisibility(false); + _logger.LogDebug("Nameplate {Index} missing required nodes during update, skipping.", nameplateIndex); + pNode->AtkResNode.ToggleVisibility(enable: false); continue; } + + root->Component->UldManager.UpdateDrawNodeList(); + + bool isVisible = + ((marker != null) && marker->AtkResNode.IsVisible()) || + (nameContainer->IsVisible() && nameText->AtkResNode.IsVisible()) || + config.LightfinderLabelShowHidden; + + pNode->AtkResNode.ToggleVisibility(isVisible); + if (!isVisible) + continue; var labelColor = UIColors.Get("Lightfinder"); var edgeColor = UIColors.Get("LightfinderEdge"); - var config = _configService.Current; var scaleMultiplier = System.Math.Clamp(config.LightfinderLabelScale, 0.5f, 2.0f); var baseScale = config.LightfinderLabelUseIcon ? 1.0f : 0.5f; @@ -437,7 +519,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber positionY += config.LightfinderLabelOffsetY; alignment = (AlignmentType)System.Math.Clamp((int)alignment, 0, 8); - pNode->AtkResNode.SetUseDepthBasedPriority(true); + pNode->AtkResNode.SetUseDepthBasedPriority(enable: true); pNode->AtkResNode.Color.A = 255; @@ -545,7 +627,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber } private void HideNameplateTextNode(int i) { - var pNode = mTextNodes[i]; + var pNode = _mTextNodes[i]; if (pNode != null) { pNode->AtkResNode.ToggleVisibility(false); @@ -555,10 +637,10 @@ public unsafe class NameplateHandler : IMediatorSubscriber private AddonNamePlate.NamePlateObject? GetNameplateObject(int i) { if (i < AddonNamePlate.NumNamePlateObjects && - mpNameplateAddon != null && - mpNameplateAddon->NamePlateObjectArray[i].RootComponentNode != null) + _mpNameplateAddon != null && + _mpNameplateAddon->NamePlateObjectArray[i].RootComponentNode != null) { - return mpNameplateAddon->NamePlateObjectArray[i]; + return _mpNameplateAddon->NamePlateObjectArray[i]; } else { @@ -571,10 +653,12 @@ public unsafe class NameplateHandler : IMediatorSubscriber var nameplateObject = GetNameplateObject(i); return nameplateObject != null ? nameplateObject.Value.RootComponentNode : null; } + private HashSet VisibleUserIds => [.. _pairManager.GetOnlineUserPairs() .Where(u => u.IsVisible && u.PlayerCharacterId != uint.MaxValue) .Select(u => (ulong)u.PlayerCharacterId)]; + public void FlagRefresh() { _needsLabelRefresh = true; @@ -591,18 +675,12 @@ public unsafe class NameplateHandler : IMediatorSubscriber public void UpdateBroadcastingCids(IEnumerable cids) { - var newSet = cids.ToHashSet(); - - var changed = !_activeBroadcastingCids.SetEquals(newSet); - if (!changed) + var newSet = cids.ToImmutableHashSet(StringComparer.Ordinal); + if (ReferenceEquals(_activeBroadcastingCids, newSet) || _activeBroadcastingCids.SetEquals(newSet)) return; - _activeBroadcastingCids.Clear(); - foreach (var cid in newSet) - _activeBroadcastingCids.Add(cid); - - _logger.LogInformation("Active broadcast CIDs: {Cids}", string.Join(",", _activeBroadcastingCids)); - + _activeBroadcastingCids = newSet; + _logger.LogInformation("Active broadcast CIDs: {Cids}", string.Join(',', _activeBroadcastingCids)); FlagRefresh(); } diff --git a/LightlessSync/Services/PairRequestService.cs b/LightlessSync/Services/PairRequestService.cs index 7190825..2531a3a 100644 --- a/LightlessSync/Services/PairRequestService.cs +++ b/LightlessSync/Services/PairRequestService.cs @@ -1,7 +1,3 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; using LightlessSync.LightlessConfiguration.Models; using LightlessSync.PlayerData.Pairs; using LightlessSync.Services.Mediator; @@ -14,10 +10,10 @@ public sealed class PairRequestService : DisposableMediatorSubscriberBase private readonly DalamudUtilService _dalamudUtil; private readonly PairManager _pairManager; private readonly Lazy _apiController; - private readonly object _syncRoot = new(); + private readonly Lock _syncRoot = new(); private readonly List _requests = []; - private static readonly TimeSpan Expiration = TimeSpan.FromMinutes(5); + private static readonly TimeSpan _expiration = TimeSpan.FromMinutes(5); public PairRequestService( ILogger logger, @@ -189,7 +185,7 @@ public sealed class PairRequestService : DisposableMediatorSubscriberBase } var now = DateTime.UtcNow; - return _requests.RemoveAll(r => now - r.ReceivedAt > Expiration) > 0; + return _requests.RemoveAll(r => now - r.ReceivedAt > _expiration) > 0; } public void AcceptPairRequest(string hashedCid, string displayName) diff --git a/LightlessSync/UI/CharaDataHubUi.cs b/LightlessSync/UI/CharaDataHubUi.cs index 9016e6c..51723b9 100644 --- a/LightlessSync/UI/CharaDataHubUi.cs +++ b/LightlessSync/UI/CharaDataHubUi.cs @@ -170,7 +170,7 @@ internal sealed partial class CharaDataHubUi : WindowMediatorSubscriberBase if (!_charaDataManager.BrioAvailable) { ImGuiHelpers.ScaledDummy(3); - UiSharedService.DrawGroupedCenteredColorText("To utilize any features related to posing or spawning characters you require to have Brio installed.", ImGuiColors.DalamudRed); + UiSharedService.DrawGroupedCenteredColorText("To utilize any features related to posing or spawning characters, you are required to have Brio installed.", ImGuiColors.DalamudRed); UiSharedService.DistanceSeparator(); } diff --git a/LightlessSync/UI/CompactUI.cs b/LightlessSync/UI/CompactUI.cs index 0700de3..cc8d326 100644 --- a/LightlessSync/UI/CompactUI.cs +++ b/LightlessSync/UI/CompactUI.cs @@ -1,4 +1,3 @@ -using System; using Dalamud.Bindings.ImGui; using Dalamud.Interface; using Dalamud.Interface.Utility; @@ -16,12 +15,14 @@ using LightlessSync.Services.Mediator; using LightlessSync.Services.ServerConfiguration; using LightlessSync.UI.Components; using LightlessSync.UI.Handlers; +using LightlessSync.UI.Models; using LightlessSync.Utils; using LightlessSync.WebAPI; using LightlessSync.WebAPI.Files; using LightlessSync.WebAPI.Files.Models; using LightlessSync.WebAPI.SignalR.Utils; using Microsoft.Extensions.Logging; +using System; using System.Collections.Concurrent; using System.Collections.Immutable; using System.Globalization; @@ -708,23 +709,23 @@ public class CompactUi : WindowMediatorSubscriberBase } //Filter of not foldered syncshells - var groupFolders = new List(); + var groupFolders = new List(); foreach (var group in _pairManager.GroupPairs.Select(g => g.Key).OrderBy(g => g.GroupAliasOrGID, StringComparer.OrdinalIgnoreCase)) { GetGroups(allPairs, filteredPairs, group, out ImmutableList allGroupPairs, out Dictionary> filteredGroupPairs); if (FilterNotTaggedSyncshells(group)) { - groupFolders.Add(_drawEntityFactory.CreateDrawGroupFolder(group, filteredGroupPairs, allGroupPairs)); + groupFolders.Add(new GroupFolder(group, _drawEntityFactory.CreateDrawGroupFolder(group, filteredGroupPairs, allGroupPairs))); } } //Filter of grouped up syncshells (All Syncshells Folder) if (_configService.Current.GroupUpSyncshells) - drawFolders.Add(new DrawGroupedGroupFolder(groupFolders, _tagHandler, _uiSharedService, + drawFolders.Add(new DrawGroupedGroupFolder(groupFolders, _tagHandler, _apiController, _uiSharedService, _selectSyncshellForTagUi, _renameSyncshellTagUi, "")); else - drawFolders.AddRange(groupFolders); + drawFolders.AddRange(groupFolders.Select(v => v.GroupDrawFolder)); //Filter of grouped/foldered pairs foreach (var tag in _tagHandler.GetAllPairTagsSorted()) @@ -738,7 +739,7 @@ public class CompactUi : WindowMediatorSubscriberBase //Filter of grouped/foldered syncshells foreach (var syncshellTag in _tagHandler.GetAllSyncshellTagsSorted()) { - var syncshellFolderTags = new List(); + var syncshellFolderTags = new List(); foreach (var group in _pairManager.GroupPairs.Select(g => g.Key).OrderBy(g => g.GroupAliasOrGID, StringComparer.OrdinalIgnoreCase)) { if (_tagHandler.HasSyncshellTag(group.GID, syncshellTag)) @@ -747,11 +748,11 @@ public class CompactUi : WindowMediatorSubscriberBase out ImmutableList allGroupPairs, out Dictionary> filteredGroupPairs); - syncshellFolderTags.Add(_drawEntityFactory.CreateDrawGroupFolder($"tag_{group.GID}", group, filteredGroupPairs, allGroupPairs)); + syncshellFolderTags.Add(new GroupFolder(group, _drawEntityFactory.CreateDrawGroupFolder($"tag_{group.GID}", group, filteredGroupPairs, allGroupPairs))); } } - drawFolders.Add(new DrawGroupedGroupFolder(syncshellFolderTags, _tagHandler, _uiSharedService, _selectSyncshellForTagUi, _renameSyncshellTagUi, syncshellTag)); + drawFolders.Add(new DrawGroupedGroupFolder(syncshellFolderTags, _tagHandler, _apiController, _uiSharedService, _selectSyncshellForTagUi, _renameSyncshellTagUi, syncshellTag)); } //Filter of not grouped/foldered and offline pairs diff --git a/LightlessSync/UI/Components/DrawGroupedGroupFolder.cs b/LightlessSync/UI/Components/DrawGroupedGroupFolder.cs index 2aa3d5c..1bb3d79 100644 --- a/LightlessSync/UI/Components/DrawGroupedGroupFolder.cs +++ b/LightlessSync/UI/Components/DrawGroupedGroupFolder.cs @@ -1,7 +1,11 @@ using Dalamud.Bindings.ImGui; using Dalamud.Interface; using Dalamud.Interface.Utility.Raii; +using LightlessSync.API.Data.Extensions; +using LightlessSync.API.Dto.Group; using LightlessSync.UI.Handlers; +using LightlessSync.UI.Models; +using LightlessSync.WebAPI; using System.Collections.Immutable; using System.Numerics; @@ -10,19 +14,20 @@ namespace LightlessSync.UI.Components; public class DrawGroupedGroupFolder : IDrawFolder { private readonly string _tag; - private readonly IEnumerable _groups; + private readonly IEnumerable _groups; private readonly TagHandler _tagHandler; private readonly UiSharedService _uiSharedService; + private readonly ApiController _apiController; private readonly SelectSyncshellForTagUi _selectSyncshellForTagUi; private readonly RenameSyncshellTagUi _renameSyncshellTagUi; private bool _wasHovered = false; private float _menuWidth; - public IImmutableList DrawPairs => throw new NotSupportedException(); - public int OnlinePairs => _groups.SelectMany(g => g.DrawPairs).Where(g => g.Pair.IsOnline).DistinctBy(g => g.Pair.UserData.UID).Count(); - public int TotalPairs => _groups.Sum(g => g.TotalPairs); + public IImmutableList DrawPairs => _groups.SelectMany(g => g.GroupDrawFolder.DrawPairs).ToImmutableList(); + public int OnlinePairs => _groups.SelectMany(g => g.GroupDrawFolder.DrawPairs).Where(g => g.Pair.IsOnline).DistinctBy(g => g.Pair.UserData.UID).Count(); + public int TotalPairs => _groups.Sum(g => g.GroupDrawFolder.TotalPairs); - public DrawGroupedGroupFolder(IEnumerable groups, TagHandler tagHandler, UiSharedService uiSharedService, SelectSyncshellForTagUi selectSyncshellForTagUi, RenameSyncshellTagUi renameSyncshellTagUi, string tag) + public DrawGroupedGroupFolder(IEnumerable groups, TagHandler tagHandler, ApiController apiController, UiSharedService uiSharedService, SelectSyncshellForTagUi selectSyncshellForTagUi, RenameSyncshellTagUi renameSyncshellTagUi, string tag) { _groups = groups; _tagHandler = tagHandler; @@ -30,6 +35,7 @@ public class DrawGroupedGroupFolder : IDrawFolder _selectSyncshellForTagUi = selectSyncshellForTagUi; _renameSyncshellTagUi = renameSyncshellTagUi; _tag = tag; + _apiController = apiController; } public void Draw() @@ -42,7 +48,7 @@ public class DrawGroupedGroupFolder : IDrawFolder using var id = ImRaii.PushId(_id); var color = ImRaii.PushColor(ImGuiCol.ChildBg, ImGui.GetColorU32(ImGuiCol.FrameBgHovered), _wasHovered); - using (ImRaii.Child("folder__" + _id, new System.Numerics.Vector2(UiSharedService.GetWindowContentRegionWidth() - ImGui.GetCursorPosX(), ImGui.GetFrameHeight()))) + using (ImRaii.Child("folder__" + _id, new Vector2(UiSharedService.GetWindowContentRegionWidth() - ImGui.GetCursorPosX(), ImGui.GetFrameHeight()))) { ImGui.Dummy(new Vector2(0f, ImGui.GetFrameHeight())); using (ImRaii.PushStyle(ImGuiStyleVar.ItemSpacing, new Vector2(0f, 0f))) @@ -83,11 +89,16 @@ public class DrawGroupedGroupFolder : IDrawFolder { ImGui.TextUnformatted(_tag); + ImGui.SameLine(); + DrawPauseButton(); ImGui.SameLine(); DrawMenu(); } else { ImGui.TextUnformatted("All Syncshells"); + + ImGui.SameLine(); + DrawPauseButton(); } } color.Dispose(); @@ -100,11 +111,49 @@ public class DrawGroupedGroupFolder : IDrawFolder using var indent = ImRaii.PushIndent(20f); foreach (var entry in _groups) { - entry.Draw(); + entry.GroupDrawFolder.Draw(); } } } + protected void DrawPauseButton() + { + if (DrawPairs.Count > 0) + { + var isPaused = _groups.Select(g => g.GroupFullInfo).All(g => g.GroupUserPermissions.IsPaused()); + FontAwesomeIcon pauseIcon = isPaused ? FontAwesomeIcon.Play : FontAwesomeIcon.Pause; + + var pauseButtonSize = _uiSharedService.GetIconButtonSize(pauseIcon); + var windowEndX = ImGui.GetWindowContentRegionMin().X + UiSharedService.GetWindowContentRegionWidth(); + if (_tag != "") + { + var spacingX = ImGui.GetStyle().ItemSpacing.X; + var menuButtonSize = _uiSharedService.GetIconButtonSize(FontAwesomeIcon.EllipsisV); + ImGui.SameLine(windowEndX - pauseButtonSize.X - menuButtonSize.X - spacingX); + } + else + { + ImGui.SameLine(windowEndX - pauseButtonSize.X); + } + + + if (_uiSharedService.IconButton(pauseIcon)) + { + ChangePauseStateGroups(); + } + } + } + + protected void ChangePauseStateGroups() + { + foreach(var group in _groups) + { + var perm = group.GroupFullInfo.GroupUserPermissions; + perm.SetPaused(!perm.IsPaused()); + _ = _apiController.GroupChangeIndividualPermissionState(new GroupPairUserPermissionDto(group.GroupFullInfo.Group, new(_apiController.UID), perm)); + } + } + protected void DrawMenu() { var barButtonSize = _uiSharedService.GetIconButtonSize(FontAwesomeIcon.EllipsisV); diff --git a/LightlessSync/UI/Components/IDrawFolder.cs b/LightlessSync/UI/Components/IDrawFolder.cs index eda1fce..faf8a69 100644 --- a/LightlessSync/UI/Components/IDrawFolder.cs +++ b/LightlessSync/UI/Components/IDrawFolder.cs @@ -1,5 +1,4 @@ - -using System.Collections.Immutable; +using System.Collections.Immutable; namespace LightlessSync.UI.Components; diff --git a/LightlessSync/UI/DtrEntry.cs b/LightlessSync/UI/DtrEntry.cs index 89c7389..17bc871 100644 --- a/LightlessSync/UI/DtrEntry.cs +++ b/LightlessSync/UI/DtrEntry.cs @@ -2,19 +2,22 @@ using Dalamud.Game.Gui.Dtr; using Dalamud.Game.Text.SeStringHandling; using Dalamud.Game.Text.SeStringHandling.Payloads; using Dalamud.Plugin.Services; +using Dalamud.Utility; using LightlessSync.LightlessConfiguration; using LightlessSync.LightlessConfiguration.Configurations; using LightlessSync.PlayerData.Pairs; using LightlessSync.Services; using LightlessSync.Services.Mediator; using LightlessSync.Services.ServerConfiguration; +using LightlessSync.Utils; using LightlessSync.WebAPI; using LightlessSync.WebAPI.SignalR.Utils; -using LightlessSync.Utils; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; using System.Runtime.InteropServices; using System.Text; +using static LightlessSync.Services.PairRequestService; namespace LightlessSync.UI; @@ -106,7 +109,7 @@ public sealed class DtrEntry : IDisposable, IHostedService } catch (OperationCanceledException) { - + _logger.LogInformation("Lightfinder operation was canceled."); } finally { @@ -363,29 +366,46 @@ public sealed class DtrEntry : IDisposable, IHostedService } } - private int GetNearbyBroadcastCount() - { - var localHashedCid = GetLocalHashedCid(); - return _broadcastScannerService.CountActiveBroadcasts( - string.IsNullOrEmpty(localHashedCid) ? null : localHashedCid); - } - - private int GetPendingPairRequestCount() + private List GetNearbyBroadcasts() { try { - return _pairRequestService.GetActiveRequests().Count; + var localHashedCid = GetLocalHashedCid(); + return [.. _broadcastScannerService + .GetActiveBroadcasts(string.IsNullOrEmpty(localHashedCid) ? null : localHashedCid) + .Select(b => _dalamudUtilService.FindPlayerByNameHash(b.Key).Name)]; } catch (Exception ex) { var now = DateTime.UtcNow; + + if (now >= _pairRequestNextErrorLog) + { + _logger.LogDebug(ex, "Failed to retrieve nearby broadcasts for Lightfinder DTR entry."); + _pairRequestNextErrorLog = now + _localHashedCidErrorCooldown; + } + + return []; + } + } + + private IReadOnlyList GetPendingPairRequest() + { + try + { + return _pairRequestService.GetActiveRequests(); + } + catch (Exception ex) + { + var now = DateTime.UtcNow; + if (now >= _pairRequestNextErrorLog) { _logger.LogDebug(ex, "Failed to retrieve pair request count for Lightfinder DTR entry."); _pairRequestNextErrorLog = now + _localHashedCidErrorCooldown; } - return 0; + return []; } } @@ -400,23 +420,15 @@ public sealed class DtrEntry : IDisposable, IHostedService if (_broadcastService.IsBroadcasting) { - var tooltipBuilder = new StringBuilder("Lightfinder - Enabled"); - switch (config.LightfinderDtrDisplayMode) { case LightfinderDtrDisplayMode.PendingPairRequests: { - var requestCount = GetPendingPairRequestCount(); - tooltipBuilder.AppendLine(); - tooltipBuilder.Append("Pending pair requests: ").Append(requestCount); - return ($"{icon} Requests {requestCount}", SwapColorChannels(config.DtrColorsLightfinderEnabled), tooltipBuilder.ToString()); + return FormatTooltip("Pending pair requests", GetPendingPairRequest().Select(x => x.DisplayName), icon, SwapColorChannels(config.DtrColorsLightfinderEnabled)); } default: { - var broadcastCount = GetNearbyBroadcastCount(); - tooltipBuilder.AppendLine(); - tooltipBuilder.Append("Nearby Lightfinder users: ").Append(broadcastCount); - return ($"{icon} {broadcastCount}", SwapColorChannels(config.DtrColorsLightfinderEnabled), tooltipBuilder.ToString()); + return FormatTooltip("Nearby Lightfinder users", GetNearbyBroadcasts(), icon, SwapColorChannels(config.DtrColorsLightfinderEnabled)); } } } @@ -433,6 +445,18 @@ public sealed class DtrEntry : IDisposable, IHostedService return ($"{icon} OFF", colors, tooltip.ToString()); } + private (string, Colors, string) FormatTooltip(string title, IEnumerable names, string icon, Colors color) + { + var list = names.Where(x => !string.IsNullOrEmpty(x)).ToList(); + var tooltip = new StringBuilder() + .Append($"Lightfinder - Enabled{Environment.NewLine}") + .Append($"{title}: {list.Count}{Environment.NewLine}") + .AppendJoin(Environment.NewLine, list) + .ToString(); + + return ($"{icon} {list.Count}", color, tooltip); + } + private static string BuildLightfinderTooltip(string baseTooltip) { var builder = new StringBuilder(); diff --git a/LightlessSync/UI/Models/GroupFolder.cs b/LightlessSync/UI/Models/GroupFolder.cs new file mode 100644 index 0000000..cedeef0 --- /dev/null +++ b/LightlessSync/UI/Models/GroupFolder.cs @@ -0,0 +1,6 @@ +using LightlessSync.API.Dto.Group; +using LightlessSync.UI.Components; + +namespace LightlessSync.UI.Models; + +public record GroupFolder(GroupFullInfoDto GroupFullInfo, IDrawFolder GroupDrawFolder); diff --git a/LightlessSync/UI/SettingsUi.cs b/LightlessSync/UI/SettingsUi.cs index f1da3c0..c841fd7 100644 --- a/LightlessSync/UI/SettingsUi.cs +++ b/LightlessSync/UI/SettingsUi.cs @@ -1227,16 +1227,16 @@ public class SettingsUi : WindowMediatorSubscriberBase ImGui.TextUnformatted($"Currently utilized local storage: Calculating..."); ImGui.TextUnformatted( $"Remaining space free on drive: {UiSharedService.ByteToString(_cacheMonitor.FileCacheDriveFree)}"); + bool useFileCompactor = _configService.Current.UseCompactor; - bool isLinux = _dalamudUtilService.IsWine; - if (!useFileCompactor && !isLinux) + if (!useFileCompactor) { UiSharedService.ColorTextWrapped( "Hint: To free up space when using Lightless consider enabling the File Compactor", UIColors.Get("LightlessYellow")); } - if (isLinux || !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); + if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); if (ImGui.Checkbox("Use file compactor", ref useFileCompactor)) { _configService.Current.UseCompactor = useFileCompactor; @@ -1281,10 +1281,20 @@ public class SettingsUi : WindowMediatorSubscriberBase UIColors.Get("LightlessYellow")); } - if (isLinux || !_cacheMonitor.StorageisNTFS) + if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) { ImGui.EndDisabled(); - ImGui.TextUnformatted("The file compactor is only available on Windows and NTFS drives."); + ImGui.TextUnformatted("The file compactor is only available on BTRFS and NTFS drives."); + } + + if (_cacheMonitor.StorageisNTFS) + { + ImGui.TextUnformatted("The file compactor is running on NTFS Drive."); + } + + if (_cacheMonitor.StorageIsBtrfs) + { + ImGui.TextUnformatted("The file compactor is running on Btrfs Drive."); } ImGuiHelpers.ScaledDummy(new Vector2(10, 10)); @@ -3113,22 +3123,6 @@ public class SettingsUi : WindowMediatorSubscriberBase UiSharedService.TooltipSeparator + "Note: if the server does not support a specific Transport Type it will fall through to the next automatically: WebSockets > ServerSentEvents > LongPolling"); - if (_dalamudUtilService.IsWine) - { - bool forceWebSockets = selectedServer.ForceWebSockets; - if (ImGui.Checkbox("[wine only] Force WebSockets", ref forceWebSockets)) - { - selectedServer.ForceWebSockets = forceWebSockets; - _serverConfigurationManager.Save(); - } - - _uiShared.DrawHelpText( - "On wine, Lightless will automatically fall back to ServerSentEvents/LongPolling, even if WebSockets is selected. " - + "WebSockets are known to crash XIV entirely on wine 8.5 shipped with Dalamud. " - + "Only enable this if you are not running wine 8.5." + Environment.NewLine - + "Note: If the issue gets resolved at some point this option will be removed."); - } - ImGuiHelpers.ScaledDummy(5); if (ImGui.Checkbox("Use Discord OAuth2 Authentication", ref useOauth)) diff --git a/LightlessSync/Utils/Crypto.cs b/LightlessSync/Utils/Crypto.cs index de04d26..c31f82f 100644 --- a/LightlessSync/Utils/Crypto.cs +++ b/LightlessSync/Utils/Crypto.cs @@ -1,16 +1,15 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Security.Cryptography; +using System.Security.Cryptography; using System.Text; namespace LightlessSync.Utils; public static class Crypto { + //This buffersize seems to be the best sweetpoint for Linux and Windows + private const int _bufferSize = 65536; #pragma warning disable SYSLIB0021 // Type or member is obsolete - private static readonly Dictionary<(string, ushort), string> _hashListPlayersSHA256 = new(); + private static readonly Dictionary<(string, ushort), string> _hashListPlayersSHA256 = []; private static readonly Dictionary _hashListSHA256 = new(StringComparer.Ordinal); private static readonly SHA256CryptoServiceProvider _sha256CryptoProvider = new(); @@ -21,6 +20,26 @@ public static class Crypto return BitConverter.ToString(sha1.ComputeHash(stream)).Replace("-", "", StringComparison.Ordinal); } + public static async Task GetFileHashAsync(string filePath, CancellationToken cancellationToken = default) + { + var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, bufferSize: _bufferSize, options: FileOptions.Asynchronous); + await using (stream.ConfigureAwait(false)) + { + using var sha1 = SHA1.Create(); + + var buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0) + { + sha1.TransformBlock(buffer, 0, bytesRead, outputBuffer: null, 0); + } + + sha1.TransformFinalBlock([], 0, 0); + + return Convert.ToHexString(sha1.Hash!); + } + } + public static string GetHash256(this (string, ushort) playerToHash) { if (_hashListPlayersSHA256.TryGetValue(playerToHash, out var hash)) diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs new file mode 100644 index 0000000..af4c98b --- /dev/null +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -0,0 +1,282 @@ +using Microsoft.Extensions.Logging; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace LightlessSync.Utils +{ + public static class FileSystemHelper + { + public enum FilesystemType + { + Unknown = 0, + NTFS, // Compressable on file level + Btrfs, // Compressable on file level + Ext4, // Uncompressable + Xfs, // Uncompressable + Apfs, // Compressable on OS + HfsPlus, // Compressable on OS + Fat, // Uncompressable + Exfat, // Uncompressable + Zfs // Compressable, not on file level + } + + private const string _mountPath = "/proc/mounts"; + private const int _defaultBlockSize = 4096; + private static readonly Dictionary _blockSizeCache = new(StringComparer.OrdinalIgnoreCase); + private static readonly ConcurrentDictionary _filesystemTypeCache = new(StringComparer.OrdinalIgnoreCase); + + public static FilesystemType GetFilesystemType(string filePath, bool isWine = false) + { + try + { + string rootPath; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (!IsProbablyWine() || !isWine)) + { + var info = new FileInfo(filePath); + var dir = info.Directory ?? new DirectoryInfo(filePath); + rootPath = dir.Root.FullName; + } + else + { + rootPath = GetMountPoint(filePath); + if (string.IsNullOrEmpty(rootPath)) + rootPath = "/"; + } + + if (_filesystemTypeCache.TryGetValue(rootPath, out var cachedType)) + return cachedType; + + FilesystemType detected; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (!IsProbablyWine() || !isWine)) + { + var root = new DriveInfo(rootPath); + var format = root.DriveFormat?.ToUpperInvariant() ?? string.Empty; + + detected = format switch + { + "NTFS" => FilesystemType.NTFS, + "FAT32" => FilesystemType.Fat, + "EXFAT" => FilesystemType.Exfat, + _ => FilesystemType.Unknown + }; + } + else + { + detected = GetLinuxFilesystemType(filePath); + } + + if (isWine || IsProbablyWine()) + { + switch (detected) + { + case FilesystemType.NTFS: + case FilesystemType.Unknown: + { + var linuxDetected = GetLinuxFilesystemType(filePath); + if (linuxDetected != FilesystemType.Unknown) + { + detected = linuxDetected; + } + + break; + } + } + } + + _filesystemTypeCache[rootPath] = detected; + return detected; + } + catch + { + return FilesystemType.Unknown; + } + } + + private static string GetMountPoint(string filePath) + { + try + { + var path = Path.GetFullPath(filePath); + if (!File.Exists(_mountPath)) return "/"; + var mounts = File.ReadAllLines(_mountPath); + + string bestMount = "/"; + foreach (var line in mounts) + { + var parts = line.Split(' '); + if (parts.Length < 3) continue; + var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); + + string normalizedMount; + try { normalizedMount = Path.GetFullPath(mountPoint); } + catch { normalizedMount = mountPoint; } + + if (path.StartsWith(normalizedMount, StringComparison.Ordinal) && + normalizedMount.Length > bestMount.Length) + { + bestMount = normalizedMount; + } + } + + return bestMount; + } + catch + { + return "/"; + } + } + + public static string GetMountOptionsForPath(string path) + { + try + { + var fullPath = Path.GetFullPath(path); + var mounts = File.ReadAllLines("/proc/mounts"); + string bestMount = string.Empty; + string mountOptions = string.Empty; + + foreach (var line in mounts) + { + var parts = line.Split(' '); + if (parts.Length < 4) continue; + var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); + string normalized; + try { normalized = Path.GetFullPath(mountPoint); } + catch { normalized = mountPoint; } + + if (fullPath.StartsWith(normalized, StringComparison.Ordinal) && + normalized.Length > bestMount.Length) + { + bestMount = normalized; + mountOptions = parts[3]; + } + } + + return mountOptions; + } + catch (Exception ex) + { + return string.Empty; + } + } + + private static FilesystemType GetLinuxFilesystemType(string filePath) + { + try + { + var mountPoint = GetMountPoint(filePath); + var mounts = File.ReadAllLines(_mountPath); + + foreach (var line in mounts) + { + var parts = line.Split(' '); + if (parts.Length < 3) continue; + var mount = parts[1].Replace("\\040", " ", StringComparison.Ordinal); + if (string.Equals(mount, mountPoint, StringComparison.Ordinal)) + { + var fstype = parts[2].ToLowerInvariant(); + return fstype switch + { + "btrfs" => FilesystemType.Btrfs, + "ext4" => FilesystemType.Ext4, + "xfs" => FilesystemType.Xfs, + "zfs" => FilesystemType.Zfs, + "apfs" => FilesystemType.Apfs, + "hfsplus" => FilesystemType.HfsPlus, + _ => FilesystemType.Unknown + }; + } + } + + return FilesystemType.Unknown; + } + catch + { + return FilesystemType.Unknown; + } + } + + public static int GetBlockSizeForPath(string path, ILogger? logger = null, bool isWine = false) + { + try + { + if (string.IsNullOrWhiteSpace(path)) + return _defaultBlockSize; + + var fi = new FileInfo(path); + if (!fi.Exists) + return _defaultBlockSize; + + var root = fi.Directory?.Root.FullName.ToLowerInvariant() ?? "/"; + if (_blockSizeCache.TryGetValue(root, out int cached)) + return cached; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !isWine) + { + int result = GetDiskFreeSpaceW(root, + out uint sectorsPerCluster, + out uint bytesPerSector, + out _, + out _); + + if (result == 0) + { + logger?.LogWarning("Failed to determine block size for {root}", root); + return _defaultBlockSize; + } + + int clusterSize = (int)(sectorsPerCluster * bytesPerSector); + _blockSizeCache[root] = clusterSize; + logger?.LogTrace("NTFS cluster size for {root}: {cluster}", root, clusterSize); + return clusterSize; + } + + string realPath = fi.FullName; + if (isWine && realPath.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + { + realPath = "/" + realPath.Substring(3).Replace('\\', '/'); + } + + var psi = new ProcessStartInfo + { + FileName = "/bin/bash", + Arguments = $"-c \"stat -f -c %s '{realPath.Replace("'", "'\\''")}'\"", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = "/" + }; + + using var proc = Process.Start(psi); + string stdout = proc?.StandardOutput.ReadToEnd().Trim() ?? ""; + proc?.WaitForExit(); + + if (int.TryParse(stdout, out int blockSize) && blockSize > 0) + { + _blockSizeCache[root] = blockSize; + logger?.LogTrace("Filesystem block size via stat for {root}: {block}", root, blockSize); + return blockSize; + } + + logger?.LogTrace("stat did not return valid block size for {file}, output: {out}", fi.FullName, stdout); + _blockSizeCache[root] = _defaultBlockSize; + return _defaultBlockSize; + } + catch (Exception ex) + { + logger?.LogTrace(ex, "Error determining block size for {path}", path); + return _defaultBlockSize; + } + } + + [DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)] + private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters, out uint lpTotalNumberOfClusters); + + //Extra check on + public static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); + } +} diff --git a/LightlessSync/WebAPI/SignalR/HubFactory.cs b/LightlessSync/WebAPI/SignalR/HubFactory.cs index ef9f919..1d5a0c8 100644 --- a/LightlessSync/WebAPI/SignalR/HubFactory.cs +++ b/LightlessSync/WebAPI/SignalR/HubFactory.cs @@ -70,13 +70,6 @@ public class HubFactory : MediatorSubscriberBase _ => HttpTransportType.WebSockets | HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling }; - if (_isWine && !_serverConfigurationManager.CurrentServer.ForceWebSockets - && transportType.HasFlag(HttpTransportType.WebSockets)) - { - Logger.LogDebug("Wine detected, falling back to ServerSentEvents / LongPolling"); - transportType = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; - } - Logger.LogDebug("Building new HubConnection using transport {transport}", transportType); _instance = new HubConnectionBuilder()