LightlessClient/LightlessCompactor/FileCache/FileCompactor.cs

using LightlessSync.Services.Compactor;
using Microsoft.Extensions.Logging;
using Microsoft.Win32.SafeHandles;
using System.Collections.Concurrent;
using System.ComponentModel;
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Threading.Channels;
using static LightlessSync.Utils.FileSystemHelper;

namespace LightlessSync.FileCache;

public sealed partial class FileCompactor : IDisposable
{
    public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U;
    public const ulong WOF_PROVIDER_FILE = 2UL;
    public const int _maxRetries = 3;

    private readonly ConcurrentDictionary<string, byte> _pendingCompactions;
    private readonly ILogger<FileCompactor> _logger;
    private readonly ICompactorContext _context;
    private readonly ICompactionExecutor _compactionExecutor;

    private readonly Channel<string> _compactionQueue;
    private readonly CancellationTokenSource _compactionCts = new();

    private readonly List<Task> _workers = [];
    private readonly SemaphoreSlim _globalGate;

    //Limit btrfs gate on half of threads given to compactor.
    private readonly SemaphoreSlim _btrfsGate;
    private readonly BatchFilefragService _fragBatch;

    private readonly bool _isWindows;
    private readonly int _workerCount;

    private readonly WofFileCompressionInfoV1 _efInfo = new()
    {
        Algorithm = (int)CompressionAlgorithm.XPRESS8K,
        Flags = 0
    };

    [StructLayout(LayoutKind.Sequential, Pack = 1)]
    private struct WofFileCompressionInfoV1
    {
        public int Algorithm;
        public ulong Flags;
    }

    private enum CompressionAlgorithm
    {
        NO_COMPRESSION = -2,
        LZNT1 = -1,
        XPRESS4K = 0,
        LZX = 1,
        XPRESS8K = 2,
        XPRESS16K = 3
    }

    public FileCompactor(ILogger<FileCompactor> logger, ICompactorContext context, ICompactionExecutor compactionExecutor)
    {
        _pendingCompactions = new(StringComparer.OrdinalIgnoreCase);
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
        _context = context ?? throw new ArgumentNullException(nameof(context));
        _compactionExecutor = compactionExecutor ?? throw new ArgumentNullException(nameof(compactionExecutor));
        _isWindows = OperatingSystem.IsWindows();

        _compactionQueue = Channel.CreateUnbounded<string>(new UnboundedChannelOptions
        {
            SingleReader = false,
            SingleWriter = false
        });

        //Amount of threads given for the compactor
        int workers = Math.Clamp(Math.Min(Environment.ProcessorCount / 2, 4), 1, 8);
        //Setup gates for the threads and setup worker count
        _globalGate = new SemaphoreSlim(workers, workers);
        _btrfsGate = new SemaphoreSlim(workers / 2, workers / 2);
        _workerCount = Math.Max(workers * 2, workers);

        //Setup workers on the queue
        for (int i = 0; i < _workerCount; i++)
        {
            int workerId = i;

            _workers.Add(Task.Factory.StartNew(
                () => ProcessQueueWorkerAsync(workerId, _compactionCts.Token),
                _compactionCts.Token,
                TaskCreationOptions.LongRunning,
                TaskScheduler.Default).Unwrap());
        }

        //Uses an batching service for the filefrag command on Linux
        _fragBatch = new BatchFilefragService(
            useShell: _context.IsWine,
            log: _logger,
            batchSize: 64,
            flushMs: 25,
            runDirect: RunProcessDirect,
            runShell: RunProcessShell
         );

        _logger.LogInformation("FileCompactor started with {workers} workers", _workerCount);
    }

    public bool MassCompactRunning { get; private set; }
    public string Progress { get; private set; } = string.Empty;

    /// <summary>
    /// Compact the storage of the Cache Folder
    /// </summary>
    /// <param name="compress">Used to check if files needs to be compressed</param>
    public void CompactStorage(bool compress, int? maxDegree = null)
    {
        MassCompactRunning = true;

        try
        {
            var folder = _context.CacheFolder;
            if (string.IsNullOrWhiteSpace(folder) || !Directory.Exists(folder))
            {
                if (_logger.IsEnabled(LogLevel.Warning))
                    _logger.LogWarning("Filecompacator couldnt find your Cache folder: {folder}", folder);
                Progress = "0/0";
                return;
            }

            var files = Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories).ToArray();
            var total = files.Length;
            Progress = $"0/{total}";
            if (total == 0) return;

            var degree = maxDegree ?? Math.Clamp(Environment.ProcessorCount / 2, 1, 8);

            var done = 0;
            int workerCounter = -1;
            var po = new ParallelOptions
            {
                MaxDegreeOfParallelism = degree,
                CancellationToken = _compactionCts.Token
            };

            Parallel.ForEach(files, po, localInit: () => Interlocked.Increment(ref workerCounter), body: (file, state, workerId) =>
            {
                _globalGate.WaitAsync(po.CancellationToken).GetAwaiter().GetResult();

                if (!_pendingCompactions.TryAdd(file, 0))
                    return -1;

                try
                {
                    try
                    {
                        if (compress)
                        {
                            if (_context.UseCompactor)
                                CompactFile(file, workerId);
                        }
                        else
                        {
                            DecompressFile(file, workerId);
                        }
                    }
                    catch (IOException ioEx)
                    {
                        _logger.LogDebug(ioEx, "[W{worker}] File being read/written, skipping file: {file}", workerId, file);
                    }
                    catch (OperationCanceledException)
                    {
                        throw;
                    }
                    catch (Exception ex)
                    {
                        _logger.LogWarning(ex, "[W{worker}] Error processing file: {file}", workerId, file);
                    }
                    finally
                    {
                        var n = Interlocked.Increment(ref done);
                        Progress = $"{n}/{total}";
                    }
                }
                finally
                {
                    _pendingCompactions.TryRemove(file, out _);
                    _globalGate.Release();
                }

                return workerId;
            },
            localFinally: _ =>
            {
                //Ignore local finally for now
            });
        }
        catch (OperationCanceledException ex)
        {
            _logger.LogDebug(ex, "Mass compaction call got cancelled, shutting off compactor.");
        }
        finally
        {
            MassCompactRunning = false;
            Progress = string.Empty;
        }
    }


    /// <summary>
    /// Write all bytes into a directory async
    /// </summary>
    /// <param name="filePath">Bytes will be writen to this filepath</param>
    /// <param name="bytes">Bytes that have to be written</param>
    /// <param name="token">Cancellation Token for interupts</param>
    /// <returns>Writing Task</returns>
    public async Task WriteAllBytesAsync(string filePath, byte[] bytes, CancellationToken token)
    {
        var dir = Path.GetDirectoryName(filePath);
        if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir))
            Directory.CreateDirectory(dir);

        await File.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false);

        if (_context.UseCompactor)
            EnqueueCompaction(filePath);
    }

    /// <summary>
    /// Notify the compactor that a file was written directly (streamed) so it can enqueue compaction.
    /// </summary>
    public void NotifyFileWritten(string filePath)
    {
        EnqueueCompaction(filePath);
    }

    public bool TryCompactFile(string filePath)
    {
        if (string.IsNullOrWhiteSpace(filePath))
            return false;

        if (!_context.UseCompactor || !File.Exists(filePath))
            return false;

        try
        {
            CompactFile(filePath, workerId: -1);
            return true;
        }
        catch (IOException ioEx)
        {
            _logger.LogDebug(ioEx, "File being read/written, skipping file: {file}", filePath);
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex, "Error compacting file: {file}", filePath);
        }

        return false;
    }

    /// <summary>
    /// Gets the File size for an BTRFS or NTFS file system for the given FileInfo
    /// </summary>
    /// <param name="path">Amount of blocks used in the disk</param>
    public long GetFileSizeOnDisk(FileInfo fileInfo)
    {
        var fsType = GetFilesystemType(fileInfo.FullName, _context.IsWine);

        if (fsType == FilesystemType.NTFS && !_context.IsWine)
        {
            (bool flowControl, long value) = GetFileSizeNTFS(fileInfo);
            if (!flowControl)
            {
                return value;
            }
        }

        if (fsType == FilesystemType.Btrfs)
        {
            (bool flowControl, long value) = GetFileSizeBtrfs(fileInfo);
            if (!flowControl)
            {
                return value;
            }
        }

        return fileInfo.Length;
    }

    /// <summary>
    /// Get File Size in an Btrfs file system (Linux/Wine).
    /// </summary>
    /// <param name="fileInfo">File that you want the size from.</param>
    /// <returns>Succesful check and value of the filesize.</returns>
    /// <exception cref="InvalidOperationException">Fails on the Process in StartProcessInfo</exception>
    private (bool flowControl, long value) GetFileSizeBtrfs(FileInfo fileInfo)
    {
        try
        {
            var (_, linuxPath) = ResolvePathsForBtrfs(fileInfo.FullName);

            var (ok, output, err, code) =
                _isWindows
                    ? RunProcessShell($"stat -c='%b' {QuoteSingle(linuxPath)}", workingDir: null, 10000)
                    : RunProcessDirect("stat", ["-c='%b'", linuxPath], workingDir: null, 10000);

            return (flowControl: false, value: fileInfo.Length);
        }
        catch (Exception ex)
        {
            if (_logger.IsEnabled(LogLevel.Debug))
                _logger.LogDebug(ex, "Failed Btrfs size probe for {file}, using Length", fileInfo.FullName);
            return (flowControl: true, value: fileInfo.Length);
        }
    }

    /// <summary>
    /// Get File Size in an NTFS file system (Windows).
    /// </summary>
    /// <param name="fileInfo">File that you want the size from.</param>
    /// <returns>Succesful check and value of the filesize.</returns>
    private (bool flowControl, long value) GetFileSizeNTFS(FileInfo fileInfo)
    {
        try
        {
            var blockSize = GetBlockSizeForPath(fileInfo.FullName, _logger, _context.IsWine);
            if (blockSize <= 0)
                throw new InvalidOperationException($"Invalid block size {blockSize} for {fileInfo.FullName}");

            uint lo = GetCompressedFileSizeW(fileInfo.FullName, out uint hi);

            if (lo == 0xFFFFFFFF)
            {
                int err = Marshal.GetLastWin32Error();
                if (err != 0)
                    throw new Win32Exception(err);
            }

            long size = ((long)hi << 32) | lo;
            long rounded = ((size + blockSize - 1) / blockSize) * blockSize;

            return (flowControl: false, value: rounded);
        }
        catch (Exception ex)
        {
            _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName);
            return (flowControl: true, value: default);
        }
    }

    /// <summary>
    /// Compressing the given path with BTRFS or NTFS file system.
    /// </summary>
    /// <param name="filePath">Path of the decompressed/normal file</param>
    /// <param name="workerId">Worker/Process Id</param>
    private void CompactFile(string filePath, int workerId)
    {
        var fi = new FileInfo(filePath);
        if (!fi.Exists)
        {
            if (_logger.IsEnabled(LogLevel.Trace))
                _logger.LogTrace("[W{worker}] Skip compaction: missing {file}", workerId, filePath);
            return;
        }

        var fsType = GetFilesystemType(filePath, _context.IsWine);
        var oldSize = fi.Length;
        int blockSize = (int)(GetFileSizeOnDisk(fi) / 512);

        // We skipping small files (128KiB) as they slow down the system a lot for BTRFS. as BTRFS has a different blocksize it requires an different calculation.
        long minSizeBytes = fsType == FilesystemType.Btrfs
            ? Math.Max(blockSize * 2L, 128 * 1024L)
            : Math.Max(blockSize, 8 * 1024L);

        if (oldSize < minSizeBytes)
        {
            if (_logger.IsEnabled(LogLevel.Trace))
                _logger.LogTrace("[W{worker}] Skip compaction: {file} ({size} B) < threshold ({th} B)", workerId, filePath, oldSize, minSizeBytes);
            return;
        }

        if (fsType == FilesystemType.NTFS && !_context.IsWine)
        {
            if (!IsWOFCompactedFile(filePath))
            {
                if (WOFCompressFile(filePath))
                {
                    var newSize = GetFileSizeOnDisk(fi);
                    _logger.LogDebug("[W{worker}] NTFS compressed XPRESS8K {file} {old} -> {new}", workerId, filePath, oldSize, newSize);
                }
                else
                {
                    _logger.LogWarning("[W{worker}] NTFS compression failed or unavailable for {file}", workerId, filePath);
                }
            }
            else
            {
                if (_logger.IsEnabled(LogLevel.Trace))
                    _logger.LogTrace("[W{worker}] Already NTFS-compressed with XPRESS8K: {file}", workerId, filePath);
            }
            return;
        }

        if (fsType == FilesystemType.Btrfs)
        {
            if (!IsBtrfsCompressedFile(filePath))
            {
                if (BtrfsCompressFile(filePath))
                {
                    var newSize = GetFileSizeOnDisk(fi);
                    _logger.LogDebug("[W{worker}] Btrfs compressed clzo {file} {old} -> {new}", workerId, filePath, oldSize, newSize);
                }
                else
                {
                    _logger.LogWarning("[W{worker}] Btrfs compression failed or unavailable for {file}", workerId, filePath);
                }
            }
            else
            {
                if (_logger.IsEnabled(LogLevel.Trace))
                    _logger.LogTrace("[W{worker}] Already Btrfs-compressed with clzo: {file}", workerId, filePath);
            }
            return;
        }

        if (_logger.IsEnabled(LogLevel.Trace))
            _logger.LogTrace("[W{worker}] Skip compact: unsupported FS for {file}", workerId, filePath);
    }

    /// <summary>
    /// Decompressing the given path with BTRFS file system or NTFS file system.
    /// </summary>
    /// <param name="filePath">Path of the decompressed/normal file</param>
    /// <param name="workerId">Worker/Process Id</param>
    private void DecompressFile(string filePath, int workerId)
    {
        _logger.LogDebug("[W{worker}] Decompress request: {file}", workerId, filePath);
        var fsType = GetFilesystemType(filePath, _context.IsWine);

        if (fsType == FilesystemType.NTFS && !_context.IsWine)
        {
            try
            {
                bool flowControl = DecompressWOFFile(filePath, workerId);
                if (!flowControl)
                {
                    return;
                }
            }
            catch (Exception ex)
            {
                _logger.LogWarning(ex, "[W{worker}] NTFS decompress error {file}", workerId, filePath);
            }
        }

        if (fsType == FilesystemType.Btrfs)
        {
            try
            {
                bool flowControl = DecompressBtrfsFile(filePath);
                if (!flowControl)
                {
                    return;
                }
            }
            catch (Exception ex)
            {
                _logger.LogWarning(ex, "[W{worker}] Btrfs decompress error {file}", workerId, filePath);
            }
        }
    }

    /// <summary>
    /// Decompress an BTRFS File on Wine/Linux
    /// </summary>
    /// <param name="path">Path of the compressed file</param>
    /// <returns>Decompressing state</returns>
    private bool DecompressBtrfsFile(string path)
    {
        return RunWithBtrfsGate(() =>
        {
            try
            {
                bool isWine = _context.IsWine;
                string linuxPath = isWine ? ToLinuxPathIfWine(path, isWine) : path;

                var opts = GetMountOptionsForPath(linuxPath);
                if (!string.IsNullOrEmpty(opts))
                    _logger.LogTrace("Mount opts for {file}: {opts}", linuxPath, opts);

                var probe = RunProcessShell("command -v btrfs || which btrfs", timeoutMs: 5000);
                var _btrfsAvailable = probe.ok && !string.IsNullOrWhiteSpace(probe.stdout);
                if (!_btrfsAvailable)
                    _logger.LogWarning("btrfs cli not found in path. Compression will be skipped.");

                var prop = isWine
                    ? RunProcessShell($"btrfs property set -- {QuoteSingle(linuxPath)} compression none", timeoutMs: 15000)
                    : RunProcessDirect("btrfs", ["property", "set", "--", linuxPath, "compression", "none"], "/", 15000);

                if (prop.ok) _logger.LogTrace("Set per-file 'compression none' on {file}", linuxPath);
                else _logger.LogTrace("btrfs property set failed for {file} (exit {code}): {err}", linuxPath, prop.exitCode, prop.stderr);

                var defrag = isWine
                    ? RunProcessShell($"btrfs filesystem defragment -f -- {QuoteSingle(linuxPath)}", timeoutMs: 60000)
                    : RunProcessDirect("btrfs", ["filesystem", "defragment", "-f", "--", linuxPath], "/", 60000);

                if (!defrag.ok)
                {
                    _logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit {code}): {err}",
                        linuxPath, defrag.exitCode, defrag.stderr);
                    return false;
                }

                if (!string.IsNullOrWhiteSpace(defrag.stdout))
                    _logger.LogTrace("btrfs defragment output for {file}: {out}", linuxPath, defrag.stdout.Trim());

                _logger.LogInformation("Decompressed (rewritten uncompressed) Btrfs file: {file}", linuxPath);

                try
                {
                    if (_fragBatch != null)
                    {
                        var compressed = _fragBatch.IsCompressedAsync(linuxPath, _compactionCts.Token).GetAwaiter().GetResult();
                        if (compressed)
                            _logger.LogTrace("Post-check: {file} still shows 'compressed' flag (may be stale).", linuxPath);
                    }
                }
                catch { /* ignore verification noisy */ }

                return true;
            }
            catch (Exception ex)
            {
                _logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path);
                return false;
            }
        });
    }

    /// <summary>
    /// Decompress an NTFS File
    /// </summary>
    /// <param name="path">Path of the compressed file</param>
    /// <returns>Decompressing state</returns>
    private bool DecompressWOFFile(string path, int workerID)
    {
        //Check if its already been compressed
        if (TryIsWofExternal(path, out bool isExternal, out int algo))
        {
            if (!isExternal)
            {
                _logger.LogTrace("[W{worker}] Already decompressed file: {file}", workerID, path);
                return true;
            }
            var compressString = ((CompressionAlgorithm)algo).ToString();
            _logger.LogTrace("[W{worker}] WOF compression (algo={algo}) detected for {file}", workerID, compressString, path);
        }

        //This will attempt to start WOF thread.
        return WithFileHandleForWOF(path, FileAccess.ReadWrite, h =>
        {
            if (!DeviceIoControl(h, FSCTL_DELETE_EXTERNAL_BACKING,
                                 IntPtr.Zero, 0, IntPtr.Zero, 0,
                                 out uint _, IntPtr.Zero))
            {
                int err = Marshal.GetLastWin32Error();
                // 342 error code means its been decompressed after the control, we handle it as it succesfully been decompressed.
                if (err == 342)
                {
                    _logger.LogTrace("[W{worker}] Successfully decompressed NTFS file {file}", workerID, path);
                    return true;
                }

                _logger.LogWarning("[W{worker}] DeviceIoControl failed for {file} with Win32 error {err}", workerID, path, err);
                return false;
            }

            _logger.LogTrace("[W{worker}] Successfully decompressed NTFS file {file}", workerID, path);
            return true;
        });
    }

    /// <summary>
    /// Converts to Linux Path if its using Wine (diferent pathing system in Wine)
    /// </summary>
    /// <param name="path">Path that has to be converted</param>
    /// <param name="isWine">Extra check if using the wine enviroment</param>
    /// <returns>Converted path to be used in Linux</returns>
    private string ToLinuxPathIfWine(string path, bool isWine, bool preferShell = true)
    {
        //Return if not wine
        if (!isWine || !IsProbablyWine())
            return path;

        if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase))
            return ("/" + path[3..].Replace('\\', '/')).Replace("//", "/", StringComparison.Ordinal);

        if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase))
        {
            const string usersPrefix = "C:\\Users\\";
            var p = path.Replace('/', '\\');

            if (p.StartsWith(usersPrefix, StringComparison.OrdinalIgnoreCase))
            {
                int afterUsers = usersPrefix.Length;
                int slash = p.IndexOf('\\', afterUsers);
                if (slash > 0 && slash + 1 < p.Length)
                {
                    var rel = p[(slash + 1)..].Replace('\\', '/');
                    var home = Environment.GetEnvironmentVariable("HOME");
                    if (string.IsNullOrEmpty(home))
                    {
                        var linuxUser = Environment.GetEnvironmentVariable("USER") ?? Environment.UserName;
                        home = "/home/" + linuxUser;
                    }
                    return (home!.TrimEnd('/') + "/" + rel).Replace("//", "/", StringComparison.Ordinal);
                }
            }

            try
            {
                (bool ok, string stdout, string stderr, int code) = preferShell
                        ? RunProcessShell($"winepath -u {QuoteSingle(path)}", timeoutMs: 5000, workingDir: "/")
                        : RunProcessDirect("winepath", ["-u", path], workingDir: "/", timeoutMs: 5000);

                if (ok)
                {
                    var outp = (stdout ?? "").Trim();
                    if (!string.IsNullOrEmpty(outp) && outp.StartsWith('/'))
                        return outp.Replace("//", "/", StringComparison.Ordinal);
                }
                else
                {
                    _logger.LogTrace("winepath failed for {path} (exit {code}): {err}", path, code, stderr);
                }
            }
            catch (Exception ex)
            {
                _logger.LogTrace(ex, "winepath invocation failed for {path}", path);
            }
        }

        return path.Replace('\\', '/').Replace("//", "/", StringComparison.Ordinal);
    }

    /// <summary>
    /// Compress an File using the WOF methods (NTFS)
    /// </summary>
    /// <param name="path">Path of the decompressed/normal file</param>
    /// <returns>Compessing state</returns>
    private bool WOFCompressFile(string path)
    {
        int size = Marshal.SizeOf<WofFileCompressionInfoV1>();
        IntPtr efInfoPtr = Marshal.AllocHGlobal(size);

        try
        {
            Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: false);
            ulong length = (ulong)size;

            return WithFileHandleForWOF(path, FileAccess.ReadWrite, h =>
            {
                int ret = WofSetFileDataLocation(h, WOF_PROVIDER_FILE, efInfoPtr, length);

                // 0x80070158 is the being "already compressed/unsupported" style return
                if (ret != 0 && ret != unchecked((int)0x80070158))
                {
                    _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X"));
                    return false;
                }

                return true;
            });
        }
        catch (DllNotFoundException ex)
        {
            _logger.LogTrace(ex, "WofUtil not available; skipping NTFS compaction for {file}", path);
            return false;
        }
        catch (EntryPointNotFoundException ex)
        {
            _logger.LogTrace(ex, "WOF entrypoint missing on this system (Wine/older OS); skipping NTFS compaction for {file}", path);
            return false;
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex, "Error compacting file {path}", path);
            return false;
        }
        finally
        {
            if (efInfoPtr != IntPtr.Zero)
                Marshal.FreeHGlobal(efInfoPtr);
        }
    }

    /// <summary>
    /// Checks if an File is compacted with WOF compression (NTFS)
    /// </summary>
    /// <param name="path">Path of the file</param>
    /// <returns>State of the file</returns>
    private static bool IsWOFCompactedFile(string filePath)
    {
        try
        {
            uint buf = (uint)Marshal.SizeOf<WofFileCompressionInfoV1>();
            int result = WofIsExternalFile(filePath, out int isExternal, out _, out var info, ref buf);
            if (result != 0 || isExternal == 0)
                return false;

            return info.Algorithm == (int)CompressionAlgorithm.XPRESS8K
                || info.Algorithm == (int)CompressionAlgorithm.XPRESS4K
                || info.Algorithm == (int)CompressionAlgorithm.XPRESS16K
                || info.Algorithm == (int)CompressionAlgorithm.LZX
                || info.Algorithm == (int)CompressionAlgorithm.LZNT1
                || info.Algorithm == (int)CompressionAlgorithm.NO_COMPRESSION;
        }
        catch
        {
            return false;
        }
    }

    /// <summary>
    /// Checks if an File is compacted any WOF compression with an WOF backing (NTFS)
    /// </summary>
    /// <param name="path">Path of the file</param>
    /// <returns>State of the file, if its an external (no backing) and which algorithm if detected</returns>
    private static bool TryIsWofExternal(string path, out bool isExternal, out int algorithm)
    {
        isExternal = false;
        algorithm = 0;
        try
        {
            uint buf = (uint)Marshal.SizeOf<WofFileCompressionInfoV1>();
            int hr = WofIsExternalFile(path, out int ext, out _, out var info, ref buf);
            if (hr == 0 && ext != 0)
            {
                isExternal = true;
                algorithm = info.Algorithm;
            }
            return true;
        }
        catch (DllNotFoundException)
        {
            return false;
        }
        catch (EntryPointNotFoundException)
        {
            return false;
        }
    }

    /// <summary>
    /// Checks if an File is compacted with Btrfs compression
    /// </summary>
    /// <param name="path">Path of the file</param>
    /// <returns>State of the file</returns>
    private bool IsBtrfsCompressedFile(string path)
    {
        return RunWithBtrfsGate(() =>
        {
            try
            {
                string linuxPath = _isWindows ? ResolveLinuxPathForWine(path) : path;

                var task = _fragBatch.IsCompressedAsync(linuxPath, _compactionCts.Token);

                if (task.Wait(TimeSpan.FromSeconds(5), _compactionCts.Token) && task.IsCompletedSuccessfully)
                    return task.Result;

                _logger.LogTrace("filefrag batch timed out for {file}", linuxPath);
                return false;
            }
            catch (OperationCanceledException)
            {
                return false;
            }
            catch (Exception ex)
            {
                _logger.LogDebug(ex, "filefrag batch check failed for {file}", path);
                return false;
            }
        });
    }

    /// <summary>
    /// Compress an Btrfs File
    /// </summary>
    /// <param name="path">Path of the decompressed/normal file</param>
    /// <returns>Compessing state</returns>
    private bool BtrfsCompressFile(string path)
    {
        return RunWithBtrfsGate(() => {
            try
            {
                var (winPath, linuxPath) = ResolvePathsForBtrfs(path);

                if (IsBtrfsCompressedFile(linuxPath))
                {
                    _logger.LogTrace("Already Btrfs compressed: {file} (linux={linux})", winPath, linuxPath);
                    return true;
                }

                if (!ProbeFileReadableForBtrfs(winPath, linuxPath))
                {
                    _logger.LogTrace("Probe failed; cannot open file for compress: {file} (linux={linux})", winPath, linuxPath);
                    return false;
                }

                var probe = RunProcessShell("command -v btrfs || which btrfs", timeoutMs: 5000);
                var _btrfsAvailable = probe.ok && !string.IsNullOrWhiteSpace(probe.stdout);
                if (!_btrfsAvailable)
                    _logger.LogWarning("btrfs cli not found in path. Compression will be skipped.");

                (bool ok, string stdout, string stderr, int code) =
                    _isWindows
                        ? RunProcessShell($"btrfs filesystem defragment -clzo -- {QuoteSingle(linuxPath)}")
                        : RunProcessDirect("btrfs", ["filesystem", "defragment", "-clzo", "--", linuxPath]);

                if (!ok)
                {
                    _logger.LogWarning("btrfs defragment failed for {file} (linux={linux}) exit {code}: {stderr}",
                        winPath, linuxPath, code, stderr);
                    return false;
                }

                if (!string.IsNullOrWhiteSpace(stdout))
                    _logger.LogTrace("btrfs output for {file}: {out}", winPath, stdout.Trim());

                _logger.LogInformation("Compressed btrfs file successfully: {file} (linux={linux})", winPath, linuxPath);
                return true;
            }
            catch (Exception ex)
            {
                _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path);
                return false;
            }
        });
    }

    /// <summary>
    /// Attempt opening file stream for WOF functions
    /// </summary>
    /// <param name="path">File that has to be accessed</param>
    /// <param name="access">Permissions for the file</param>
    /// <param name="body">Access of the file stream for the WOF function to handle.</param>
    /// <returns>State of the attempt for the file</returns>
    private bool WithFileHandleForWOF(string path, FileAccess access, Func<SafeFileHandle, bool> body)
    {
        const FileShare share = FileShare.ReadWrite | FileShare.Delete;

        for (int attempt = 0; attempt < _maxRetries; attempt++)
        {
            try
            {
                using var fs = new FileStream(path, FileMode.Open, access, share);

                var handle = fs.SafeFileHandle;
                if (handle.IsInvalid)
                {
                    _logger.LogWarning("Invalid file handle for {file}", path);
                    return false;
                }

                return body(handle);
            }
            catch (IOException ex)
            {
                if (attempt == _maxRetries - 1)
                {
                    _logger.LogWarning(ex, "File still in use after {attempts} attempts, skipping {file}", _maxRetries, path);
                    return false;
                }

                int delay = 150 * (attempt + 1);
                _logger.LogTrace(ex, "File busy, retrying in {delay}ms for {file}", delay, path);
                Thread.Sleep(delay);
            }
        }

        return false;
    }

    /// <summary>
    /// Runs an nonshell process meant for Linux enviroments
    /// </summary>
    /// <param name="fileName">File that has to be excuted</param>
    /// <param name="args">Arguments meant for the file/command</param>
    /// <param name="workingDir">Working directory used to execute the file with/without arguments</param>
    /// <param name="timeoutMs">Timeout timer for the process</param>
    /// <returns>State of the process, output of the process and error with exit code</returns>
    private (bool ok, string stdout, string stderr, int exitCode) RunProcessDirect(string fileName, IEnumerable<string> args, string? workingDir = null, int timeoutMs = 60000)
    {
        var psi = new ProcessStartInfo(fileName)
        {
            RedirectStandardOutput = true,
            RedirectStandardError = true,
            UseShellExecute = false,
            CreateNoWindow = true,
            WorkingDirectory = workingDir ?? "/",
        };

        foreach (var a in args) psi.ArgumentList.Add(a);
        EnsureUnixPathEnv(psi);

        using var proc = Process.Start(psi);
        if (proc is null) return (false, "", "failed to start process", -1);

        var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token);
        if (!success)
        {
            return (false, so2, se2, -1);
        }

        int code;
        try { code = proc.ExitCode; }
        catch { code = -1; }

        bool ok = code == 0;

        if (!ok && code == -1 &&
            string.IsNullOrWhiteSpace(se2) && !string.IsNullOrWhiteSpace(so2))
        {
            ok = true;
        }

        return (ok, so2, se2, code);
    }

    /// <summary>
    /// Runs an shell using '/bin/bash'/ command meant for Linux/Wine enviroments
    /// </summary>
    /// <param name="command">Command that has to be excuted</param>
    /// <param name="timeoutMs">Timeout timer for the process</param>
    /// <returns>State of the process, output of the process and error with exit code</returns>
    private (bool ok, string stdout, string stderr, int exitCode) RunProcessShell(string command, string? workingDir = null, int timeoutMs = 60000)
    {
        var psi = new ProcessStartInfo("/bin/bash")
        {
            RedirectStandardOutput = true,
            RedirectStandardError = true,
            UseShellExecute = false,
            CreateNoWindow = true,
            WorkingDirectory = workingDir ?? "/",
        };

        // Use a Login shell so PATH includes /usr/sbin etc. AKA -lc for login shell
        psi.ArgumentList.Add("-lc");
        psi.ArgumentList.Add(QuoteDouble(command));
        EnsureUnixPathEnv(psi);

        using var proc = Process.Start(psi);
        if (proc is null) return (false, "", "failed to start /bin/bash", -1);

        var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token);
        if (!success)
        {
            return (false, so2, se2, -1);
        }

        int code;
        try { code = proc.ExitCode; }
        catch { code = -1; }

        bool ok = code == 0;

        if (!ok && code == -1 && string.IsNullOrWhiteSpace(se2) && !string.IsNullOrWhiteSpace(so2))
        {
            ok = true;
        }

        return (ok, so2, se2, code);
    }

    /// <summary>
    /// Checking the process result for shell or direct processes
    /// </summary>
    /// <param name="proc">Process</param>
    /// <param name="timeoutMs">How long when timeout goes over threshold</param>
    /// <param name="token">Cancellation Token</param>
    /// <returns>Multiple variables</returns>
    private (bool success, string output, string errorCode) CheckProcessResult(Process proc, int timeoutMs, CancellationToken token)
    {
        var outTask = proc.StandardOutput.ReadToEndAsync(token);
        var errTask = proc.StandardError.ReadToEndAsync(token);
        var bothTasks = Task.WhenAll(outTask, errTask);

        var finished = Task.WhenAny(bothTasks, Task.Delay(timeoutMs, token)).GetAwaiter().GetResult();

        if (token.IsCancellationRequested)
            return KillProcess(proc, outTask, errTask, token);

        if (finished != bothTasks)
            return KillProcess(proc, outTask, errTask, token);

        bool isWine = _context.IsWine;
        if (!isWine)
        {
            try { proc.WaitForExit(); } catch { /* ignore quirks */ }
        }
        else
        {
            var sw = Stopwatch.StartNew();
            while (!proc.HasExited && sw.ElapsedMilliseconds < 75)
                Thread.Sleep(5);
        }

        var stdout = outTask.Status == TaskStatus.RanToCompletion ? outTask.Result : "";
        var stderr = errTask.Status == TaskStatus.RanToCompletion ? errTask.Result : "";

        int code = -1;
        try { if (proc.HasExited) code = proc.ExitCode; } catch { /* Wine may still throw */ }

        bool ok = code == 0 || (isWine && string.IsNullOrWhiteSpace(stderr));

        return (ok, stdout, stderr);

        static (bool success, string output, string errorCode) KillProcess(
            Process proc, Task<string> outTask, Task<string> errTask, CancellationToken token)
        {
            try { proc.Kill(entireProcessTree: true); } catch { /* ignore */ }
            try { Task.WaitAll([outTask, errTask], 1000, token); } catch { /* ignore */ }

            var so = outTask.IsCompleted ? outTask.Result : "";
            var se = errTask.IsCompleted ? errTask.Result : "canceled/timeout";
            return (false, so, se);
        }
    }

    /// <summary>
    /// Enqueues the compaction/decompation of an filepath.
    /// </summary>
    /// <param name="filePath">Filepath that will be enqueued</param>
    private void EnqueueCompaction(string filePath)
    {
        // Safe-checks
        if (string.IsNullOrWhiteSpace(filePath))
            return;

        if (!_context.UseCompactor)
            return;

        if (!File.Exists(filePath))
            return;

        if (!_pendingCompactions.TryAdd(filePath, 0))
            return;

        bool enqueued = false;
        try
        {
            bool isWine = _context.IsWine;
            var fsType = GetFilesystemType(filePath, isWine);

            // If under Wine, we should skip NTFS because its not Windows but might return NTFS.
            if (fsType == FilesystemType.NTFS && isWine)
            {
                _logger.LogTrace("Skip enqueue (NTFS under Wine) {file}", filePath);
                return;
            }

            // Unknown file system should be skipped.
            if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs)
            {
                _logger.LogTrace("Skip enqueue (unsupported fs) {fs} {file}", fsType, filePath);
                return;
            }

            // Channel got closed, skip enqueue on file
            if (!_compactionQueue.Writer.TryWrite(filePath))
            {
                _logger.LogTrace("Skip enqueue: compaction channel is/got closed {file}", filePath);
                return;
            }

            enqueued = true;
            _logger.LogTrace("Queued compaction for {file} (fs={fs})", filePath, fsType);
        }
        finally
        {
            if (!enqueued)
                _pendingCompactions.TryRemove(filePath, out _);
        }
    }

    /// <summary>
    /// Process the queue, meant for a worker/thread
    /// </summary>
    /// <param name="token">Cancellation token for the worker whenever it needs to be stopped</param>
    private async Task ProcessQueueWorkerAsync(int workerId, CancellationToken token)
    {
        try
        {
            while (await _compactionQueue.Reader.WaitToReadAsync(token).ConfigureAwait(false))
            {
                while (_compactionQueue.Reader.TryRead(out var filePath))
                {
                    try
                    {
                        token.ThrowIfCancellationRequested();
                        await _globalGate.WaitAsync(token).ConfigureAwait(false);

                        try
                        {
                            if (_context.UseCompactor && File.Exists(filePath))
                            {
                                if (!_compactionExecutor.TryCompact(filePath))
                                    CompactFile(filePath, workerId);
                            }
                        }
                        finally
                        {
                            _globalGate.Release();
                        }
                    }
                    catch (OperationCanceledException) { return; }
                    catch (Exception ex)
                    {
                        _logger.LogWarning(ex, "Error compacting file {file}", filePath);
                    }
                    finally
                    {
                        _pendingCompactions.TryRemove(filePath, out _);
                    }
                }
            }
        }
        catch (OperationCanceledException)
        {
            // Shutting down worker, this exception is expected
        }
    }

    /// <summary>
    /// Resolves linux path from wine pathing
    /// </summary>
    /// <param name="windowsPath">Windows path given from Wine</param>
    /// <returns>Linux path to be used in Linux</returns>
    private string ResolveLinuxPathForWine(string windowsPath)
    {
        var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(windowsPath)}", workingDir: null, 5000);
        if (ok && !string.IsNullOrWhiteSpace(outp)) return outp.Trim();
        return ToLinuxPathIfWine(windowsPath, isWine: true);
    }

    /// <summary>
    /// Ensures the Unix pathing to be included into the process start
    /// </summary>
    /// <param name="psi">Process</param>
    private static void EnsureUnixPathEnv(ProcessStartInfo psi)
    {
        if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p))
            psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin";
        else if (!p.Contains("/usr/sbin", StringComparison.Ordinal))
            psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p;
    }

    /// <summary>
    /// Resolves paths for Btrfs to be used on wine or linux and windows in case
    /// </summary>
    /// <param name="path">Path given t</param>
    /// <returns></returns>
    private (string windowsPath, string linuxPath) ResolvePathsForBtrfs(string path)
    {
        if (!_isWindows)
            return (path, path);

        var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(path)}", workingDir: null, 5000);
        var linux = (ok && !string.IsNullOrWhiteSpace(outp)) ? outp.Trim() : ToLinuxPathIfWine(path, isWine: true);

        return (path, linux);
    }

    /// <summary>
    /// Probes file if its readable to be used
    /// </summary>
    /// <param name="winePath">Windows path</param>
    /// <param name="linuxPath">Linux path</param>
    /// <returns>Succesfully probed or not</returns>
    private bool ProbeFileReadableForBtrfs(string winePath, string linuxPath)
    {
        try
        {
            if (_isWindows)
            {
                using var _ = new FileStream(winePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
            }
            else
            {
                using var _ = new FileStream(linuxPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
            }
            return true;
        }
        catch (Exception ex)
        {
            _logger.LogTrace(ex, "Probe open failed for {file} (linux={linux})", winePath, linuxPath);
            return false;
        }
    }

    /// <summary>
    /// Running functions into the Btrfs Gate/Threading.
    /// </summary>
    /// <typeparam name="T">Type of the function that wants to be run inside Btrfs Gate</typeparam>
    /// <param name="body">Body of the function</param>
    /// <returns>Task</returns>
    private T RunWithBtrfsGate<T>(Func<T> body)
    {
        bool acquired = false;
        try
        {
            _btrfsGate.Wait(_compactionCts.Token);
            acquired = true;
            return body();
        }
        finally
        {
            if (acquired) _btrfsGate.Release();
        }
    }


    [LibraryImport("kernel32.dll", SetLastError = true)]
    private static partial uint GetCompressedFileSizeW([MarshalAs(UnmanagedType.LPWStr)] string lpFileName, out uint lpFileSizeHigh);

    [LibraryImport("kernel32.dll", SetLastError = true)]
    [return: MarshalAs(UnmanagedType.Bool)]
    private static partial bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped);

    [LibraryImport("WofUtil.dll")]
    private static partial int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WofFileCompressionInfoV1 Info, ref uint BufferLength);

    [LibraryImport("WofUtil.dll")]
    private static partial int WofSetFileDataLocation(SafeFileHandle FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length);

    private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'";

    private static string QuoteDouble(string s) => "\"" + s.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal).Replace("$", "\\$", StringComparison.Ordinal).Replace("`", "\\`", StringComparison.Ordinal) + "\"";

    public void Dispose()
    {
        //Cleanup of gates and frag service
        _fragBatch?.Dispose();
        _btrfsGate?.Dispose();
        _globalGate?.Dispose();

        _compactionQueue.Writer.TryComplete();
        _compactionCts.Cancel();

        try
        {
            Task.WaitAll([.. _workers.Where(t => t != null)], TimeSpan.FromSeconds(5));
        }
        catch
        {
            // Ignore this catch on the dispose
        }
        finally
        {
            _compactionCts.Dispose();
        }

        GC.SuppressFinalize(this);
    }
}