sigma update

LightlessCompactor/FileCache/CompactorInterfaces.cs

@@ -0,0 +1,18 @@
+namespace LightlessSync.FileCache;
+
+public interface ICompactorContext
+{
+    bool UseCompactor { get; }
+    string CacheFolder { get; }
+    bool IsWine { get; }
+}
+
+public interface ICompactionExecutor
+{
+    bool TryCompact(string filePath);
+}
+
+public sealed class NoopCompactionExecutor : ICompactionExecutor
+{
+    public bool TryCompact(string filePath) => false;
+}

LightlessCompactor/LightlessCompactor.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <LangVersion>latest</LangVersion>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.1" />
+  </ItemGroup>
+
+</Project>

LightlessCompactor/Services/Compactor/BatchFileFragService.cs

@@ -0,0 +1,224 @@
+using Microsoft.Extensions.Logging;
+using System.Text.RegularExpressions;
+using System.Threading.Channels;
+
+namespace LightlessSync.Services.Compactor
+{
+    /// <summary>
+    /// This batch service is made for the File Frag command, because of each file needing to use this command.
+    /// It's better to combine into one big command in batches then doing each command on each compressed call.
+    /// </summary>
+    public sealed partial class BatchFilefragService : IDisposable
+    {
+        private readonly Channel<(string path, TaskCompletionSource<bool> tcs)> _ch;
+        private readonly Task _worker;
+        private readonly bool _useShell;
+        private readonly ILogger _log;
+        private readonly int _batchSize;
+        private readonly TimeSpan _flushDelay;
+        private readonly CancellationTokenSource _cts = new();
+
+        public delegate (bool ok, string stdout, string stderr, int exitCode) RunDirect(string fileName, IEnumerable<string> args, string? workingDir, int timeoutMs);
+        private readonly RunDirect _runDirect;
+
+        public delegate (bool ok, string stdout, string stderr, int exitCode) RunShell(string command, string? workingDir, int timeoutMs);
+        private readonly RunShell _runShell;
+
+        public BatchFilefragService(bool useShell, ILogger log, int batchSize = 128, int flushMs = 25, RunDirect? runDirect = null, RunShell? runShell = null)
+        {
+            _useShell = useShell;
+            _log = log;
+            _batchSize = Math.Max(8, batchSize);
+            _flushDelay = TimeSpan.FromMilliseconds(Math.Max(5, flushMs));
+            _ch = Channel.CreateUnbounded<(string, TaskCompletionSource<bool>)>(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false });
+
+            // require runners to be setup, wouldnt start otherwise
+            if (runDirect is null || runShell is null)
+                throw new ArgumentNullException(nameof(runDirect), "Provide process runners from FileCompactor");
+            _runDirect = runDirect;
+            _runShell = runShell;
+
+            _worker = Task.Run(ProcessAsync, _cts.Token);
+        }
+
+        /// <summary>
+        /// Checks if the file is compressed using Btrfs using tasks
+        /// </summary>
+        /// <param name="linuxPath">Linux/Wine path given for the file.</param>
+        /// <param name="ct">Cancellation Token</param>
+        /// <returns>If it was compressed or not</returns>
+        public Task<bool> IsCompressedAsync(string linuxPath, CancellationToken ct = default)
+        {
+            var tcs = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
+
+            if (!_ch.Writer.TryWrite((linuxPath, tcs)))
+            {
+                tcs.TrySetResult(false);
+                return tcs.Task;
+            }
+
+            if (ct.CanBeCanceled)
+            {
+                var reg = ct.Register(() => tcs.TrySetCanceled(ct));
+                _ = tcs.Task.ContinueWith(_ => reg.Dispose(), CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
+            }
+
+            return tcs.Task;
+        }
+
+        /// <summary>
+        /// Process the pending compression tasks asynchronously
+        /// </summary>
+        /// <returns>Task</returns>
+        private async Task ProcessAsync()
+        {
+            var reader = _ch.Reader;
+            var pending = new List<(string path, TaskCompletionSource<bool> tcs)>(_batchSize);
+
+            try
+            {
+                while (await reader.WaitToReadAsync(_cts.Token).ConfigureAwait(false))
+                {
+                    if (!reader.TryRead(out var first)) continue;
+                    pending.Add(first);
+
+                    var flushAt = DateTime.UtcNow + _flushDelay;
+                    while (pending.Count < _batchSize && DateTime.UtcNow < flushAt)
+                    {
+                        if (reader.TryRead(out var item))
+                        {
+                            pending.Add(item);
+                            continue;
+                        }
+
+                        if ((flushAt - DateTime.UtcNow) <= TimeSpan.Zero) break;
+                        try
+                        {
+                            await Task.Delay(TimeSpan.FromMilliseconds(5), _cts.Token).ConfigureAwait(false);
+                        }
+                        catch
+                        {
+                            break;
+                        }
+                    }
+
+                    try
+                    {
+                        var map = RunBatch(pending.Select(p => p.path));
+                        foreach (var (path, tcs) in pending)
+                        {
+                            tcs.TrySetResult(map.TryGetValue(path, out var c) && c);
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        _log.LogDebug(ex, "filefrag batch failed. falling back to false");
+                        foreach (var (_, tcs) in pending)
+                        {
+                            tcs.TrySetResult(false);
+                        }
+                    }
+                    finally
+                    {
+                        pending.Clear();
+                    }
+                }
+            }
+            catch (OperationCanceledException)
+            {
+                //Shutting down worker, exception called
+            }
+        }
+
+        /// <summary>
+        /// Running the batch of each file in the queue in one file frag command.
+        /// </summary>
+        /// <param name="paths">Paths that are needed for the command building for the batch return</param>
+        /// <returns>Path of the file and if it went correctly</returns>
+        /// <exception cref="InvalidOperationException">Failing to start filefrag on the system if this exception is found</exception>
+        private Dictionary<string, bool> RunBatch(IEnumerable<string> paths)
+        {
+            var list = paths.Distinct(StringComparer.Ordinal).ToList();
+            var result = list.ToDictionary(p => p, _ => false, StringComparer.Ordinal);
+
+            (bool ok, string stdout, string stderr, int code) res;
+
+            if (_useShell)
+            {
+                var inner = "filefrag -v -- " + string.Join(' ', list.Select(QuoteSingle));
+                res = _runShell(inner, timeoutMs: 15000, workingDir: "/");
+            }
+            else
+            {
+                var args = new List<string> { "-v", "--" };
+                args.AddRange(list);
+                res = _runDirect("filefrag", args, workingDir: "/", timeoutMs: 15000);
+            }
+
+            if (!string.IsNullOrWhiteSpace(res.stderr))
+                _log.LogTrace("filefrag stderr (batch): {err}", res.stderr.Trim());
+
+            ParseFilefrag(res.stdout, result);
+            return result;
+        }
+
+        /// <summary>
+        /// Parsing the string given from the File Frag command into mapping
+        /// </summary>
+        /// <param name="output">Output of the process from the File Frag</param>
+        /// <param name="map">Mapping of the processed files</param>
+        private static void ParseFilefrag(string output, Dictionary<string, bool> map)
+        {
+            var reHeaderColon = ColonRegex();
+            var reHeaderSize = SizeRegex();
+
+            string? current = null;
+            using var sr = new StringReader(output);
+            for (string? line = sr.ReadLine(); line != null; line = sr.ReadLine())
+            {
+                var m1 = reHeaderColon.Match(line);
+                if (m1.Success) { current = m1.Groups[1].Value; continue; }
+
+                var m2 = reHeaderSize.Match(line);
+                if (m2.Success) { current = m2.Groups[1].Value; continue; }
+
+                if (current is not null && line.Contains("flags:", StringComparison.OrdinalIgnoreCase) &&
+                    line.Contains("compressed", StringComparison.OrdinalIgnoreCase) && map.ContainsKey(current))
+                {
+                    map[current] = true;
+                }
+            }
+        }
+
+        private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'";
+
+        /// <summary>
+        /// Regex of the File Size return on the Linux/Wine systems, giving back the amount
+        /// </summary>
+        /// <returns>Regex of the File Size</returns>
+        [GeneratedRegex(@"^File size of (/.+?) is ", RegexOptions.ExplicitCapture | RegexOptions.CultureInvariant, matchTimeoutMilliseconds: 500)]
+        private static partial Regex SizeRegex();
+
+        /// <summary>
+        /// Regex on colons return on the Linux/Wine systems
+        /// </summary>
+        /// <returns>Regex of the colons in the given path</returns>
+        [GeneratedRegex(@"^(/.+?):\s", RegexOptions.ExplicitCapture | RegexOptions.CultureInvariant, matchTimeoutMilliseconds: 500)]
+        private static partial Regex ColonRegex();
+
+        public void Dispose()
+        {
+            _ch.Writer.TryComplete();
+            _cts.Cancel();
+            try
+            {
+                _worker.Wait(TimeSpan.FromSeconds(2), _cts.Token);
+            }
+            catch
+            {
+                // Ignore the catch in dispose
+            }
+            _cts.Dispose();
+        }
+    }
+}

LightlessCompactor/Utils/FileSystemHelper.cs

@@ -0,0 +1,251 @@
+using Microsoft.Extensions.Logging;
+using System.Collections.Concurrent;
+using System.Runtime.InteropServices;
+
+namespace LightlessSync.Utils
+{
+    public static class FileSystemHelper
+    {
+        public enum FilesystemType
+        {
+            Unknown = 0,
+            NTFS, // Compressable on file level
+            Btrfs, // Compressable on file level
+            Ext4, // Uncompressable
+            Xfs, // Uncompressable
+            Apfs, // Compressable on OS
+            HfsPlus, // Compressable on OS
+            Fat, // Uncompressable
+            Exfat, // Uncompressable
+            Zfs // Compressable, not on file level
+        }
+
+        private const string _mountPath = "/proc/mounts";
+        private const int _defaultBlockSize = 4096;
+        private static readonly Dictionary<string, int> _blockSizeCache = new(StringComparer.OrdinalIgnoreCase);
+        private static readonly ConcurrentDictionary<string, FilesystemType> _filesystemTypeCache = new(StringComparer.OrdinalIgnoreCase);
+
+        public static FilesystemType GetFilesystemType(string filePath, bool isWine = false)
+        {
+            try
+            {
+                string rootPath;
+
+                if (OperatingSystem.IsWindows() && (!IsProbablyWine() || !isWine))
+                {
+                    var info = new FileInfo(filePath);
+                    var dir = info.Directory ?? new DirectoryInfo(filePath);
+                    rootPath = dir.Root.FullName;
+                }
+                else
+                {
+                    rootPath = GetMountPoint(filePath);
+                    if (string.IsNullOrEmpty(rootPath))
+                        rootPath = "/";
+                }
+
+                if (_filesystemTypeCache.TryGetValue(rootPath, out var cachedType))
+                    return cachedType;
+
+                FilesystemType detected;
+
+                if (OperatingSystem.IsWindows() && (!IsProbablyWine() || !isWine))
+                {
+                    var root = new DriveInfo(rootPath);
+                    var format = root.DriveFormat?.ToUpperInvariant() ?? string.Empty;
+
+                    detected = format switch
+                    {
+                        "NTFS" => FilesystemType.NTFS,
+                        "FAT32" => FilesystemType.Fat,
+                        "EXFAT" => FilesystemType.Exfat,
+                        _ => FilesystemType.Unknown
+                    };
+                }
+                else
+                {
+                    detected = GetLinuxFilesystemType(filePath);
+                }
+
+                if (isWine || IsProbablyWine())
+                {
+                    switch (detected)
+                    {
+                        case FilesystemType.NTFS:
+                        case FilesystemType.Unknown:
+                            {
+                                var linuxDetected = GetLinuxFilesystemType(filePath);
+                                if (linuxDetected != FilesystemType.Unknown)
+                                {
+                                    detected = linuxDetected;
+                                }
+
+                                break;
+                            }
+                    }
+                }
+
+                _filesystemTypeCache[rootPath] = detected;
+                return detected;
+            }
+            catch
+            {
+                return FilesystemType.Unknown;
+            }
+        }
+
+        private static string GetMountPoint(string filePath)
+        {
+            try
+            {
+                var path = Path.GetFullPath(filePath);
+                if (!File.Exists(_mountPath)) return "/";
+                var mounts = File.ReadAllLines(_mountPath);
+
+                string bestMount = "/";
+                foreach (var line in mounts)
+                {
+                    var parts = line.Split(' ');
+                    if (parts.Length < 3) continue;
+                    var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal);
+
+                    string normalizedMount;
+                    try { normalizedMount = Path.GetFullPath(mountPoint); }
+                    catch { normalizedMount = mountPoint; }
+
+                    if (path.StartsWith(normalizedMount, StringComparison.Ordinal) &&
+                        normalizedMount.Length > bestMount.Length)
+                    {
+                        bestMount = normalizedMount;
+                    }
+                }
+
+                return bestMount;
+            }
+            catch
+            {
+                return "/";
+            }
+        }
+
+        public static string GetMountOptionsForPath(string path)
+        {
+            try
+            {
+                var fullPath = Path.GetFullPath(path);
+                var mounts = File.ReadAllLines("/proc/mounts");
+                string bestMount = string.Empty;
+                string mountOptions = string.Empty;
+
+                foreach (var line in mounts)
+                {
+                    var parts = line.Split(' ');
+                    if (parts.Length < 4) continue;
+                    var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal);
+                    string normalized;
+                    try { normalized = Path.GetFullPath(mountPoint); }
+                    catch { normalized = mountPoint; }
+
+                    if (fullPath.StartsWith(normalized, StringComparison.Ordinal) &&
+                        normalized.Length > bestMount.Length)
+                    {
+                        bestMount = normalized;
+                        mountOptions = parts[3];
+                    }
+                }
+
+                return mountOptions;
+            }
+            catch (Exception)
+            {
+                return string.Empty;
+            }
+        }
+
+        private static FilesystemType GetLinuxFilesystemType(string filePath)
+        {
+            try
+            {
+                var mountPoint = GetMountPoint(filePath);
+                var mounts = File.ReadAllLines(_mountPath);
+
+                foreach (var line in mounts)
+                {
+                    var parts = line.Split(' ');
+                    if (parts.Length < 3) continue;
+                    var mount = parts[1].Replace("\\040", " ", StringComparison.Ordinal);
+                    if (string.Equals(mount, mountPoint, StringComparison.Ordinal))
+                    {
+                        var fstype = parts[2].ToLowerInvariant();
+                        return fstype switch
+                        {
+                            "btrfs" => FilesystemType.Btrfs,
+                            "ext4" => FilesystemType.Ext4,
+                            "xfs" => FilesystemType.Xfs,
+                            "zfs" => FilesystemType.Zfs,
+                            "apfs" => FilesystemType.Apfs,
+                            "hfsplus" => FilesystemType.HfsPlus,
+                            _ => FilesystemType.Unknown
+                        };
+                    }
+                }
+
+                return FilesystemType.Unknown;
+            }
+            catch
+            {
+                return FilesystemType.Unknown;
+            }
+        }
+
+        public static int GetBlockSizeForPath(string path, ILogger? logger = null, bool isWine = false)
+        {
+            try
+            {
+                if (string.IsNullOrWhiteSpace(path))
+                    return _defaultBlockSize;
+
+                var fi = new FileInfo(path);
+                if (!fi.Exists)
+                    return _defaultBlockSize;
+
+                var root = fi.Directory?.Root.FullName.ToLowerInvariant() ?? "/";
+                if (_blockSizeCache.TryGetValue(root, out int cached))
+                    return cached;
+
+                if (OperatingSystem.IsWindows() && !isWine)
+                {
+                    int result = GetDiskFreeSpaceW(root,
+                        out uint sectorsPerCluster,
+                        out uint bytesPerSector,
+                        out _,
+                        out _);
+
+                    if (result == 0)
+                    {
+                        logger?.LogWarning("Failed to determine block size for {root}", root);
+                        return _defaultBlockSize;
+                    }
+
+                    int clusterSize = (int)(sectorsPerCluster * bytesPerSector);
+                    _blockSizeCache[root] = clusterSize;
+                    logger?.LogTrace("NTFS cluster size for {root}: {cluster}", root, clusterSize);
+                    return clusterSize;
+                }
+
+                return _defaultBlockSize;
+            }
+            catch (Exception ex)
+            {
+                logger?.LogTrace(ex, "Error determining block size for {path}", path);
+                return _defaultBlockSize;
+            }
+        }
+
+        [DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)]
+        private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters, out uint lpTotalNumberOfClusters);
+
+        //Extra check on 
+        public static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts");
+    }
+}