1137 lines
41 KiB
C#
1137 lines
41 KiB
C#
using LightlessSync.LightlessConfiguration;
|
|
using LightlessSync.Services;
|
|
using LightlessSync.Services.Compactor;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Win32.SafeHandles;
|
|
using System.Collections.Concurrent;
|
|
using System.ComponentModel;
|
|
using System.Diagnostics;
|
|
using System.Runtime.InteropServices;
|
|
using System.Threading.Channels;
|
|
using static LightlessSync.Utils.FileSystemHelper;
|
|
|
|
namespace LightlessSync.FileCache;
|
|
|
|
public sealed class FileCompactor : IDisposable
|
|
{
|
|
public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U;
|
|
public const ulong WOF_PROVIDER_FILE = 2UL;
|
|
public const int _maxRetries = 3;
|
|
private readonly bool _isWindows;
|
|
|
|
private readonly ConcurrentDictionary<string, byte> _pendingCompactions;
|
|
private readonly ILogger<FileCompactor> _logger;
|
|
private readonly LightlessConfigService _lightlessConfigService;
|
|
private readonly DalamudUtilService _dalamudUtilService;
|
|
|
|
private readonly Channel<string> _compactionQueue;
|
|
private readonly CancellationTokenSource _compactionCts = new();
|
|
|
|
private readonly List<Task> _workers = [];
|
|
private readonly SemaphoreSlim _globalGate;
|
|
|
|
//Limit btrfs gate on half of threads given to compactor.
|
|
private static readonly SemaphoreSlim _btrfsGate = new(4, 4);
|
|
private readonly BatchFilefragService _fragBatch;
|
|
|
|
private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo = new()
|
|
{
|
|
Algorithm = (int)CompressionAlgorithm.XPRESS8K,
|
|
Flags = 0
|
|
};
|
|
|
|
[StructLayout(LayoutKind.Sequential, Pack = 1)]
|
|
private struct WOF_FILE_COMPRESSION_INFO_V1
|
|
{
|
|
public int Algorithm;
|
|
public ulong Flags;
|
|
}
|
|
|
|
private enum CompressionAlgorithm
|
|
{
|
|
NO_COMPRESSION = -2,
|
|
LZNT1 = -1,
|
|
XPRESS4K = 0,
|
|
LZX = 1,
|
|
XPRESS8K = 2,
|
|
XPRESS16K = 3
|
|
}
|
|
|
|
public FileCompactor(ILogger<FileCompactor> logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService)
|
|
{
|
|
_pendingCompactions = new(StringComparer.OrdinalIgnoreCase);
|
|
_logger = logger;
|
|
_lightlessConfigService = lightlessConfigService;
|
|
_dalamudUtilService = dalamudUtilService;
|
|
_isWindows = OperatingSystem.IsWindows();
|
|
|
|
_compactionQueue = Channel.CreateUnbounded<string>(new UnboundedChannelOptions
|
|
{
|
|
SingleReader = false,
|
|
SingleWriter = false
|
|
});
|
|
|
|
int workers = Math.Clamp(Math.Min(Environment.ProcessorCount / 2, 4), 1, 8);
|
|
_globalGate = new SemaphoreSlim(workers, workers);
|
|
int workerCount = Math.Max(workers * 2, workers);
|
|
|
|
for (int i = 0; i < workerCount; i++)
|
|
{
|
|
_workers.Add(Task.Factory.StartNew(
|
|
() => ProcessQueueWorkerAsync(_compactionCts.Token),
|
|
_compactionCts.Token,
|
|
TaskCreationOptions.LongRunning,
|
|
TaskScheduler.Default).Unwrap());
|
|
}
|
|
|
|
_fragBatch = new BatchFilefragService(
|
|
useShell: _dalamudUtilService.IsWine,
|
|
log: _logger,
|
|
batchSize: 64,
|
|
flushMs: 25,
|
|
runDirect: RunProcessDirect,
|
|
runShell: RunProcessShell
|
|
);
|
|
|
|
_logger.LogInformation("FileCompactor started with {workers} workers", workerCount);
|
|
}
|
|
|
|
public bool MassCompactRunning { get; private set; }
|
|
public string Progress { get; private set; } = string.Empty;
|
|
|
|
/// <summary>
|
|
/// Compact the storage of the Cache Folder
|
|
/// </summary>
|
|
/// <param name="compress">Used to check if files needs to be compressed</param>
|
|
public void CompactStorage(bool compress)
|
|
{
|
|
MassCompactRunning = true;
|
|
try
|
|
{
|
|
var allFiles = Directory.EnumerateFiles(_lightlessConfigService.Current.CacheFolder).ToList();
|
|
int total = allFiles.Count;
|
|
int current = 0;
|
|
|
|
foreach (var file in allFiles)
|
|
{
|
|
current++;
|
|
Progress = $"{current}/{total}";
|
|
|
|
try
|
|
{
|
|
// Compress or decompress files
|
|
if (compress)
|
|
CompactFile(file);
|
|
else
|
|
DecompressFile(file);
|
|
}
|
|
catch (IOException ioEx)
|
|
{
|
|
_logger.LogDebug(ioEx, "File {file} locked or busy, skipping", file);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Error compacting/decompressing file {file}", file);
|
|
}
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
MassCompactRunning = false;
|
|
Progress = string.Empty;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Write all bytes into a directory async
|
|
/// </summary>
|
|
/// <param name="filePath">Bytes will be writen to this filepath</param>
|
|
/// <param name="bytes">Bytes that have to be written</param>
|
|
/// <param name="token">Cancellation Token for interupts</param>
|
|
/// <returns>Writing Task</returns>
|
|
public async Task WriteAllBytesAsync(string filePath, byte[] bytes, CancellationToken token)
|
|
{
|
|
var dir = Path.GetDirectoryName(filePath);
|
|
if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir))
|
|
Directory.CreateDirectory(dir);
|
|
|
|
await File.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false);
|
|
|
|
if (_lightlessConfigService.Current.UseCompactor)
|
|
EnqueueCompaction(filePath);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the File size for an BTRFS or NTFS file system for the given FileInfo
|
|
/// </summary>
|
|
/// <param name="path">Amount of blocks used in the disk</param>
|
|
public long GetFileSizeOnDisk(FileInfo fileInfo)
|
|
{
|
|
var fsType = GetFilesystemType(fileInfo.FullName, _dalamudUtilService.IsWine);
|
|
|
|
if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine)
|
|
{
|
|
(bool flowControl, long value) = GetFileSizeNTFS(fileInfo);
|
|
if (!flowControl)
|
|
{
|
|
return value;
|
|
}
|
|
}
|
|
|
|
if (fsType == FilesystemType.Btrfs)
|
|
{
|
|
(bool flowControl, long value) = GetFileSizeBtrfs(fileInfo);
|
|
if (!flowControl)
|
|
{
|
|
return value;
|
|
}
|
|
}
|
|
|
|
return fileInfo.Length;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get File Size in an Btrfs file system (Linux/Wine).
|
|
/// </summary>
|
|
/// <param name="fileInfo">File that you want the size from.</param>
|
|
/// <returns>Succesful check and value of the filesize.</returns>
|
|
/// <exception cref="InvalidOperationException">Fails on the Process in StartProcessInfo</exception>
|
|
private (bool flowControl, long value) GetFileSizeBtrfs(FileInfo fileInfo)
|
|
{
|
|
try
|
|
{
|
|
var (_, linuxPath) = ResolvePathsForBtrfs(fileInfo.FullName);
|
|
|
|
var (ok, output, err, code) =
|
|
_isWindows
|
|
? RunProcessShell($"stat -c='%b' {QuoteSingle(linuxPath)}", workingDir: null, 10000)
|
|
: RunProcessDirect("stat", ["-c='%b'", linuxPath], workingDir: null, 10000);
|
|
|
|
if (ok && long.TryParse(output.Trim(), out long blocks))
|
|
return (false, blocks * 512L); // st_blocks are always 512B units
|
|
|
|
_logger.LogDebug("Btrfs size probe failed for {linux} (stat {code}, err {err}). Falling back to Length.", linuxPath, code, err);
|
|
return (false, fileInfo.Length);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogDebug(ex, "Failed Btrfs size probe for {file}, using Length", fileInfo.FullName);
|
|
return (false, fileInfo.Length);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get File Size in an NTFS file system (Windows).
|
|
/// </summary>
|
|
/// <param name="fileInfo">File that you want the size from.</param>
|
|
/// <returns>Succesful check and value of the filesize.</returns>
|
|
private (bool flowControl, long value) GetFileSizeNTFS(FileInfo fileInfo)
|
|
{
|
|
try
|
|
{
|
|
var blockSize = GetBlockSizeForPath(fileInfo.FullName, _logger, _dalamudUtilService.IsWine);
|
|
if (blockSize <= 0)
|
|
throw new InvalidOperationException($"Invalid block size {blockSize} for {fileInfo.FullName}");
|
|
|
|
uint lo = GetCompressedFileSizeW(fileInfo.FullName, out uint hi);
|
|
|
|
if (lo == 0xFFFFFFFF)
|
|
{
|
|
int err = Marshal.GetLastWin32Error();
|
|
if (err != 0)
|
|
throw new Win32Exception(err);
|
|
}
|
|
|
|
long size = ((long)hi << 32) | lo;
|
|
long rounded = ((size + blockSize - 1) / blockSize) * blockSize;
|
|
|
|
return (flowControl: false, value: rounded);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName);
|
|
return (flowControl: true, value: default);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Compressing the given path with BTRFS or NTFS file system.
|
|
/// </summary>
|
|
/// <param name="path">Path of the decompressed/normal file</param>
|
|
private void CompactFile(string filePath)
|
|
{
|
|
var fi = new FileInfo(filePath);
|
|
if (!fi.Exists)
|
|
{
|
|
_logger.LogTrace("Skip compaction: missing {file}", filePath);
|
|
return;
|
|
}
|
|
|
|
var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine);
|
|
var oldSize = fi.Length;
|
|
int blockSize = GetBlockSizeForPath(fi.FullName, _logger, _dalamudUtilService.IsWine);
|
|
|
|
// We skipping small files (128KiB) as they slow down the system a lot for BTRFS. as BTRFS has a different blocksize it requires an different calculation.
|
|
long minSizeBytes = fsType == FilesystemType.Btrfs
|
|
? Math.Max(blockSize * 2L, 128 * 1024L)
|
|
: Math.Max(blockSize, 8 * 1024L);
|
|
|
|
if (oldSize < minSizeBytes)
|
|
{
|
|
_logger.LogTrace("Skip compaction: {file} ({size} B) < threshold ({th} B)", filePath, oldSize, minSizeBytes);
|
|
return;
|
|
}
|
|
|
|
if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine)
|
|
{
|
|
if (!IsWOFCompactedFile(filePath))
|
|
{
|
|
_logger.LogDebug("NTFS compaction XPRESS8K: {file}", filePath);
|
|
if (WOFCompressFile(filePath))
|
|
{
|
|
var newSize = GetFileSizeOnDisk(fi);
|
|
_logger.LogDebug("NTFS compressed {file} {old} -> {new}", filePath, oldSize, newSize);
|
|
}
|
|
else
|
|
{
|
|
_logger.LogWarning("NTFS compression failed or unavailable for {file}", filePath);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
_logger.LogTrace("Already NTFS-compressed: {file}", filePath);
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (fsType == FilesystemType.Btrfs)
|
|
{
|
|
if (!IsBtrfsCompressedFile(filePath))
|
|
{
|
|
_logger.LogDebug("Btrfs compression zstd: {file}", filePath);
|
|
if (BtrfsCompressFile(filePath))
|
|
{
|
|
var newSize = GetFileSizeOnDisk(fi);
|
|
_logger.LogDebug("Btrfs compressed {file} {old} -> {new}", filePath, oldSize, newSize);
|
|
}
|
|
else
|
|
{
|
|
_logger.LogWarning("Btrfs compression failed or unavailable for {file}", filePath);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
_logger.LogTrace("Already Btrfs-compressed: {file}", filePath);
|
|
}
|
|
return;
|
|
}
|
|
|
|
_logger.LogTrace("Skip compact: unsupported FS for {file}", filePath);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Decompressing the given path with BTRFS file system or NTFS file system.
|
|
/// </summary>
|
|
/// <param name="path">Path of the compressed file</param>
|
|
private void DecompressFile(string path)
|
|
{
|
|
_logger.LogDebug("Decompress request: {file}", path);
|
|
var fsType = GetFilesystemType(path, _dalamudUtilService.IsWine);
|
|
|
|
if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine)
|
|
{
|
|
try
|
|
{
|
|
bool flowControl = DecompressWOFFile(path);
|
|
if (!flowControl)
|
|
{
|
|
return;
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "NTFS decompress error {file}", path);
|
|
}
|
|
}
|
|
|
|
if (fsType == FilesystemType.Btrfs)
|
|
{
|
|
try
|
|
{
|
|
bool flowControl = DecompressBtrfsFile(path);
|
|
if (!flowControl)
|
|
{
|
|
return;
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Btrfs decompress error {file}", path);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Decompress an BTRFS File on Wine/Linux
|
|
/// </summary>
|
|
/// <param name="path">Path of the compressed file</param>
|
|
/// <returns>Decompressing state</returns>
|
|
private bool DecompressBtrfsFile(string path)
|
|
{
|
|
return RunWithBtrfsGate(() =>
|
|
{
|
|
try
|
|
{
|
|
bool isWine = _dalamudUtilService?.IsWine ?? false;
|
|
string linuxPath = isWine ? ToLinuxPathIfWine(path, isWine) : path;
|
|
|
|
var opts = GetMountOptionsForPath(linuxPath);
|
|
bool hasCompress = opts.Contains("compress", StringComparison.OrdinalIgnoreCase);
|
|
bool hasCompressForce = opts.Contains("compress-force", StringComparison.OrdinalIgnoreCase);
|
|
|
|
if (hasCompressForce)
|
|
{
|
|
_logger.LogWarning("Cannot safely decompress {file}: mount options contains compress-force ({opts}).", linuxPath, opts);
|
|
return false;
|
|
}
|
|
|
|
if (hasCompress)
|
|
{
|
|
var setCmd = $"btrfs property set -- {QuoteDouble(linuxPath)} compression none";
|
|
var (okSet, _, errSet, codeSet) = isWine
|
|
? RunProcessShell(setCmd)
|
|
: RunProcessDirect("btrfs", ["property", "set", "--", linuxPath, "compression", "none"]);
|
|
|
|
if (!okSet)
|
|
{
|
|
_logger.LogWarning("Failed to set 'compression none' on {file}, please check drive options (exit code is: {code}): {err}", linuxPath, codeSet, errSet);
|
|
return false;
|
|
}
|
|
_logger.LogTrace("Set per-file 'compression none' on {file}", linuxPath);
|
|
}
|
|
|
|
if (!IsBtrfsCompressedFile(linuxPath))
|
|
{
|
|
_logger.LogTrace("{file} is not compressed, skipping decompression completely", linuxPath);
|
|
return true;
|
|
}
|
|
|
|
var (ok, stdout, stderr, code) = isWine
|
|
? RunProcessShell($"btrfs filesystem defragment -- {QuoteDouble(linuxPath)}")
|
|
: RunProcessDirect("btrfs", ["filesystem", "defragment", "--", linuxPath]);
|
|
|
|
if (!ok)
|
|
{
|
|
_logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit code is: {code}): {stderr}",
|
|
linuxPath, code, stderr);
|
|
return false;
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(stdout))
|
|
_logger.LogTrace("btrfs defragment output for {file}: {out}", linuxPath, stdout.Trim());
|
|
|
|
_logger.LogInformation("Decompressed (rewritten uncompressed) Btrfs file: {file}", linuxPath);
|
|
return true;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path);
|
|
return false;
|
|
}
|
|
});
|
|
}
|
|
|
|
/// <summary>
|
|
/// Decompress an NTFS File
|
|
/// </summary>
|
|
/// <param name="path">Path of the compressed file</param>
|
|
/// <returns>Decompressing state</returns>
|
|
private bool DecompressWOFFile(string path)
|
|
{
|
|
//Check if its already been compressed
|
|
if (TryIsWofExternal(path, out bool isExternal, out int algo))
|
|
{
|
|
if (!isExternal)
|
|
{
|
|
_logger.LogTrace("Already decompressed file: {file}", path);
|
|
return true;
|
|
}
|
|
var compressString = ((CompressionAlgorithm)algo).ToString();
|
|
_logger.LogTrace("WOF compression (algo={algo}) detected for {file}", compressString, path);
|
|
}
|
|
|
|
//This will attempt to start WOF thread.
|
|
return WithFileHandleForWOF(path, FileAccess.ReadWrite, h =>
|
|
{
|
|
if (!DeviceIoControl(h, FSCTL_DELETE_EXTERNAL_BACKING,
|
|
IntPtr.Zero, 0, IntPtr.Zero, 0,
|
|
out uint _, IntPtr.Zero))
|
|
{
|
|
int err = Marshal.GetLastWin32Error();
|
|
// 342 error code means its been decompressed after the control, we handle it as it succesfully been decompressed.
|
|
if (err == 342)
|
|
{
|
|
_logger.LogTrace("Successfully decompressed NTFS file {file}", path);
|
|
return true;
|
|
}
|
|
|
|
_logger.LogWarning("DeviceIoControl failed for {file} with Win32 error {err}", path, err);
|
|
return false;
|
|
}
|
|
|
|
_logger.LogTrace("Successfully decompressed NTFS file {file}", path);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
/// <summary>
|
|
/// Converts to Linux Path if its using Wine (diferent pathing system in Wine)
|
|
/// </summary>
|
|
/// <param name="path">Path that has to be converted</param>
|
|
/// <param name="isWine">Extra check if using the wine enviroment</param>
|
|
/// <returns>Converted path to be used in Linux</returns>
|
|
private string ToLinuxPathIfWine(string path, bool isWine, bool preferShell = true)
|
|
{
|
|
if (!isWine || !IsProbablyWine())
|
|
return path;
|
|
|
|
if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase))
|
|
return ("/" + path[3..].Replace('\\', '/')).Replace("//", "/", StringComparison.Ordinal);
|
|
|
|
if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
const string usersPrefix = "C:\\Users\\";
|
|
var p = path.Replace('/', '\\');
|
|
|
|
if (p.StartsWith(usersPrefix, StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
int afterUsers = usersPrefix.Length;
|
|
int slash = p.IndexOf('\\', afterUsers);
|
|
if (slash > 0 && slash + 1 < p.Length)
|
|
{
|
|
var rel = p[(slash + 1)..].Replace('\\', '/');
|
|
var home = Environment.GetEnvironmentVariable("HOME");
|
|
if (string.IsNullOrEmpty(home))
|
|
{
|
|
var linuxUser = Environment.GetEnvironmentVariable("USER") ?? Environment.UserName;
|
|
home = "/home/" + linuxUser;
|
|
}
|
|
return (home!.TrimEnd('/') + "/" + rel).Replace("//", "/", StringComparison.Ordinal);
|
|
}
|
|
}
|
|
|
|
try
|
|
{
|
|
(bool ok, string stdout, string stderr, int code) = preferShell
|
|
? RunProcessShell($"winepath -u {QuoteSingle(path)}", timeoutMs: 5000, workingDir: "/")
|
|
: RunProcessDirect("winepath", ["-u", path], workingDir: "/", timeoutMs: 5000);
|
|
|
|
if (ok)
|
|
{
|
|
var outp = (stdout ?? "").Trim();
|
|
if (!string.IsNullOrEmpty(outp) && outp.StartsWith('/'))
|
|
return outp.Replace("//", "/", StringComparison.Ordinal);
|
|
}
|
|
else
|
|
{
|
|
_logger.LogTrace("winepath failed for {path} (exit {code}): {err}", path, code, stderr);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogTrace(ex, "winepath invocation failed for {path}", path);
|
|
}
|
|
}
|
|
|
|
return path.Replace('\\', '/').Replace("//", "/", StringComparison.Ordinal);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Compress an File using the WOF methods (NTFS)
|
|
/// </summary>
|
|
/// <param name="path">Path of the decompressed/normal file</param>
|
|
/// <returns>Compessing state</returns>
|
|
private bool WOFCompressFile(string path)
|
|
{
|
|
int size = Marshal.SizeOf<WOF_FILE_COMPRESSION_INFO_V1>();
|
|
IntPtr efInfoPtr = Marshal.AllocHGlobal(size);
|
|
|
|
try
|
|
{
|
|
Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: false);
|
|
ulong length = (ulong)size;
|
|
|
|
return WithFileHandleForWOF(path, FileAccess.ReadWrite, h =>
|
|
{
|
|
int ret = WofSetFileDataLocation(h, WOF_PROVIDER_FILE, efInfoPtr, length);
|
|
|
|
// 0x80070158 is the being "already compressed/unsupported" style return
|
|
if (ret != 0 && ret != unchecked((int)0x80070158))
|
|
{
|
|
_logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X"));
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
});
|
|
}
|
|
catch (DllNotFoundException ex)
|
|
{
|
|
_logger.LogTrace(ex, "WofUtil not available; skipping NTFS compaction for {file}", path);
|
|
return false;
|
|
}
|
|
catch (EntryPointNotFoundException ex)
|
|
{
|
|
_logger.LogTrace(ex, "WOF entrypoint missing on this system (Wine/older OS); skipping NTFS compaction for {file}", path);
|
|
return false;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Error compacting file {path}", path);
|
|
return false;
|
|
}
|
|
finally
|
|
{
|
|
if (efInfoPtr != IntPtr.Zero)
|
|
Marshal.FreeHGlobal(efInfoPtr);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Checks if an File is compacted with WOF compression (NTFS)
|
|
/// </summary>
|
|
/// <param name="path">Path of the file</param>
|
|
/// <returns>State of the file</returns>
|
|
private static bool IsWOFCompactedFile(string filePath)
|
|
{
|
|
try
|
|
{
|
|
uint buf = (uint)Marshal.SizeOf<WOF_FILE_COMPRESSION_INFO_V1>();
|
|
int result = WofIsExternalFile(filePath, out int isExternal, out _, out var info, ref buf);
|
|
if (result != 0 || isExternal == 0)
|
|
return false;
|
|
|
|
return info.Algorithm == (int)CompressionAlgorithm.XPRESS8K
|
|
|| info.Algorithm == (int)CompressionAlgorithm.XPRESS4K
|
|
|| info.Algorithm == (int)CompressionAlgorithm.XPRESS16K
|
|
|| info.Algorithm == (int)CompressionAlgorithm.LZX
|
|
|| info.Algorithm == (int)CompressionAlgorithm.LZNT1
|
|
|| info.Algorithm == (int)CompressionAlgorithm.NO_COMPRESSION;
|
|
}
|
|
catch
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Checks if an File is compacted any WOF compression with an WOF backing (NTFS)
|
|
/// </summary>
|
|
/// <param name="path">Path of the file</param>
|
|
/// <returns>State of the file, if its an external (no backing) and which algorithm if detected</returns>
|
|
private static bool TryIsWofExternal(string path, out bool isExternal, out int algorithm)
|
|
{
|
|
isExternal = false;
|
|
algorithm = 0;
|
|
try
|
|
{
|
|
uint buf = (uint)Marshal.SizeOf<WOF_FILE_COMPRESSION_INFO_V1>();
|
|
int hr = WofIsExternalFile(path, out int ext, out _, out var info, ref buf);
|
|
if (hr == 0 && ext != 0)
|
|
{
|
|
isExternal = true;
|
|
algorithm = info.Algorithm;
|
|
}
|
|
return true;
|
|
}
|
|
catch (DllNotFoundException)
|
|
{
|
|
return false;
|
|
}
|
|
catch (EntryPointNotFoundException)
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Checks if an File is compacted with Btrfs compression
|
|
/// </summary>
|
|
/// <param name="path">Path of the file</param>
|
|
/// <returns>State of the file</returns>
|
|
private bool IsBtrfsCompressedFile(string path)
|
|
{
|
|
return RunWithBtrfsGate(() =>
|
|
{
|
|
try
|
|
{
|
|
bool windowsProc = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
|
|
string linuxPath = windowsProc ? ResolveLinuxPathForWine(path) : path;
|
|
|
|
var task = _fragBatch.IsCompressedAsync(linuxPath, _compactionCts.Token);
|
|
|
|
if (task.Wait(TimeSpan.FromSeconds(5), _compactionCts.Token) && task.IsCompletedSuccessfully)
|
|
return task.Result;
|
|
|
|
_logger.LogTrace("filefrag batch timed out for {file}", linuxPath);
|
|
return false;
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
return false;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogDebug(ex, "filefrag batch check failed for {file}", path);
|
|
return false;
|
|
}
|
|
});
|
|
}
|
|
|
|
/// <summary>
|
|
/// Compress an Btrfs File
|
|
/// </summary>
|
|
/// <param name="path">Path of the decompressed/normal file</param>
|
|
/// <returns>Compessing state</returns>
|
|
private bool BtrfsCompressFile(string path)
|
|
{
|
|
return RunWithBtrfsGate(() => {
|
|
try
|
|
{
|
|
var (winPath, linuxPath) = ResolvePathsForBtrfs(path);
|
|
|
|
if (IsBtrfsCompressedFile(linuxPath))
|
|
{
|
|
_logger.LogTrace("Already Btrfs compressed: {file} (linux={linux})", winPath, linuxPath);
|
|
return true;
|
|
}
|
|
|
|
if (!ProbeFileReadableForBtrfs(winPath, linuxPath))
|
|
{
|
|
_logger.LogTrace("Probe failed; cannot open file for compress: {file} (linux={linux})", winPath, linuxPath);
|
|
return false;
|
|
}
|
|
|
|
(bool ok, string stdout, string stderr, int code) =
|
|
_isWindows
|
|
? RunProcessShell($"btrfs filesystem defragment -clzo -- {QuoteSingle(linuxPath)}")
|
|
: RunProcessDirect("btrfs", ["filesystem", "defragment", "-clzo", "--", linuxPath]);
|
|
|
|
if (!ok)
|
|
{
|
|
_logger.LogWarning("btrfs defragment failed for {file} (linux={linux}) exit {code}: {stderr}",
|
|
winPath, linuxPath, code, stderr);
|
|
return false;
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(stdout))
|
|
_logger.LogTrace("btrfs output for {file}: {out}", winPath, stdout.Trim());
|
|
|
|
_logger.LogInformation("Compressed btrfs file successfully: {file} (linux={linux})", winPath, linuxPath);
|
|
return true;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Error running btrfs defragment for {file}", path);
|
|
return false;
|
|
}
|
|
});
|
|
}
|
|
|
|
/// <summary>
|
|
/// Attempt opening file stream for WOF functions
|
|
/// </summary>
|
|
/// <param name="path">File that has to be accessed</param>
|
|
/// <param name="access">Permissions for the file</param>
|
|
/// <param name="body">Access of the file stream for the WOF function to handle.</param>
|
|
/// <returns>State of the attempt for the file</returns>
|
|
private bool WithFileHandleForWOF(string path, FileAccess access, Func<SafeFileHandle, bool> body)
|
|
{
|
|
const FileShare share = FileShare.ReadWrite | FileShare.Delete;
|
|
|
|
for (int attempt = 0; attempt < _maxRetries; attempt++)
|
|
{
|
|
try
|
|
{
|
|
using var fs = new FileStream(path, FileMode.Open, access, share);
|
|
|
|
var handle = fs.SafeFileHandle;
|
|
if (handle.IsInvalid)
|
|
{
|
|
_logger.LogWarning("Invalid file handle for {file}", path);
|
|
return false;
|
|
}
|
|
|
|
return body(handle);
|
|
}
|
|
catch (IOException ex)
|
|
{
|
|
if (attempt == _maxRetries - 1)
|
|
{
|
|
_logger.LogWarning(ex, "File still in use after {attempts} attempts, skipping {file}", _maxRetries, path);
|
|
return false;
|
|
}
|
|
|
|
int delay = 150 * (attempt + 1);
|
|
_logger.LogTrace(ex, "File busy, retrying in {delay}ms for {file}", delay, path);
|
|
Thread.Sleep(delay);
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Runs an nonshell process meant for Linux enviroments
|
|
/// </summary>
|
|
/// <param name="fileName">File that has to be excuted</param>
|
|
/// <param name="args">Arguments meant for the file/command</param>
|
|
/// <param name="workingDir">Working directory used to execute the file with/without arguments</param>
|
|
/// <param name="timeoutMs">Timeout timer for the process</param>
|
|
/// <returns>State of the process, output of the process and error with exit code</returns>
|
|
private (bool ok, string stdout, string stderr, int exitCode) RunProcessDirect(string fileName, IEnumerable<string> args, string? workingDir = null, int timeoutMs = 60000)
|
|
{
|
|
var psi = new ProcessStartInfo(fileName)
|
|
{
|
|
RedirectStandardOutput = true,
|
|
RedirectStandardError = true,
|
|
UseShellExecute = false,
|
|
CreateNoWindow = true
|
|
};
|
|
if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir;
|
|
foreach (var a in args) psi.ArgumentList.Add(a);
|
|
EnsureUnixPathEnv(psi);
|
|
|
|
using var proc = Process.Start(psi);
|
|
if (proc is null) return (false, "", "failed to start process", -1);
|
|
|
|
var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token);
|
|
if (!success)
|
|
{
|
|
return (false, so2, se2, -1);
|
|
}
|
|
|
|
int code;
|
|
try { code = proc.ExitCode; } catch { code = -1; }
|
|
return (code == 0, so2, se2, code);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Runs an shell using '/bin/bash'/ command meant for Linux/Wine enviroments
|
|
/// </summary>
|
|
/// <param name="command">Command that has to be excuted</param>
|
|
/// <param name="timeoutMs">Timeout timer for the process</param>
|
|
/// <returns>State of the process, output of the process and error with exit code</returns>
|
|
private (bool ok, string stdout, string stderr, int exitCode) RunProcessShell(string command, string? workingDir = null, int timeoutMs = 60000)
|
|
{
|
|
|
|
var psi = new ProcessStartInfo("/bin/bash")
|
|
{
|
|
RedirectStandardOutput = true,
|
|
RedirectStandardError = true,
|
|
UseShellExecute = false,
|
|
CreateNoWindow = true
|
|
};
|
|
if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir;
|
|
|
|
// Use a Login shell so PATH includes /usr/sbin etc. AKA -lc for login shell
|
|
psi.ArgumentList.Add("-lc");
|
|
psi.ArgumentList.Add(QuoteDouble(command));
|
|
EnsureUnixPathEnv(psi);
|
|
|
|
using var proc = Process.Start(psi);
|
|
if (proc is null) return (false, "", "failed to start /bin/bash", -1);
|
|
|
|
var (success, so2, se2) = CheckProcessResult(proc, timeoutMs, _compactionCts.Token);
|
|
if (!success)
|
|
{
|
|
return (false, so2, se2, -1);
|
|
}
|
|
|
|
int code;
|
|
try { code = proc.ExitCode; } catch { code = -1; }
|
|
return (code == 0, so2, se2, code);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Checking the process result for shell or direct processes
|
|
/// </summary>
|
|
/// <param name="proc">Process</param>
|
|
/// <param name="timeoutMs">How long when timeout is gotten</param>
|
|
/// <param name="token">Cancellation Token</param>
|
|
/// <returns>Multiple variables</returns>
|
|
private (bool success, string testy, string testi) CheckProcessResult(Process proc, int timeoutMs, CancellationToken token)
|
|
{
|
|
var outTask = proc.StandardOutput.ReadToEndAsync(token);
|
|
var errTask = proc.StandardError.ReadToEndAsync(token);
|
|
var bothTasks = Task.WhenAll(outTask, errTask);
|
|
|
|
//On wine, we dont wanna use waitforexit as it will be always broken and giving an error.
|
|
if (_dalamudUtilService.IsWine)
|
|
{
|
|
var finished = Task.WhenAny(bothTasks, Task.Delay(timeoutMs, token)).GetAwaiter().GetResult();
|
|
if (finished != bothTasks)
|
|
{
|
|
try
|
|
{
|
|
proc.Kill(entireProcessTree: true);
|
|
Task.WaitAll([outTask, errTask], 1000, token);
|
|
}
|
|
catch
|
|
{
|
|
// ignore this
|
|
}
|
|
var so = outTask.IsCompleted ? outTask.Result : "";
|
|
var se = errTask.IsCompleted ? errTask.Result : "timeout";
|
|
return (false, so, se);
|
|
}
|
|
|
|
var stderr = errTask.Result;
|
|
var ok = string.IsNullOrWhiteSpace(stderr);
|
|
return (ok, outTask.Result, stderr);
|
|
}
|
|
|
|
// On linux, we can use it as we please
|
|
if (!proc.WaitForExit(timeoutMs))
|
|
{
|
|
try
|
|
{
|
|
proc.Kill(entireProcessTree: true);
|
|
Task.WaitAll([outTask, errTask], 1000, token);
|
|
}
|
|
catch
|
|
{
|
|
// ignore this
|
|
}
|
|
return (false, outTask.IsCompleted ? outTask.Result : "", "timeout");
|
|
}
|
|
|
|
Task.WaitAll(outTask, errTask);
|
|
return (true, outTask.Result, errTask.Result);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Enqueues the compaction/decompation of an filepath.
|
|
/// </summary>
|
|
/// <param name="filePath">Filepath that will be enqueued</param>
|
|
private void EnqueueCompaction(string filePath)
|
|
{
|
|
// Safe-checks
|
|
if (string.IsNullOrWhiteSpace(filePath))
|
|
return;
|
|
|
|
if (!_lightlessConfigService.Current.UseCompactor)
|
|
return;
|
|
|
|
if (!File.Exists(filePath))
|
|
return;
|
|
|
|
if (!_pendingCompactions.TryAdd(filePath, 0))
|
|
return;
|
|
|
|
bool enqueued = false;
|
|
try
|
|
{
|
|
bool isWine = _dalamudUtilService?.IsWine ?? false;
|
|
var fsType = GetFilesystemType(filePath, isWine);
|
|
|
|
// If under Wine, we should skip NTFS because its not Windows but might return NTFS.
|
|
if (fsType == FilesystemType.NTFS && isWine)
|
|
{
|
|
_logger.LogTrace("Skip enqueue (NTFS under Wine) {file}", filePath);
|
|
return;
|
|
}
|
|
|
|
// Unknown file system should be skipped.
|
|
if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs)
|
|
{
|
|
_logger.LogTrace("Skip enqueue (unsupported fs) {fs} {file}", fsType, filePath);
|
|
return;
|
|
}
|
|
|
|
// Channel got closed, skip enqueue on file
|
|
if (!_compactionQueue.Writer.TryWrite(filePath))
|
|
{
|
|
_logger.LogTrace("Skip enqueue: compaction channel is/got closed {file}", filePath);
|
|
return;
|
|
}
|
|
|
|
enqueued = true;
|
|
_logger.LogTrace("Queued compaction for {file} (fs={fs})", filePath, fsType);
|
|
}
|
|
finally
|
|
{
|
|
if (!enqueued)
|
|
_pendingCompactions.TryRemove(filePath, out _);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Process the queue with, meant for a worker/thread
|
|
/// </summary>
|
|
/// <param name="token">Cancellation token for the worker whenever it needs to be stopped</param>
|
|
private async Task ProcessQueueWorkerAsync(CancellationToken token)
|
|
{
|
|
try
|
|
{
|
|
while (await _compactionQueue.Reader.WaitToReadAsync(token).ConfigureAwait(false))
|
|
{
|
|
while (_compactionQueue.Reader.TryRead(out var filePath))
|
|
{
|
|
try
|
|
{
|
|
token.ThrowIfCancellationRequested();
|
|
await _globalGate.WaitAsync(token).ConfigureAwait(false);
|
|
|
|
try
|
|
{
|
|
if (_lightlessConfigService.Current.UseCompactor && File.Exists(filePath))
|
|
CompactFile(filePath);
|
|
}
|
|
finally
|
|
{
|
|
_globalGate.Release();
|
|
}
|
|
}
|
|
catch (OperationCanceledException) { return; }
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Error compacting file {file}", filePath);
|
|
}
|
|
finally
|
|
{
|
|
_pendingCompactions.TryRemove(filePath, out _);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
// Shutting down worker, this exception is expected
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Resolves linux path from wine pathing
|
|
/// </summary>
|
|
/// <param name="windowsPath">Windows path given from Wine</param>
|
|
/// <returns>Linux path to be used in Linux</returns>
|
|
private string ResolveLinuxPathForWine(string windowsPath)
|
|
{
|
|
var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(windowsPath)}", null, 5000);
|
|
if (ok && !string.IsNullOrWhiteSpace(outp)) return outp.Trim();
|
|
return ToLinuxPathIfWine(windowsPath, isWine: true);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Ensures the Unix pathing to be included into the process start
|
|
/// </summary>
|
|
/// <param name="psi">Process</param>
|
|
private static void EnsureUnixPathEnv(ProcessStartInfo psi)
|
|
{
|
|
if (!psi.Environment.TryGetValue("PATH", out var p) || string.IsNullOrWhiteSpace(p))
|
|
psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin";
|
|
else if (!p.Contains("/usr/sbin", StringComparison.Ordinal))
|
|
psi.Environment["PATH"] = "/usr/sbin:/usr/bin:/bin:" + p;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Resolves paths for Btrfs to be used on wine or linux and windows in case
|
|
/// </summary>
|
|
/// <param name="path">Path given t</param>
|
|
/// <returns></returns>
|
|
private (string windowsPath, string linuxPath) ResolvePathsForBtrfs(string path)
|
|
{
|
|
if (!_isWindows)
|
|
return (path, path);
|
|
|
|
var (ok, outp, _, _) = RunProcessShell($"winepath -u {QuoteSingle(path)}", workingDir: null, 5000);
|
|
var linux = (ok && !string.IsNullOrWhiteSpace(outp)) ? outp.Trim() : ToLinuxPathIfWine(path, isWine: true);
|
|
|
|
return (path, linux);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Probes file if its readable to be used
|
|
/// </summary>
|
|
/// <param name="winePath">Windows path</param>
|
|
/// <param name="linuxPath">Linux path</param>
|
|
/// <returns>Succesfully probed or not</returns>
|
|
private bool ProbeFileReadableForBtrfs(string winePath, string linuxPath)
|
|
{
|
|
try
|
|
{
|
|
if (_isWindows)
|
|
{
|
|
using var _ = new FileStream(winePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
|
|
}
|
|
else
|
|
{
|
|
using var _ = new FileStream(linuxPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
|
|
}
|
|
return true;
|
|
}
|
|
catch { return false; }
|
|
}
|
|
|
|
/// <summary>
|
|
/// Running functions into the Btrfs Gate/Threading.
|
|
/// </summary>
|
|
/// <typeparam name="T">Type of the function that wants to be run inside Btrfs Gate</typeparam>
|
|
/// <param name="body">Body of the function</param>
|
|
/// <returns>Task</returns>
|
|
private T RunWithBtrfsGate<T>(Func<T> body)
|
|
{
|
|
bool acquired = false;
|
|
try
|
|
{
|
|
_btrfsGate.Wait(_compactionCts.Token);
|
|
acquired = true;
|
|
return body();
|
|
}
|
|
finally
|
|
{
|
|
if (acquired) _btrfsGate.Release();
|
|
}
|
|
}
|
|
|
|
|
|
[DllImport("kernel32.dll", SetLastError = true)]
|
|
private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped);
|
|
|
|
[DllImport("kernel32.dll")]
|
|
private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh);
|
|
|
|
[DllImport("WofUtil.dll")]
|
|
private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength);
|
|
|
|
[DllImport("WofUtil.dll", SetLastError = true)]
|
|
private static extern int WofSetFileDataLocation(SafeFileHandle FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length);
|
|
|
|
private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'";
|
|
|
|
private static string QuoteDouble(string s) => "\"" + s.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal).Replace("$", "\\$", StringComparison.Ordinal).Replace("`", "\\`", StringComparison.Ordinal) + "\"";
|
|
|
|
public void Dispose()
|
|
{
|
|
_fragBatch?.Dispose();
|
|
_compactionQueue.Writer.TryComplete();
|
|
_compactionCts.Cancel();
|
|
|
|
try
|
|
{
|
|
Task.WaitAll([.. _workers.Where(t => t != null)], TimeSpan.FromSeconds(5));
|
|
}
|
|
catch
|
|
{
|
|
// Ignore this catch on the dispose
|
|
}
|
|
finally
|
|
{
|
|
_compactionCts.Dispose();
|
|
}
|
|
|
|
GC.SuppressFinalize(this);
|
|
}
|
|
}
|