Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 51 Wiki Activity

Clean-up, added extra checks on linux in cache monitor, documentation added

Browse Source
This commit is contained in:
cake
2025-11-03 18:54:35 +01:00
parent 6e3c60f627
commit d4dca455ba
2 changed files with 260 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
LightlessSync/FileCache/CacheMonitor.cs
View File

@@ -403,57 +403,94 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
public void RecalculateFileCacheSize(CancellationToken token)
{
if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder))
if (string.IsNullOrEmpty(_configService.Current.CacheFolder) ||
!Directory.Exists(_configService.Current.CacheFolder))
{
FileCacheSize = 0;
return;
}
FileCacheSize = -1;
var drive = DriveInfo.GetDrives().FirstOrDefault(d => _configService.Current.CacheFolder.StartsWith(d.Name, StringComparison.Ordinal));
if (drive == null)
{
return;
}
FileCacheSize = -1;
bool isWine = _dalamudUtil?.IsWine ?? false;
try
{
FileCacheDriveFree = drive.AvailableFreeSpace;
var drive = DriveInfo.GetDrives()
.FirstOrDefault(d => _configService.Current.CacheFolder
.StartsWith(d.Name, StringComparison.OrdinalIgnoreCase));
if (drive != null)
FileCacheDriveFree = drive.AvailableFreeSpace;
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder);
Logger.LogWarning(ex, "Could not determine drive size for storage folder {folder}", _configService.Current.CacheFolder);
}
var files = Directory.EnumerateFiles(_configService.Current.CacheFolder).Select(f => new FileInfo(f))
.OrderBy(f => f.LastAccessTime).ToList();
FileCacheSize = files
.Sum(f =>
{
token.ThrowIfCancellationRequested();
var files = Directory.EnumerateFiles(_configService.Current.CacheFolder)
.Select(f => new FileInfo(f))
.OrderBy(f => f.LastAccessTime)
.ToList();
try
long totalSize = 0;
foreach (var f in files)
{
token.ThrowIfCancellationRequested();
try
{
long size = 0;
if (!isWine)
{
return _fileCompactor.GetFileSizeOnDisk(f);
try
{
size = _fileCompactor.GetFileSizeOnDisk(f);
}
catch (Exception ex)
{
Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName);
size = f.Length;
}
}
catch
else
{
return 0;
size = f.Length;
}
});
totalSize += size;
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Error getting size for {file}", f.FullName);
}
}
FileCacheSize = totalSize;
var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d);
if (FileCacheSize < maxCacheInBytes) return;
if (FileCacheSize < maxCacheInBytes)
return;
var maxCacheBuffer = maxCacheInBytes * 0.05d;
while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer)
while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer && files.Count > 0)
{
var oldestFile = files[0];
FileCacheSize -= _fileCompactor.GetFileSizeOnDisk(oldestFile);
File.Delete(oldestFile.FullName);
files.Remove(oldestFile);
try
{
long fileSize = oldestFile.Length;
File.Delete(oldestFile.FullName);
FileCacheSize -= fileSize;
}
catch (Exception ex)
{
Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullName);
}
files.RemoveAt(0);
}
}

372
LightlessSync/FileCache/FileCompactor.cs
View File

@@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging;
using Microsoft.Win32.SafeHandles;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
using System.Threading.Channels;
using static LightlessSync.Utils.FileSystemHelper;
@@ -14,6 +15,7 @@ public sealed class FileCompactor : IDisposable
{
public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U;
public const ulong WOF_PROVIDER_FILE = 2UL;
public const int _maxRetries = 3;
private readonly ConcurrentDictionary<string, byte> _pendingCompactions;
private readonly ILogger<FileCompactor> _logger;
@@ -29,6 +31,14 @@ public sealed class FileCompactor : IDisposable
Algorithm = (int)CompressionAlgorithm.XPRESS8K,
Flags = 0
};
[StructLayout(LayoutKind.Sequential, Pack = 1)]
private struct WOF_FILE_COMPRESSION_INFO_V1
{
public int Algorithm;
public ulong Flags;
}
private enum CompressionAlgorithm
{
NO_COMPRESSION = -2,
@@ -58,6 +68,10 @@ public sealed class FileCompactor : IDisposable
public bool MassCompactRunning { get; private set; }
public string Progress { get; private set; } = string.Empty;
/// <summary>
/// Compact the storage of the Cache Folder
/// </summary>
/// <param name="compress">Used to check if files needs to be compressed</param>
public void CompactStorage(bool compress)
{
MassCompactRunning = true;
@@ -74,6 +88,7 @@ public sealed class FileCompactor : IDisposable
try
{
// Compress or decompress files
if (compress)
CompactFile(file);
else
@@ -135,25 +150,19 @@ public sealed class FileCompactor : IDisposable
{
try
{
var realPath = fileInfo.FullName.Replace("\"", "\\\"", StringComparison.Ordinal);
var psi = new ProcessStartInfo("stat", $"-c %b \"{realPath}\"")
{
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
bool isWine = _dalamudUtilService?.IsWine ?? false;
string realPath = isWine ? ToLinuxPathIfWine(fileInfo.FullName, isWine) : fileInfo.FullName;
using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat");
var outp = proc.StandardOutput.ReadToEnd();
var err = proc.StandardError.ReadToEnd();
proc.WaitForExit();
var fileName = "stat";
var arguments = $"-c %b \"{realPath}\"";
if (proc.ExitCode != 0)
throw new InvalidOperationException($"stat failed: {err}");
(bool processControl, bool success) = StartProcessInfo(realPath, fileName, arguments, out Process? proc, out string stdout);
if (!long.TryParse(outp.Trim(), out var blocks))
throw new InvalidOperationException($"invalid stat output: {outp}");
if (!processControl && !success)
throw new InvalidOperationException($"stat failed: {proc}");
if (!long.TryParse(stdout.Trim(), out var blocks))
throw new InvalidOperationException($"invalid stat output: {stdout}");
// st_blocks are always 512-byte on Linux enviroment.
return blocks * 512L;
@@ -287,57 +296,57 @@ public sealed class FileCompactor : IDisposable
/// <returns>Decompessing state</returns>
private bool DecompressBtrfsFile(string path)
{
var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read);
try
{
var opts = GetMountOptionsForPath(path);
if (opts.Contains("compress", StringComparison.OrdinalIgnoreCase))
bool isWine = _dalamudUtilService?.IsWine ?? false;
string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path;
var mountOptions = GetMountOptionsForPath(realPath);
if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase))
{
_logger.LogWarning("Cannot safely decompress {file}: mount options include compression ({opts})", path, opts);
_logger.LogWarning(
"Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " +
"Remount with 'compress=no' before running decompression.",
realPath, mountOptions);
return false;
}
string realPath = ToLinuxPathIfWine(path, _dalamudUtilService.IsWine);
var psi = new ProcessStartInfo
if (!IsBtrfsCompressedFile(realPath))
{
FileName = _dalamudUtilService.IsWine ? "/bin/bash" : "btrfs",
Arguments = _dalamudUtilService.IsWine
? $"-c \"btrfs filesystem defragment -- '{EscapeSingle(realPath)}'\""
: $"filesystem defragment -- \"{realPath}\"",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true,
WorkingDirectory = "/"
};
using var proc = Process.Start(psi);
if (proc == null)
{
_logger.LogWarning("Failed to start btrfs defragment for {file}", path);
return false;
_logger.LogTrace("File {file} is not compressed, skipping decompression.", realPath);
return true;
}
var stdout = proc.StandardOutput.ReadToEnd();
var stderr = proc.StandardError.ReadToEnd();
proc.WaitForExit();
(bool flowControl, bool value) = FileStreamOpening(realPath, ref fs);
if (proc.ExitCode != 0)
if (!flowControl)
{
_logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr);
return false;
return value;
}
string fileName = isWine ? "/bin/bash" : "btrfs";
string command = isWine ? $"-c \"filesystem defragment -- \"{realPath}\"\"" : $"filesystem defragment -- \"{realPath}\"";
(bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout);
if (!processControl && !success)
{
return value;
}
if (!string.IsNullOrWhiteSpace(stdout))
_logger.LogTrace("btrfs defragment output {file}: {out}", path, stdout.Trim());
_logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim());
_logger.LogInformation("Btrfs rewritten uncompressed: {file}", path);
_logger.LogInformation("Decompressed btrfs file successfully: {file}", realPath);
return true;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Btrfs decompress error {file}", path);
_logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path);
return false;
}
return true;
}
/// <summary>
@@ -379,23 +388,25 @@ public sealed class FileCompactor : IDisposable
return true;
}
private static string EscapeSingle(string p) => p.Replace("'", "'\\'", StringComparison.Ordinal);
private static string ToLinuxPathIfWine(string path, bool isWine)
/// <summary>
/// Converts to Linux Path if its using Wine (diferent pathing system in Wine)
/// </summary>
/// <param name="path">Path that has to be converted</param>
/// <param name="isWine">Extra check if using the wine enviroment</param>
/// <returns>Converted path to be used in Linux</returns>
private string ToLinuxPathIfWine(string path, bool isWine)
{
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
return path;
if (!IsProbablyWine() && !isWine)
return path;
string realPath = path;
string linuxPath = path;
if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase))
realPath = "/" + path[3..].Replace('\\', '/');
linuxPath = "/" + path[3..].Replace('\\', '/');
else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase))
realPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/');
linuxPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/');
return realPath;
_logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", linuxPath);
return linuxPath;
}
/// <summary>
@@ -414,27 +425,11 @@ public sealed class FileCompactor : IDisposable
Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: false);
ulong length = (ulong)size;
const int maxRetries = 3;
(bool flowControl, bool value) = FileStreamOpening(path, ref fs);
for (int attempt = 0; attempt < maxRetries; attempt++)
if (!flowControl)
{
try
{
fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read);
break;
}
catch (IOException)
{
if (attempt == maxRetries - 1)
{
_logger.LogWarning("File still in use after {attempts} attempts, skipping compression for {file}", maxRetries, path);
return false;
}
int delay = 150 * (attempt + 1);
_logger.LogTrace("File in use, retrying in {delay}ms for {file}", delay, path);
Thread.Sleep(delay);
}
return value;
}
if (fs == null)
@@ -462,14 +457,14 @@ public sealed class FileCompactor : IDisposable
return true;
}
catch (DllNotFoundException)
catch (DllNotFoundException ex)
{
_logger.LogTrace("WofUtil.dll not available; skipping NTFS compaction for {file}", path);
_logger.LogTrace(ex, "WofUtil.dll not available, this DLL is needed for compression; skipping NTFS compaction for {file}", path);
return false;
}
catch (EntryPointNotFoundException)
catch (EntryPointNotFoundException ex)
{
_logger.LogTrace("WOF entrypoint missing (Wine/older OS); skipping NTFS compaction for {file}", path);
_logger.LogTrace(ex, "WOF entrypoint missing (Wine/older OS); skipping NTFS compaction for {file}", path);
return false;
}
catch (Exception ex)
@@ -527,37 +522,24 @@ public sealed class FileCompactor : IDisposable
bool isWine = _dalamudUtilService?.IsWine ?? false;
string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path;
var psi = new ProcessStartInfo
{
FileName = isWine ? "/bin/bash" : "filefrag",
Arguments = isWine
? $"-c \"filefrag -v '{EscapeSingle(realPath)}'\""
: $"-v \"{realPath}\"",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true,
WorkingDirectory = "/"
};
var fi = new FileInfo(realPath);
using var proc = Process.Start(psi);
if (proc == null)
if (fi == null)
{
_logger.LogWarning("Failed to start filefrag for {file}", path);
_logger.LogWarning("Failed to open {file} for checking on compression; skipping", realPath);
return false;
}
string stdout = proc.StandardOutput.ReadToEnd();
string stderr = proc.StandardError.ReadToEnd();
proc.WaitForExit();
if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr))
string fileName = isWine ? "/bin/bash" : "filefrag";
string command = isWine ? $"-c \"filefrag -v '{EscapeSingle(realPath)}'\"" : $"-v \"{realPath}\"";
(bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout);
if (!processControl && !success)
{
_logger.LogTrace("filefrag exited with code {code}: {stderr}", proc.ExitCode, stderr);
return success;
}
bool compressed = stdout.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase);
_logger.LogTrace("Btrfs compression check for {file}: {compressed}", path, compressed);
_logger.LogTrace("Btrfs compression check for {file}: {compressed}", realPath, compressed);
return compressed;
}
catch (Exception ex)
@@ -574,89 +556,56 @@ public sealed class FileCompactor : IDisposable
/// <returns>Compessing state</returns>
private bool BtrfsCompressFile(string path)
{
FileStream? fs = null;
try
{
bool isWine = _dalamudUtilService?.IsWine ?? false;
string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path;
if (isWine && IsProbablyWine())
var fi = new FileInfo(realPath);
if (fi == null)
{
if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase))
{
realPath = "/" + path[3..].Replace('\\', '/');
}
else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase))
{
string linuxHome = Environment.GetEnvironmentVariable("HOME") ?? "/home";
realPath = Path.Combine(linuxHome, path[3..].Replace('\\', '/')).Replace('\\', '/');
}
_logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", realPath);
}
const int maxRetries = 3;
for (int attempt = 0; attempt < maxRetries; attempt++)
{
try
{
using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
break;
}
catch (IOException)
{
if (attempt == maxRetries - 1)
{
_logger.LogWarning("File still in use after {attempts} attempts; skipping btrfs compression for {file}", maxRetries, path);
return false;
}
int delay = 150 * (attempt + 1);
_logger.LogTrace("File busy, retrying in {delay}ms for {file}", delay, path);
Thread.Sleep(delay);
}
}
string command = $"btrfs filesystem defragment -czstd -- \"{realPath}\"";
var psi = new ProcessStartInfo
{
FileName = isWine ? "/bin/bash" : "btrfs",
Arguments = isWine ? $"-c \"{command}\"" : $"filesystem defragment -czstd -- \"{realPath}\"",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true,
WorkingDirectory = "/"
};
using var proc = Process.Start(psi);
if (proc == null)
{
_logger.LogWarning("Failed to start btrfs defragment for compression of {file}", path);
_logger.LogWarning("Failed to open {file} for compression; skipping", realPath);
return false;
}
string stdout = proc.StandardOutput.ReadToEnd();
string stderr = proc.StandardError.ReadToEnd();
try
{
proc.WaitForExit();
}
catch (Exception ex)
//Skipping small files to make compression a bit faster, its not that effective on small files.
int blockSize = GetBlockSizeForPath(realPath, _logger, isWine);
if (fi.Length < Math.Max(blockSize * 2, 128 * 1024))
{
_logger.LogTrace(ex, "Process.WaitForExit threw under Wine for {file}", path);
_logger.LogTrace("Skipping Btrfs compression for small file {file} ({size} bytes)", realPath, fi.Length);
return true;
}
if (proc.ExitCode != 0)
if (IsBtrfsCompressedFile(realPath))
{
_logger.LogWarning("btrfs defragment failed for {file}: {stderr}", path, stderr);
return false;
_logger.LogTrace("File {file} already compressed (Btrfs), skipping file", realPath);
return true;
}
(bool flowControl, bool value) = FileStreamOpening(realPath, ref fs);
if (!flowControl)
{
return value;
}
string fileName = isWine ? "/bin/bash" : "btrfs";
string command = isWine ? $"-c \"btrfs filesystem defragment -czstd:1 -- \"{realPath}\"\"" : $"btrfs filesystem defragment -czstd:1 -- \"{realPath}\"";
(bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout);
if (!processControl && !success)
{
return value;
}
if (!string.IsNullOrWhiteSpace(stdout))
_logger.LogTrace("btrfs defragment output for {file}: {stdout}", path, stdout.Trim());
_logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim());
_logger.LogInformation("Compressed btrfs file successfully: {file}", realPath);
_logger.LogInformation("Compressed btrfs file successfully: {file}", path);
return true;
}
catch (Exception ex)
@@ -666,6 +615,84 @@ public sealed class FileCompactor : IDisposable
}
}
/// <summary>
/// Trying opening file stream in certain amount of tries.
/// </summary>
/// <param name="path">Path where the file is located</param>
/// <param name="fs">Filestream used for the function</param>
/// <returns>State of the filestream opening</returns>
private (bool flowControl, bool value) FileStreamOpening(string path, ref FileStream? fs)
{
for (int attempt = 0; attempt < _maxRetries; attempt++)
{
try
{
fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read);
break;
}
catch (IOException)
{
if (attempt == _maxRetries - 1)
{
_logger.LogWarning("File still in use after {attempts} attempts, skipping compression for {file}", _maxRetries, path);
return (flowControl: false, value: false);
}
int delay = 150 * (attempt + 1);
_logger.LogTrace("File in use, retrying in {delay}ms for {file}", delay, path);
Thread.Sleep(delay);
}
}
return (flowControl: true, value: default);
}
/// <summary>
/// Starts an process with given Filename and Arguments
/// </summary>
/// <param name="path">Path you want to use for the process (Compression is using these)</param>
/// <param name="fileName">File of the command</param>
/// <param name="arguments">Arguments used for the command</param>
/// <param name="proc">Returns process of the given command</param>
/// <param name="stdout">Returns output of the given command</param>
/// <returns>Returns if the process been done succesfully or not</returns>
private (bool processControl, bool success) StartProcessInfo(string path, string fileName, string arguments, out Process? proc, out string stdout)
{
var psi = new ProcessStartInfo
{
FileName = fileName,
Arguments = arguments,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true,
WorkingDirectory = "/"
};
proc = Process.Start(psi);
if (proc == null)
{
_logger.LogWarning("Failed to start {arguments} for {file}", arguments, path);
stdout = string.Empty;
return (processControl: false, success: false);
}
stdout = proc.StandardOutput.ReadToEnd();
string stderr = proc.StandardError.ReadToEnd();
proc.WaitForExit();
if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr))
{
_logger.LogTrace("{arguments} exited with code {code}: {stderr}", arguments, proc.ExitCode, stderr);
return (processControl: false, success: false);
}
return (processControl: true, success: default);
}
private static string EscapeSingle(string p) => p.Replace("'", "'\\'", StringComparison.Ordinal);
private void EnqueueCompaction(string filePath)
{
if (!_pendingCompactions.TryAdd(filePath, 0))
@@ -735,13 +762,6 @@ public sealed class FileCompactor : IDisposable
}
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
private struct WOF_FILE_COMPRESSION_INFO_V1
{
public int Algorithm;
public ulong Flags;
}
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped);