Implemented compactor to work on BTRFS, redid cache a bit for better function on linux. Removed error for websockets, it will be forced on wine again.
This commit is contained in:
@@ -115,6 +115,8 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
|
||||
|
||||
public bool StorageisNTFS { get; private set; } = false;
|
||||
|
||||
public bool StorageIsBtrfs { get ; private set; } = false;
|
||||
|
||||
public void StartLightlessWatcher(string? lightlessPath)
|
||||
{
|
||||
LightlessWatcher?.Dispose();
|
||||
@@ -124,10 +126,19 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
|
||||
Logger.LogWarning("Lightless file path is not set, cannot start the FSW for Lightless.");
|
||||
return;
|
||||
}
|
||||
var fsType = FileSystemHelper.GetFilesystemType(_configService.Current.CacheFolder);
|
||||
|
||||
DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
|
||||
StorageisNTFS = string.Equals("NTFS", di.DriveFormat, StringComparison.OrdinalIgnoreCase);
|
||||
Logger.LogInformation("Lightless Storage is on NTFS drive: {isNtfs}", StorageisNTFS);
|
||||
if (fsType == FileSystemHelper.FilesystemType.NTFS)
|
||||
{
|
||||
StorageisNTFS = true;
|
||||
Logger.LogInformation("Lightless Storage is on NTFS drive: {isNtfs}", StorageisNTFS);
|
||||
}
|
||||
|
||||
if (fsType == FileSystemHelper.FilesystemType.Btrfs)
|
||||
{
|
||||
StorageIsBtrfs = true;
|
||||
Logger.LogInformation("Lightless Storage is on BTRFS drive: {isNtfs}", StorageIsBtrfs);
|
||||
}
|
||||
|
||||
Logger.LogDebug("Initializing Lightless FSW on {path}", lightlessPath);
|
||||
LightlessWatcher = new()
|
||||
|
||||
@@ -203,42 +203,72 @@ public sealed class FileCacheManager : IHostedService
|
||||
return output;
|
||||
}
|
||||
|
||||
public Task<List<FileCacheEntity>> ValidateLocalIntegrity(IProgress<(int, int, FileCacheEntity)> progress, CancellationToken cancellationToken)
|
||||
public async Task<List<FileCacheEntity>> ValidateLocalIntegrity(IProgress<(int completed, int total, FileCacheEntity current)> progress, CancellationToken cancellationToken)
|
||||
{
|
||||
_lightlessMediator.Publish(new HaltScanMessage(nameof(ValidateLocalIntegrity)));
|
||||
_logger.LogInformation("Validating local storage");
|
||||
var cacheEntries = _fileCaches.Values.SelectMany(v => v.Values.Where(e => e != null)).Where(v => v.IsCacheEntry).ToList();
|
||||
List<FileCacheEntity> brokenEntities = [];
|
||||
int i = 0;
|
||||
foreach (var fileCache in cacheEntries)
|
||||
|
||||
var cacheEntries = _fileCaches.Values
|
||||
.SelectMany(v => v.Values)
|
||||
.Where(v => v.IsCacheEntry)
|
||||
.ToList();
|
||||
|
||||
int total = cacheEntries.Count;
|
||||
int processed = 0;
|
||||
var brokenEntities = new ConcurrentBag<FileCacheEntity>();
|
||||
|
||||
_logger.LogInformation("Checking {count} cache entries...", total);
|
||||
|
||||
await Parallel.ForEachAsync(cacheEntries, new ParallelOptions
|
||||
{
|
||||
MaxDegreeOfParallelism = Environment.ProcessorCount,
|
||||
CancellationToken = cancellationToken
|
||||
},
|
||||
async (fileCache, token) =>
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested) break;
|
||||
|
||||
_logger.LogInformation("Validating {file}", fileCache.ResolvedFilepath);
|
||||
|
||||
progress.Report((i, cacheEntries.Count, fileCache));
|
||||
i++;
|
||||
if (!File.Exists(fileCache.ResolvedFilepath))
|
||||
{
|
||||
brokenEntities.Add(fileCache);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var computedHash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
|
||||
int current = Interlocked.Increment(ref processed);
|
||||
if (current % 10 == 0)
|
||||
progress.Report((current, total, fileCache));
|
||||
|
||||
if (!File.Exists(fileCache.ResolvedFilepath))
|
||||
{
|
||||
brokenEntities.Add(fileCache);
|
||||
return;
|
||||
}
|
||||
|
||||
string computedHash;
|
||||
try
|
||||
{
|
||||
computedHash = await Crypto.GetFileHashAsync(fileCache.ResolvedFilepath, token).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error hashing {file}", fileCache.ResolvedFilepath);
|
||||
brokenEntities.Add(fileCache);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal))
|
||||
{
|
||||
_logger.LogInformation("Failed to validate {file}, got hash {computedHash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
|
||||
_logger.LogInformation(
|
||||
"Hash mismatch: {file} (got {computedHash}, expected {expected})",
|
||||
fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
|
||||
|
||||
brokenEntities.Add(fileCache);
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
_logger.LogWarning(e, "Error during validation of {file}", fileCache.ResolvedFilepath);
|
||||
_logger.LogError("Validation got cancelled for {file}", fileCache.ResolvedFilepath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Unexpected error validating {file}", fileCache.ResolvedFilepath);
|
||||
brokenEntities.Add(fileCache);
|
||||
}
|
||||
}
|
||||
}).ConfigureAwait(false);
|
||||
|
||||
foreach (var brokenEntity in brokenEntities)
|
||||
{
|
||||
@@ -250,12 +280,14 @@ public sealed class FileCacheManager : IHostedService
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Could not delete {file}", brokenEntity.ResolvedFilepath);
|
||||
_logger.LogWarning(ex, "Failed to delete invalid cache file {file}", brokenEntity.ResolvedFilepath);
|
||||
}
|
||||
}
|
||||
|
||||
_lightlessMediator.Publish(new ResumeScanMessage(nameof(ValidateLocalIntegrity)));
|
||||
return Task.FromResult(brokenEntities);
|
||||
_logger.LogInformation("Validation complete. Found {count} invalid entries.", brokenEntities.Count);
|
||||
|
||||
return [.. brokenEntities];
|
||||
}
|
||||
|
||||
public string GetCacheFilePath(string hash, string extension)
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
using LightlessSync.LightlessConfiguration;
|
||||
using LightlessSync.Services;
|
||||
using LightlessSync.Utils;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Channels;
|
||||
using System.Threading.Tasks;
|
||||
using static LightlessSync.Utils.FileSystemHelper;
|
||||
|
||||
namespace LightlessSync.FileCache;
|
||||
|
||||
@@ -87,25 +91,51 @@ public sealed class FileCompactor : IDisposable
|
||||
|
||||
public long GetFileSizeOnDisk(FileInfo fileInfo, bool? isNTFS = null)
|
||||
{
|
||||
bool ntfs = isNTFS ?? string.Equals(new DriveInfo(fileInfo.Directory!.Root.FullName).DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase);
|
||||
var fsType = FileSystemHelper.GetFilesystemType(fileInfo.FullName);
|
||||
|
||||
if (_dalamudUtilService.IsWine || !ntfs) return fileInfo.Length;
|
||||
bool ntfs = isNTFS ?? fsType == FileSystemHelper.FilesystemType.NTFS;
|
||||
|
||||
var clusterSize = GetClusterSize(fileInfo);
|
||||
if (clusterSize == -1) return fileInfo.Length;
|
||||
var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize);
|
||||
var size = (long)hosize << 32 | losize;
|
||||
return ((size + clusterSize - 1) / clusterSize) * clusterSize;
|
||||
if (fsType != FileSystemHelper.FilesystemType.Btrfs && !ntfs)
|
||||
{
|
||||
return fileInfo.Length;
|
||||
}
|
||||
|
||||
if (ntfs && !_dalamudUtilService.IsWine)
|
||||
{
|
||||
var clusterSize = GetClusterSize(fileInfo);
|
||||
if (clusterSize == -1) return fileInfo.Length;
|
||||
var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize);
|
||||
var size = (long)hosize << 32 | losize;
|
||||
return ((size + clusterSize - 1) / clusterSize) * clusterSize;
|
||||
}
|
||||
|
||||
if (fsType == FileSystemHelper.FilesystemType.Btrfs)
|
||||
{
|
||||
try
|
||||
{
|
||||
long blocks = RunStatGetBlocks(fileInfo.FullName);
|
||||
return blocks * 512L;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "Failed to get on-disk size via stat for {file}, falling back to Length", fileInfo.FullName);
|
||||
return fileInfo.Length;
|
||||
}
|
||||
}
|
||||
|
||||
return fileInfo.Length;
|
||||
}
|
||||
|
||||
public async Task WriteAllBytesAsync(string filePath, byte[] decompressedFile, CancellationToken token)
|
||||
{
|
||||
var dir = Path.GetDirectoryName(filePath);
|
||||
if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir))
|
||||
Directory.CreateDirectory(dir);
|
||||
|
||||
await File.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false);
|
||||
|
||||
if (_dalamudUtilService.IsWine || !_lightlessConfigService.Current.UseCompactor)
|
||||
{
|
||||
if (!_lightlessConfigService.Current.UseCompactor)
|
||||
return;
|
||||
}
|
||||
|
||||
EnqueueCompaction(filePath);
|
||||
}
|
||||
@@ -153,56 +183,178 @@ public sealed class FileCompactor : IDisposable
|
||||
|
||||
private void CompactFile(string filePath)
|
||||
{
|
||||
var fs = new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName);
|
||||
bool isNTFS = string.Equals(fs.DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase);
|
||||
if (!isNTFS)
|
||||
var fi = new FileInfo(filePath);
|
||||
if (!fi.Exists)
|
||||
{
|
||||
_logger.LogWarning("Drive for file {file} is not NTFS", filePath);
|
||||
_logger.LogDebug("Skipping compaction for missing file {file}", filePath);
|
||||
return;
|
||||
}
|
||||
|
||||
var fi = new FileInfo(filePath);
|
||||
var fsType = FileSystemHelper.GetFilesystemType(filePath);
|
||||
var oldSize = fi.Length;
|
||||
var clusterSize = GetClusterSize(fi);
|
||||
|
||||
int clusterSize = GetClusterSize(fi);
|
||||
if (oldSize < Math.Max(clusterSize, 8 * 1024))
|
||||
{
|
||||
_logger.LogDebug("File {file} is smaller than cluster size ({size}), ignoring", filePath, clusterSize);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!IsCompactedFile(filePath))
|
||||
// NTFS Compression.
|
||||
if (fsType == FileSystemHelper.FilesystemType.NTFS && !_dalamudUtilService.IsWine)
|
||||
{
|
||||
_logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath);
|
||||
if (!IsWOFCompactedFile(filePath))
|
||||
{
|
||||
_logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath);
|
||||
var success = WOFCompressFile(filePath);
|
||||
|
||||
WOFCompressFile(filePath);
|
||||
if (success)
|
||||
{
|
||||
var newSize = GetFileSizeOnDisk(fi);
|
||||
_logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning("NTFS compression failed or not available for {file}", filePath);
|
||||
}
|
||||
|
||||
var newSize = GetFileSizeOnDisk(fi);
|
||||
|
||||
_logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("File {file} already compressed (NTFS)", filePath);
|
||||
}
|
||||
}
|
||||
else
|
||||
|
||||
// BTRFS Compression
|
||||
if (fsType == FileSystemHelper.FilesystemType.Btrfs)
|
||||
{
|
||||
_logger.LogDebug("File {file} already compressed", filePath);
|
||||
if (!IsBtrfsCompressedFile(filePath))
|
||||
{
|
||||
_logger.LogDebug("Attempting btrfs compression for {file}", filePath);
|
||||
var success = BtrfsCompressFile(filePath);
|
||||
|
||||
if (success)
|
||||
{
|
||||
var newSize = GetFileSizeOnDisk(fi);
|
||||
_logger.LogDebug("Btrfs-compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning("Btrfs compression failed or not available for {file}", filePath);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("File {file} already compressed (Btrfs)", filePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static long RunStatGetBlocks(string path)
|
||||
{
|
||||
var psi = new ProcessStartInfo("stat", $"-c %b \"{path}\"")
|
||||
{
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true
|
||||
};
|
||||
|
||||
using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat process");
|
||||
var outp = proc.StandardOutput.ReadToEnd();
|
||||
var err = proc.StandardError.ReadToEnd();
|
||||
proc.WaitForExit();
|
||||
if (proc.ExitCode != 0)
|
||||
{
|
||||
throw new InvalidOperationException($"stat failed: {err}");
|
||||
}
|
||||
|
||||
if (!long.TryParse(outp.Trim(), out var blocks))
|
||||
{
|
||||
throw new InvalidOperationException($"invalid stat output: {outp}");
|
||||
}
|
||||
|
||||
return blocks;
|
||||
}
|
||||
|
||||
private void DecompressFile(string path)
|
||||
{
|
||||
_logger.LogDebug("Removing compression from {file}", path);
|
||||
try
|
||||
var fsType = FileSystemHelper.GetFilesystemType(path);
|
||||
if (fsType == null) return;
|
||||
|
||||
//NTFS Decompression
|
||||
if (fsType == FileSystemHelper.FilesystemType.NTFS && !_dalamudUtilService.IsWine)
|
||||
{
|
||||
using (var fs = new FileStream(path, FileMode.Open))
|
||||
try
|
||||
{
|
||||
using (var fs = new FileStream(path, FileMode.Open))
|
||||
{
|
||||
#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called
|
||||
var hDevice = fs.SafeFileHandle.DangerousGetHandle();
|
||||
var hDevice = fs.SafeFileHandle.DangerousGetHandle();
|
||||
#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called
|
||||
_ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _);
|
||||
_ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error decompressing file {path}", path);
|
||||
}
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
//BTRFS Decompression
|
||||
if (fsType == FileSystemHelper.FilesystemType.Btrfs)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error decompressing file {path}", path);
|
||||
try
|
||||
{
|
||||
var mountOptions = GetMountOptionsForPath(path);
|
||||
if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " +
|
||||
"Remount with 'compress=no' before running decompression.", path, mountOptions);
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogDebug("Rewriting {file} to remove btrfs compression...", path);
|
||||
|
||||
var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -- \"{path}\"")
|
||||
{
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true
|
||||
};
|
||||
|
||||
using var proc = Process.Start(psi);
|
||||
if (proc == null)
|
||||
{
|
||||
_logger.LogWarning("Failed to start btrfs defragment for decompression of {file}", path);
|
||||
return;
|
||||
}
|
||||
|
||||
var stdout = proc.StandardOutput.ReadToEnd();
|
||||
var stderr = proc.StandardError.ReadToEnd();
|
||||
proc.WaitForExit();
|
||||
|
||||
if (proc.ExitCode != 0)
|
||||
{
|
||||
_logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Log output only in debug mode to avoid clutter
|
||||
if (!string.IsNullOrWhiteSpace(stdout))
|
||||
_logger.LogDebug("btrfs defragment output for {file}: {out}", path, stdout.Trim());
|
||||
|
||||
_logger.LogInformation("Decompressed (rewritten uncompressed) btrfs file: {file}", path);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error rewriting {file} for decompression", path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,7 +372,7 @@ public sealed class FileCompactor : IDisposable
|
||||
return _clusterSizes[root];
|
||||
}
|
||||
|
||||
private static bool IsCompactedFile(string filePath)
|
||||
private static bool IsWOFCompactedFile(string filePath)
|
||||
{
|
||||
uint buf = 8;
|
||||
_ = WofIsExternalFile(filePath, out int isExtFile, out uint _, out var info, ref buf);
|
||||
@@ -228,40 +380,151 @@ public sealed class FileCompactor : IDisposable
|
||||
return info.Algorithm == CompressionAlgorithm.XPRESS8K;
|
||||
}
|
||||
|
||||
private void WOFCompressFile(string path)
|
||||
private bool IsBtrfsCompressedFile(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
var psi = new ProcessStartInfo("filefrag", $"-v \"{path}\"")
|
||||
{
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true
|
||||
};
|
||||
|
||||
using var proc = Process.Start(psi);
|
||||
if (proc == null)
|
||||
{
|
||||
_logger.LogWarning("Failed to start filefrag for {file}", path);
|
||||
return false;
|
||||
}
|
||||
|
||||
string output = proc.StandardOutput.ReadToEnd();
|
||||
proc.WaitForExit();
|
||||
|
||||
// look for "flags: compressed" in the output
|
||||
if (output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "Failed to detect btrfs compression for {file}", path);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private bool WOFCompressFile(string path)
|
||||
{
|
||||
var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo));
|
||||
Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: true);
|
||||
ulong length = (ulong)Marshal.SizeOf(_efInfo);
|
||||
try
|
||||
{
|
||||
using (var fs = new FileStream(path, FileMode.Open))
|
||||
using var fs = new FileStream(path, FileMode.Open);
|
||||
#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called
|
||||
var hFile = fs.SafeFileHandle.DangerousGetHandle();
|
||||
#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called
|
||||
if (fs.SafeFileHandle.IsInvalid)
|
||||
{
|
||||
#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called
|
||||
var hFile = fs.SafeFileHandle.DangerousGetHandle();
|
||||
#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called
|
||||
if (fs.SafeFileHandle.IsInvalid)
|
||||
{
|
||||
_logger.LogWarning("Invalid file handle to {file}", path);
|
||||
}
|
||||
else
|
||||
{
|
||||
var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length);
|
||||
if (!(ret == 0 || ret == unchecked((int)0x80070158)))
|
||||
{
|
||||
_logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X"));
|
||||
}
|
||||
}
|
||||
_logger.LogWarning("Invalid file handle to {file}", path);
|
||||
return false;
|
||||
}
|
||||
|
||||
var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length);
|
||||
if (!(ret == 0 || ret == unchecked((int)0x80070158)))
|
||||
{
|
||||
_logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X"));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error compacting file {path}", path);
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
Marshal.FreeHGlobal(efInfoPtr);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool BtrfsCompressFile(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -czstd -- \"{path}\"")
|
||||
{
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true
|
||||
};
|
||||
|
||||
using var proc = Process.Start(psi);
|
||||
if (proc == null)
|
||||
{
|
||||
_logger.LogWarning("Failed to start btrfs process for {file}", path);
|
||||
return false;
|
||||
}
|
||||
|
||||
var stdout = proc.StandardOutput.ReadToEnd();
|
||||
var stderr = proc.StandardError.ReadToEnd();
|
||||
proc.WaitForExit();
|
||||
|
||||
if (proc.ExitCode != 0)
|
||||
{
|
||||
_logger.LogWarning("btrfs defrag returned {code} for {file}: {err}", proc.ExitCode, path, stderr);
|
||||
return false;
|
||||
}
|
||||
|
||||
_logger.LogDebug("btrfs output: {out}", stdout);
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error running btrfs defragment for {file}", path);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private string GetMountOptionsForPath(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
var fullPath = Path.GetFullPath(path);
|
||||
var mounts = File.ReadAllLines("/proc/mounts");
|
||||
string bestMount = string.Empty;
|
||||
string mountOptions = string.Empty;
|
||||
|
||||
foreach (var line in mounts)
|
||||
{
|
||||
var parts = line.Split(' ');
|
||||
if (parts.Length < 4) continue;
|
||||
var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); // unescape spaces
|
||||
string normalized;
|
||||
try { normalized = Path.GetFullPath(mountPoint); }
|
||||
catch { normalized = mountPoint; }
|
||||
|
||||
if (fullPath.StartsWith(normalized, StringComparison.Ordinal) &&
|
||||
normalized.Length > bestMount.Length)
|
||||
{
|
||||
bestMount = normalized;
|
||||
mountOptions = parts[3];
|
||||
}
|
||||
}
|
||||
|
||||
return mountOptions;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "Failed to get mount options for {path}", path);
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
private struct WOF_FILE_COMPRESSION_INFO_V1
|
||||
@@ -273,7 +536,14 @@ public sealed class FileCompactor : IDisposable
|
||||
private void EnqueueCompaction(string filePath)
|
||||
{
|
||||
if (!_pendingCompactions.TryAdd(filePath, 0))
|
||||
return;
|
||||
|
||||
var fsType = GetFilesystemType(filePath);
|
||||
|
||||
if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs)
|
||||
{
|
||||
_logger.LogTrace("Skipping compaction enqueue for unsupported filesystem {fs} ({file})", fsType, filePath);
|
||||
_pendingCompactions.TryRemove(filePath, out _);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -282,6 +552,10 @@ public sealed class FileCompactor : IDisposable
|
||||
_pendingCompactions.TryRemove(filePath, out _);
|
||||
_logger.LogDebug("Failed to enqueue compaction job for {file}", filePath);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogTrace("Queued compaction job for {file} (fs={fs})", filePath, fsType);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessQueueAsync(CancellationToken token)
|
||||
@@ -299,7 +573,7 @@ public sealed class FileCompactor : IDisposable
|
||||
return;
|
||||
}
|
||||
|
||||
if (_dalamudUtilService.IsWine || !_lightlessConfigService.Current.UseCompactor)
|
||||
if (!_lightlessConfigService.Current.UseCompactor)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -13,5 +13,4 @@ public class ServerStorage
|
||||
public bool UseOAuth2 { get; set; } = false;
|
||||
public string? OAuthToken { get; set; } = null;
|
||||
public HttpTransportType HttpTransportType { get; set; } = HttpTransportType.WebSockets;
|
||||
public bool ForceWebSockets { get; set; } = false;
|
||||
}
|
||||
@@ -1227,16 +1227,16 @@ public class SettingsUi : WindowMediatorSubscriberBase
|
||||
ImGui.TextUnformatted($"Currently utilized local storage: Calculating...");
|
||||
ImGui.TextUnformatted(
|
||||
$"Remaining space free on drive: {UiSharedService.ByteToString(_cacheMonitor.FileCacheDriveFree)}");
|
||||
|
||||
bool useFileCompactor = _configService.Current.UseCompactor;
|
||||
bool isLinux = _dalamudUtilService.IsWine;
|
||||
if (!useFileCompactor && !isLinux)
|
||||
if (!useFileCompactor)
|
||||
{
|
||||
UiSharedService.ColorTextWrapped(
|
||||
"Hint: To free up space when using Lightless consider enabling the File Compactor",
|
||||
UIColors.Get("LightlessYellow"));
|
||||
}
|
||||
|
||||
if (isLinux || !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled();
|
||||
if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled();
|
||||
if (ImGui.Checkbox("Use file compactor", ref useFileCompactor))
|
||||
{
|
||||
_configService.Current.UseCompactor = useFileCompactor;
|
||||
@@ -1281,10 +1281,20 @@ public class SettingsUi : WindowMediatorSubscriberBase
|
||||
UIColors.Get("LightlessYellow"));
|
||||
}
|
||||
|
||||
if (isLinux || !_cacheMonitor.StorageisNTFS)
|
||||
if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS)
|
||||
{
|
||||
ImGui.EndDisabled();
|
||||
ImGui.TextUnformatted("The file compactor is only available on Windows and NTFS drives.");
|
||||
ImGui.TextUnformatted("The file compactor is only available on BTRFS and NTFS drives.");
|
||||
}
|
||||
|
||||
if (_cacheMonitor.StorageisNTFS)
|
||||
{
|
||||
ImGui.TextUnformatted("The file compactor is running on NTFS Drive.");
|
||||
}
|
||||
|
||||
if (_cacheMonitor.StorageIsBtrfs)
|
||||
{
|
||||
ImGui.TextUnformatted("The file compactor is running on Btrfs Drive.");
|
||||
}
|
||||
|
||||
ImGuiHelpers.ScaledDummy(new Vector2(10, 10));
|
||||
@@ -3113,22 +3123,6 @@ public class SettingsUi : WindowMediatorSubscriberBase
|
||||
UiSharedService.TooltipSeparator
|
||||
+ "Note: if the server does not support a specific Transport Type it will fall through to the next automatically: WebSockets > ServerSentEvents > LongPolling");
|
||||
|
||||
if (_dalamudUtilService.IsWine)
|
||||
{
|
||||
bool forceWebSockets = selectedServer.ForceWebSockets;
|
||||
if (ImGui.Checkbox("[wine only] Force WebSockets", ref forceWebSockets))
|
||||
{
|
||||
selectedServer.ForceWebSockets = forceWebSockets;
|
||||
_serverConfigurationManager.Save();
|
||||
}
|
||||
|
||||
_uiShared.DrawHelpText(
|
||||
"On wine, Lightless will automatically fall back to ServerSentEvents/LongPolling, even if WebSockets is selected. "
|
||||
+ "WebSockets are known to crash XIV entirely on wine 8.5 shipped with Dalamud. "
|
||||
+ "Only enable this if you are not running wine 8.5." + Environment.NewLine
|
||||
+ "Note: If the issue gets resolved at some point this option will be removed.");
|
||||
}
|
||||
|
||||
ImGuiHelpers.ScaledDummy(5);
|
||||
|
||||
if (ImGui.Checkbox("Use Discord OAuth2 Authentication", ref useOauth))
|
||||
|
||||
@@ -21,6 +21,26 @@ public static class Crypto
|
||||
return BitConverter.ToString(sha1.ComputeHash(stream)).Replace("-", "", StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
public static async Task<string> GetFileHashAsync(string filePath, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, bufferSize: 65536, options: FileOptions.Asynchronous);
|
||||
await using (stream.ConfigureAwait(false))
|
||||
{
|
||||
using var sha1 = SHA1.Create();
|
||||
|
||||
var buffer = new byte[8192];
|
||||
int bytesRead;
|
||||
while ((bytesRead = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0)
|
||||
{
|
||||
sha1.TransformBlock(buffer, 0, bytesRead, outputBuffer: null, 0);
|
||||
}
|
||||
|
||||
sha1.TransformFinalBlock([], 0, 0);
|
||||
|
||||
return BitConverter.ToString(sha1.Hash!).Replace("-", "", StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
|
||||
public static string GetHash256(this (string, ushort) playerToHash)
|
||||
{
|
||||
if (_hashListPlayersSHA256.TryGetValue(playerToHash, out var hash))
|
||||
|
||||
143
LightlessSync/Utils/FileSystemHelper.cs
Normal file
143
LightlessSync/Utils/FileSystemHelper.cs
Normal file
@@ -0,0 +1,143 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace LightlessSync.Utils
|
||||
{
|
||||
public static class FileSystemHelper
|
||||
{
|
||||
public enum FilesystemType
|
||||
{
|
||||
Unknown = 0,
|
||||
NTFS,
|
||||
Btrfs,
|
||||
Ext4,
|
||||
Xfs,
|
||||
Apfs,
|
||||
HfsPlus,
|
||||
Fat,
|
||||
Exfat,
|
||||
Zfs
|
||||
}
|
||||
|
||||
private static readonly ConcurrentDictionary<string, FilesystemType> _filesystemTypeCache = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public static FilesystemType GetFilesystemType(string filePath)
|
||||
{
|
||||
try
|
||||
{
|
||||
string rootPath;
|
||||
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
|
||||
{
|
||||
var info = new FileInfo(filePath);
|
||||
var dir = info.Directory ?? new DirectoryInfo(filePath);
|
||||
rootPath = dir.Root.FullName;
|
||||
}
|
||||
else
|
||||
{
|
||||
rootPath = GetMountPoint(filePath);
|
||||
if (string.IsNullOrEmpty(rootPath))
|
||||
rootPath = "/";
|
||||
}
|
||||
|
||||
if (_filesystemTypeCache.TryGetValue(rootPath, out var cachedType))
|
||||
return cachedType;
|
||||
|
||||
FilesystemType detected;
|
||||
|
||||
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
|
||||
{
|
||||
var root = new DriveInfo(rootPath);
|
||||
var format = root.DriveFormat?.ToUpperInvariant() ?? string.Empty;
|
||||
detected = format switch
|
||||
{
|
||||
"NTFS" => FilesystemType.NTFS,
|
||||
"FAT32" => FilesystemType.Fat,
|
||||
"EXFAT" => FilesystemType.Exfat,
|
||||
_ => FilesystemType.Unknown
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
detected = GetLinuxFilesystemType(filePath);
|
||||
}
|
||||
|
||||
_filesystemTypeCache[rootPath] = detected;
|
||||
return detected;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return FilesystemType.Unknown;
|
||||
}
|
||||
}
|
||||
|
||||
private static string GetMountPoint(string filePath)
|
||||
{
|
||||
try
|
||||
{
|
||||
var path = Path.GetFullPath(filePath);
|
||||
if (!File.Exists("/proc/mounts")) return "/";
|
||||
var mounts = File.ReadAllLines("/proc/mounts");
|
||||
|
||||
string bestMount = "/";
|
||||
foreach (var line in mounts)
|
||||
{
|
||||
var parts = line.Split(' ');
|
||||
if (parts.Length < 3) continue;
|
||||
var mountPoint = parts[1].Replace("\\040", " "); // unescape spaces
|
||||
|
||||
string normalizedMount;
|
||||
try { normalizedMount = Path.GetFullPath(mountPoint); }
|
||||
catch { normalizedMount = mountPoint; }
|
||||
|
||||
if (path.StartsWith(normalizedMount, StringComparison.Ordinal) &&
|
||||
normalizedMount.Length > bestMount.Length)
|
||||
{
|
||||
bestMount = normalizedMount;
|
||||
}
|
||||
}
|
||||
|
||||
return bestMount;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return "/";
|
||||
}
|
||||
}
|
||||
|
||||
private static FilesystemType GetLinuxFilesystemType(string filePath)
|
||||
{
|
||||
try
|
||||
{
|
||||
var mountPoint = GetMountPoint(filePath);
|
||||
var mounts = File.ReadAllLines("/proc/mounts");
|
||||
|
||||
foreach (var line in mounts)
|
||||
{
|
||||
var parts = line.Split(' ');
|
||||
if (parts.Length < 3) continue;
|
||||
var mount = parts[1].Replace("\\040", " ");
|
||||
if (string.Equals(mount, mountPoint, StringComparison.Ordinal))
|
||||
{
|
||||
var fstype = parts[2].ToLowerInvariant();
|
||||
return fstype switch
|
||||
{
|
||||
"btrfs" => FilesystemType.Btrfs,
|
||||
"ext4" => FilesystemType.Ext4,
|
||||
"xfs" => FilesystemType.Xfs,
|
||||
"zfs" => FilesystemType.Zfs,
|
||||
"apfs" => FilesystemType.Apfs,
|
||||
"hfsplus" => FilesystemType.HfsPlus,
|
||||
_ => FilesystemType.Unknown
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return FilesystemType.Unknown;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return FilesystemType.Unknown;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -70,13 +70,6 @@ public class HubFactory : MediatorSubscriberBase
|
||||
_ => HttpTransportType.WebSockets | HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling
|
||||
};
|
||||
|
||||
if (_isWine && !_serverConfigurationManager.CurrentServer.ForceWebSockets
|
||||
&& transportType.HasFlag(HttpTransportType.WebSockets))
|
||||
{
|
||||
Logger.LogDebug("Wine detected, falling back to ServerSentEvents / LongPolling");
|
||||
transportType = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling;
|
||||
}
|
||||
|
||||
Logger.LogDebug("Building new HubConnection using transport {transport}", transportType);
|
||||
|
||||
_instance = new HubConnectionBuilder()
|
||||
|
||||
Reference in New Issue
Block a user