Merge pull request 'Btrfs Compactor work, defaulted linux on websockets.' (#77) from linux-improvements into 1.12.4
Reviewed-on: #77
This commit was merged in pull request #77.
This commit is contained in:
@@ -115,6 +115,8 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
|
|||||||
|
|
||||||
public bool StorageisNTFS { get; private set; } = false;
|
public bool StorageisNTFS { get; private set; } = false;
|
||||||
|
|
||||||
|
public bool StorageIsBtrfs { get ; private set; } = false;
|
||||||
|
|
||||||
public void StartLightlessWatcher(string? lightlessPath)
|
public void StartLightlessWatcher(string? lightlessPath)
|
||||||
{
|
{
|
||||||
LightlessWatcher?.Dispose();
|
LightlessWatcher?.Dispose();
|
||||||
@@ -124,10 +126,19 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
|
|||||||
Logger.LogWarning("Lightless file path is not set, cannot start the FSW for Lightless.");
|
Logger.LogWarning("Lightless file path is not set, cannot start the FSW for Lightless.");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
var fsType = FileSystemHelper.GetFilesystemType(_configService.Current.CacheFolder, _dalamudUtil.IsWine);
|
||||||
|
|
||||||
DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
|
if (fsType == FileSystemHelper.FilesystemType.NTFS)
|
||||||
StorageisNTFS = string.Equals("NTFS", di.DriveFormat, StringComparison.OrdinalIgnoreCase);
|
{
|
||||||
Logger.LogInformation("Lightless Storage is on NTFS drive: {isNtfs}", StorageisNTFS);
|
StorageisNTFS = true;
|
||||||
|
Logger.LogInformation("Lightless Storage is on NTFS drive: {isNtfs}", StorageisNTFS);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (fsType == FileSystemHelper.FilesystemType.Btrfs)
|
||||||
|
{
|
||||||
|
StorageIsBtrfs = true;
|
||||||
|
Logger.LogInformation("Lightless Storage is on BTRFS drive: {isBtrfs}", StorageIsBtrfs);
|
||||||
|
}
|
||||||
|
|
||||||
Logger.LogDebug("Initializing Lightless FSW on {path}", lightlessPath);
|
Logger.LogDebug("Initializing Lightless FSW on {path}", lightlessPath);
|
||||||
LightlessWatcher = new()
|
LightlessWatcher = new()
|
||||||
@@ -392,51 +403,94 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
|
|||||||
|
|
||||||
public void RecalculateFileCacheSize(CancellationToken token)
|
public void RecalculateFileCacheSize(CancellationToken token)
|
||||||
{
|
{
|
||||||
if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder))
|
if (string.IsNullOrEmpty(_configService.Current.CacheFolder) ||
|
||||||
|
!Directory.Exists(_configService.Current.CacheFolder))
|
||||||
{
|
{
|
||||||
FileCacheSize = 0;
|
FileCacheSize = 0;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
FileCacheSize = -1;
|
FileCacheSize = -1;
|
||||||
DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
|
bool isWine = _dalamudUtil?.IsWine ?? false;
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
FileCacheDriveFree = di.AvailableFreeSpace;
|
var drive = DriveInfo.GetDrives()
|
||||||
|
.FirstOrDefault(d => _configService.Current.CacheFolder
|
||||||
|
.StartsWith(d.Name, StringComparison.OrdinalIgnoreCase));
|
||||||
|
|
||||||
|
if (drive != null)
|
||||||
|
FileCacheDriveFree = drive.AvailableFreeSpace;
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder);
|
Logger.LogWarning(ex, "Could not determine drive size for storage folder {folder}", _configService.Current.CacheFolder);
|
||||||
}
|
}
|
||||||
|
|
||||||
var files = Directory.EnumerateFiles(_configService.Current.CacheFolder).Select(f => new FileInfo(f))
|
var files = Directory.EnumerateFiles(_configService.Current.CacheFolder)
|
||||||
.OrderBy(f => f.LastAccessTime).ToList();
|
.Select(f => new FileInfo(f))
|
||||||
FileCacheSize = files
|
.OrderBy(f => f.LastAccessTime)
|
||||||
.Sum(f =>
|
.ToList();
|
||||||
{
|
|
||||||
token.ThrowIfCancellationRequested();
|
|
||||||
|
|
||||||
try
|
long totalSize = 0;
|
||||||
|
|
||||||
|
foreach (var f in files)
|
||||||
|
{
|
||||||
|
token.ThrowIfCancellationRequested();
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
long size = 0;
|
||||||
|
|
||||||
|
if (!isWine)
|
||||||
{
|
{
|
||||||
return _fileCompactor.GetFileSizeOnDisk(f, StorageisNTFS);
|
try
|
||||||
|
{
|
||||||
|
size = _fileCompactor.GetFileSizeOnDisk(f);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName);
|
||||||
|
size = f.Length;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
catch
|
else
|
||||||
{
|
{
|
||||||
return 0;
|
size = f.Length;
|
||||||
}
|
}
|
||||||
});
|
|
||||||
|
totalSize += size;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Logger.LogTrace(ex, "Error getting size for {file}", f.FullName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
FileCacheSize = totalSize;
|
||||||
|
|
||||||
var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d);
|
var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d);
|
||||||
|
if (FileCacheSize < maxCacheInBytes)
|
||||||
if (FileCacheSize < maxCacheInBytes) return;
|
return;
|
||||||
|
|
||||||
var maxCacheBuffer = maxCacheInBytes * 0.05d;
|
var maxCacheBuffer = maxCacheInBytes * 0.05d;
|
||||||
while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer)
|
|
||||||
|
while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer && files.Count > 0)
|
||||||
{
|
{
|
||||||
var oldestFile = files[0];
|
var oldestFile = files[0];
|
||||||
FileCacheSize -= _fileCompactor.GetFileSizeOnDisk(oldestFile);
|
|
||||||
File.Delete(oldestFile.FullName);
|
try
|
||||||
files.Remove(oldestFile);
|
{
|
||||||
|
long fileSize = oldestFile.Length;
|
||||||
|
File.Delete(oldestFile.FullName);
|
||||||
|
FileCacheSize -= fileSize;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullName);
|
||||||
|
}
|
||||||
|
|
||||||
|
files.RemoveAt(0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -644,44 +698,44 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
|
|||||||
|
|
||||||
if (ct.IsCancellationRequested) return;
|
if (ct.IsCancellationRequested) return;
|
||||||
|
|
||||||
// scan new files
|
var newFiles = allScannedFiles.Where(c => !c.Value).Select(c => c.Key).ToList();
|
||||||
if (allScannedFiles.Any(c => !c.Value))
|
foreach (var cachePath in newFiles)
|
||||||
{
|
{
|
||||||
Parallel.ForEach(allScannedFiles.Where(c => !c.Value).Select(c => c.Key),
|
if (ct.IsCancellationRequested) break;
|
||||||
new ParallelOptions()
|
ProcessOne(cachePath);
|
||||||
{
|
Interlocked.Increment(ref _currentFileProgress);
|
||||||
MaxDegreeOfParallelism = threadCount,
|
}
|
||||||
CancellationToken = ct
|
|
||||||
}, (cachePath) =>
|
|
||||||
{
|
|
||||||
if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null)
|
|
||||||
{
|
|
||||||
Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}", _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (ct.IsCancellationRequested) return;
|
Logger.LogTrace("Scanner added {count} new files to db", newFiles.Count);
|
||||||
|
|
||||||
if (!_ipcManager.Penumbra.APIAvailable)
|
void ProcessOne(string? cachePath)
|
||||||
{
|
{
|
||||||
Logger.LogWarning("Penumbra not available");
|
if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null)
|
||||||
return;
|
{
|
||||||
}
|
Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}",
|
||||||
|
_fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
try
|
if (!_ipcManager.Penumbra.APIAvailable)
|
||||||
{
|
{
|
||||||
var entry = _fileDbManager.CreateFileEntry(cachePath);
|
Logger.LogWarning("Penumbra not available");
|
||||||
if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath);
|
return;
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
Logger.LogWarning(ex, "Failed adding {file}", cachePath);
|
|
||||||
}
|
|
||||||
|
|
||||||
Interlocked.Increment(ref _currentFileProgress);
|
try
|
||||||
});
|
{
|
||||||
|
var entry = _fileDbManager.CreateFileEntry(cachePath);
|
||||||
Logger.LogTrace("Scanner added {notScanned} new files to db", allScannedFiles.Count(c => !c.Value));
|
if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath);
|
||||||
|
}
|
||||||
|
catch (IOException ioex)
|
||||||
|
{
|
||||||
|
Logger.LogDebug(ioex, "File busy or locked: {file}", cachePath);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Logger.LogWarning(ex, "Failed adding {file}", cachePath);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Logger.LogDebug("Scan complete");
|
Logger.LogDebug("Scan complete");
|
||||||
|
|||||||
@@ -203,42 +203,72 @@ public sealed class FileCacheManager : IHostedService
|
|||||||
return output;
|
return output;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Task<List<FileCacheEntity>> ValidateLocalIntegrity(IProgress<(int, int, FileCacheEntity)> progress, CancellationToken cancellationToken)
|
public async Task<List<FileCacheEntity>> ValidateLocalIntegrity(IProgress<(int completed, int total, FileCacheEntity current)> progress, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
_lightlessMediator.Publish(new HaltScanMessage(nameof(ValidateLocalIntegrity)));
|
_lightlessMediator.Publish(new HaltScanMessage(nameof(ValidateLocalIntegrity)));
|
||||||
_logger.LogInformation("Validating local storage");
|
_logger.LogInformation("Validating local storage");
|
||||||
var cacheEntries = _fileCaches.Values.SelectMany(v => v.Values.Where(e => e != null)).Where(v => v.IsCacheEntry).ToList();
|
|
||||||
List<FileCacheEntity> brokenEntities = [];
|
var cacheEntries = _fileCaches.Values
|
||||||
int i = 0;
|
.SelectMany(v => v.Values)
|
||||||
foreach (var fileCache in cacheEntries)
|
.Where(v => v.IsCacheEntry)
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
int total = cacheEntries.Count;
|
||||||
|
int processed = 0;
|
||||||
|
var brokenEntities = new ConcurrentBag<FileCacheEntity>();
|
||||||
|
|
||||||
|
_logger.LogInformation("Checking {count} cache entries...", total);
|
||||||
|
|
||||||
|
await Parallel.ForEachAsync(cacheEntries, new ParallelOptions
|
||||||
|
{
|
||||||
|
MaxDegreeOfParallelism = Environment.ProcessorCount,
|
||||||
|
CancellationToken = cancellationToken
|
||||||
|
},
|
||||||
|
async (fileCache, token) =>
|
||||||
{
|
{
|
||||||
if (cancellationToken.IsCancellationRequested) break;
|
|
||||||
|
|
||||||
_logger.LogInformation("Validating {file}", fileCache.ResolvedFilepath);
|
|
||||||
|
|
||||||
progress.Report((i, cacheEntries.Count, fileCache));
|
|
||||||
i++;
|
|
||||||
if (!File.Exists(fileCache.ResolvedFilepath))
|
|
||||||
{
|
|
||||||
brokenEntities.Add(fileCache);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var computedHash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
|
int current = Interlocked.Increment(ref processed);
|
||||||
|
if (current % 10 == 0)
|
||||||
|
progress.Report((current, total, fileCache));
|
||||||
|
|
||||||
|
if (!File.Exists(fileCache.ResolvedFilepath))
|
||||||
|
{
|
||||||
|
brokenEntities.Add(fileCache);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
string computedHash;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
computedHash = await Crypto.GetFileHashAsync(fileCache.ResolvedFilepath, token).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Error hashing {file}", fileCache.ResolvedFilepath);
|
||||||
|
brokenEntities.Add(fileCache);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal))
|
if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal))
|
||||||
{
|
{
|
||||||
_logger.LogInformation("Failed to validate {file}, got hash {computedHash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
|
_logger.LogInformation(
|
||||||
|
"Hash mismatch: {file} (got {computedHash}, expected {expected})",
|
||||||
|
fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
|
||||||
|
|
||||||
brokenEntities.Add(fileCache);
|
brokenEntities.Add(fileCache);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
catch (Exception e)
|
catch (OperationCanceledException)
|
||||||
{
|
{
|
||||||
_logger.LogWarning(e, "Error during validation of {file}", fileCache.ResolvedFilepath);
|
_logger.LogError("Validation got cancelled for {file}", fileCache.ResolvedFilepath);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Unexpected error validating {file}", fileCache.ResolvedFilepath);
|
||||||
brokenEntities.Add(fileCache);
|
brokenEntities.Add(fileCache);
|
||||||
}
|
}
|
||||||
}
|
}).ConfigureAwait(false);
|
||||||
|
|
||||||
foreach (var brokenEntity in brokenEntities)
|
foreach (var brokenEntity in brokenEntities)
|
||||||
{
|
{
|
||||||
@@ -250,12 +280,14 @@ public sealed class FileCacheManager : IHostedService
|
|||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
_logger.LogWarning(ex, "Could not delete {file}", brokenEntity.ResolvedFilepath);
|
_logger.LogWarning(ex, "Failed to delete invalid cache file {file}", brokenEntity.ResolvedFilepath);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
_lightlessMediator.Publish(new ResumeScanMessage(nameof(ValidateLocalIntegrity)));
|
_lightlessMediator.Publish(new ResumeScanMessage(nameof(ValidateLocalIntegrity)));
|
||||||
return Task.FromResult(brokenEntities);
|
_logger.LogInformation("Validation complete. Found {count} invalid entries.", brokenEntities.Count);
|
||||||
|
|
||||||
|
return [.. brokenEntities];
|
||||||
}
|
}
|
||||||
|
|
||||||
public string GetCacheFilePath(string hash, string extension)
|
public string GetCacheFilePath(string hash, string extension)
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -13,5 +13,4 @@ public class ServerStorage
|
|||||||
public bool UseOAuth2 { get; set; } = false;
|
public bool UseOAuth2 { get; set; } = false;
|
||||||
public string? OAuthToken { get; set; } = null;
|
public string? OAuthToken { get; set; } = null;
|
||||||
public HttpTransportType HttpTransportType { get; set; } = HttpTransportType.WebSockets;
|
public HttpTransportType HttpTransportType { get; set; } = HttpTransportType.WebSockets;
|
||||||
public bool ForceWebSockets { get; set; } = false;
|
|
||||||
}
|
}
|
||||||
@@ -1227,16 +1227,16 @@ public class SettingsUi : WindowMediatorSubscriberBase
|
|||||||
ImGui.TextUnformatted($"Currently utilized local storage: Calculating...");
|
ImGui.TextUnformatted($"Currently utilized local storage: Calculating...");
|
||||||
ImGui.TextUnformatted(
|
ImGui.TextUnformatted(
|
||||||
$"Remaining space free on drive: {UiSharedService.ByteToString(_cacheMonitor.FileCacheDriveFree)}");
|
$"Remaining space free on drive: {UiSharedService.ByteToString(_cacheMonitor.FileCacheDriveFree)}");
|
||||||
|
|
||||||
bool useFileCompactor = _configService.Current.UseCompactor;
|
bool useFileCompactor = _configService.Current.UseCompactor;
|
||||||
bool isLinux = _dalamudUtilService.IsWine;
|
if (!useFileCompactor)
|
||||||
if (!useFileCompactor && !isLinux)
|
|
||||||
{
|
{
|
||||||
UiSharedService.ColorTextWrapped(
|
UiSharedService.ColorTextWrapped(
|
||||||
"Hint: To free up space when using Lightless consider enabling the File Compactor",
|
"Hint: To free up space when using Lightless consider enabling the File Compactor",
|
||||||
UIColors.Get("LightlessYellow"));
|
UIColors.Get("LightlessYellow"));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isLinux || !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled();
|
if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled();
|
||||||
if (ImGui.Checkbox("Use file compactor", ref useFileCompactor))
|
if (ImGui.Checkbox("Use file compactor", ref useFileCompactor))
|
||||||
{
|
{
|
||||||
_configService.Current.UseCompactor = useFileCompactor;
|
_configService.Current.UseCompactor = useFileCompactor;
|
||||||
@@ -1281,10 +1281,20 @@ public class SettingsUi : WindowMediatorSubscriberBase
|
|||||||
UIColors.Get("LightlessYellow"));
|
UIColors.Get("LightlessYellow"));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isLinux || !_cacheMonitor.StorageisNTFS)
|
if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS)
|
||||||
{
|
{
|
||||||
ImGui.EndDisabled();
|
ImGui.EndDisabled();
|
||||||
ImGui.TextUnformatted("The file compactor is only available on Windows and NTFS drives.");
|
ImGui.TextUnformatted("The file compactor is only available on BTRFS and NTFS drives.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (_cacheMonitor.StorageisNTFS)
|
||||||
|
{
|
||||||
|
ImGui.TextUnformatted("The file compactor is running on NTFS Drive.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (_cacheMonitor.StorageIsBtrfs)
|
||||||
|
{
|
||||||
|
ImGui.TextUnformatted("The file compactor is running on Btrfs Drive.");
|
||||||
}
|
}
|
||||||
|
|
||||||
ImGuiHelpers.ScaledDummy(new Vector2(10, 10));
|
ImGuiHelpers.ScaledDummy(new Vector2(10, 10));
|
||||||
@@ -3113,22 +3123,6 @@ public class SettingsUi : WindowMediatorSubscriberBase
|
|||||||
UiSharedService.TooltipSeparator
|
UiSharedService.TooltipSeparator
|
||||||
+ "Note: if the server does not support a specific Transport Type it will fall through to the next automatically: WebSockets > ServerSentEvents > LongPolling");
|
+ "Note: if the server does not support a specific Transport Type it will fall through to the next automatically: WebSockets > ServerSentEvents > LongPolling");
|
||||||
|
|
||||||
if (_dalamudUtilService.IsWine)
|
|
||||||
{
|
|
||||||
bool forceWebSockets = selectedServer.ForceWebSockets;
|
|
||||||
if (ImGui.Checkbox("[wine only] Force WebSockets", ref forceWebSockets))
|
|
||||||
{
|
|
||||||
selectedServer.ForceWebSockets = forceWebSockets;
|
|
||||||
_serverConfigurationManager.Save();
|
|
||||||
}
|
|
||||||
|
|
||||||
_uiShared.DrawHelpText(
|
|
||||||
"On wine, Lightless will automatically fall back to ServerSentEvents/LongPolling, even if WebSockets is selected. "
|
|
||||||
+ "WebSockets are known to crash XIV entirely on wine 8.5 shipped with Dalamud. "
|
|
||||||
+ "Only enable this if you are not running wine 8.5." + Environment.NewLine
|
|
||||||
+ "Note: If the issue gets resolved at some point this option will be removed.");
|
|
||||||
}
|
|
||||||
|
|
||||||
ImGuiHelpers.ScaledDummy(5);
|
ImGuiHelpers.ScaledDummy(5);
|
||||||
|
|
||||||
if (ImGui.Checkbox("Use Discord OAuth2 Authentication", ref useOauth))
|
if (ImGui.Checkbox("Use Discord OAuth2 Authentication", ref useOauth))
|
||||||
|
|||||||
@@ -1,16 +1,15 @@
|
|||||||
using System;
|
using System.Security.Cryptography;
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.IO;
|
|
||||||
using System.Security.Cryptography;
|
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
|
||||||
namespace LightlessSync.Utils;
|
namespace LightlessSync.Utils;
|
||||||
|
|
||||||
public static class Crypto
|
public static class Crypto
|
||||||
{
|
{
|
||||||
|
//This buffersize seems to be the best sweetpoint for Linux and Windows
|
||||||
|
private const int _bufferSize = 65536;
|
||||||
#pragma warning disable SYSLIB0021 // Type or member is obsolete
|
#pragma warning disable SYSLIB0021 // Type or member is obsolete
|
||||||
|
|
||||||
private static readonly Dictionary<(string, ushort), string> _hashListPlayersSHA256 = new();
|
private static readonly Dictionary<(string, ushort), string> _hashListPlayersSHA256 = [];
|
||||||
private static readonly Dictionary<string, string> _hashListSHA256 = new(StringComparer.Ordinal);
|
private static readonly Dictionary<string, string> _hashListSHA256 = new(StringComparer.Ordinal);
|
||||||
private static readonly SHA256CryptoServiceProvider _sha256CryptoProvider = new();
|
private static readonly SHA256CryptoServiceProvider _sha256CryptoProvider = new();
|
||||||
|
|
||||||
@@ -21,6 +20,26 @@ public static class Crypto
|
|||||||
return BitConverter.ToString(sha1.ComputeHash(stream)).Replace("-", "", StringComparison.Ordinal);
|
return BitConverter.ToString(sha1.ComputeHash(stream)).Replace("-", "", StringComparison.Ordinal);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static async Task<string> GetFileHashAsync(string filePath, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, bufferSize: _bufferSize, options: FileOptions.Asynchronous);
|
||||||
|
await using (stream.ConfigureAwait(false))
|
||||||
|
{
|
||||||
|
using var sha1 = SHA1.Create();
|
||||||
|
|
||||||
|
var buffer = new byte[8192];
|
||||||
|
int bytesRead;
|
||||||
|
while ((bytesRead = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0)
|
||||||
|
{
|
||||||
|
sha1.TransformBlock(buffer, 0, bytesRead, outputBuffer: null, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
sha1.TransformFinalBlock([], 0, 0);
|
||||||
|
|
||||||
|
return Convert.ToHexString(sha1.Hash!);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static string GetHash256(this (string, ushort) playerToHash)
|
public static string GetHash256(this (string, ushort) playerToHash)
|
||||||
{
|
{
|
||||||
if (_hashListPlayersSHA256.TryGetValue(playerToHash, out var hash))
|
if (_hashListPlayersSHA256.TryGetValue(playerToHash, out var hash))
|
||||||
|
|||||||
282
LightlessSync/Utils/FileSystemHelper.cs
Normal file
282
LightlessSync/Utils/FileSystemHelper.cs
Normal file
@@ -0,0 +1,282 @@
|
|||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using System.Collections.Concurrent;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
|
|
||||||
|
namespace LightlessSync.Utils
|
||||||
|
{
|
||||||
|
public static class FileSystemHelper
|
||||||
|
{
|
||||||
|
public enum FilesystemType
|
||||||
|
{
|
||||||
|
Unknown = 0,
|
||||||
|
NTFS, // Compressable on file level
|
||||||
|
Btrfs, // Compressable on file level
|
||||||
|
Ext4, // Uncompressable
|
||||||
|
Xfs, // Uncompressable
|
||||||
|
Apfs, // Compressable on OS
|
||||||
|
HfsPlus, // Compressable on OS
|
||||||
|
Fat, // Uncompressable
|
||||||
|
Exfat, // Uncompressable
|
||||||
|
Zfs // Compressable, not on file level
|
||||||
|
}
|
||||||
|
|
||||||
|
private const string _mountPath = "/proc/mounts";
|
||||||
|
private const int _defaultBlockSize = 4096;
|
||||||
|
private static readonly Dictionary<string, int> _blockSizeCache = new(StringComparer.OrdinalIgnoreCase);
|
||||||
|
private static readonly ConcurrentDictionary<string, FilesystemType> _filesystemTypeCache = new(StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
public static FilesystemType GetFilesystemType(string filePath, bool isWine = false)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
string rootPath;
|
||||||
|
|
||||||
|
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (!IsProbablyWine() || !isWine))
|
||||||
|
{
|
||||||
|
var info = new FileInfo(filePath);
|
||||||
|
var dir = info.Directory ?? new DirectoryInfo(filePath);
|
||||||
|
rootPath = dir.Root.FullName;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
rootPath = GetMountPoint(filePath);
|
||||||
|
if (string.IsNullOrEmpty(rootPath))
|
||||||
|
rootPath = "/";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (_filesystemTypeCache.TryGetValue(rootPath, out var cachedType))
|
||||||
|
return cachedType;
|
||||||
|
|
||||||
|
FilesystemType detected;
|
||||||
|
|
||||||
|
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (!IsProbablyWine() || !isWine))
|
||||||
|
{
|
||||||
|
var root = new DriveInfo(rootPath);
|
||||||
|
var format = root.DriveFormat?.ToUpperInvariant() ?? string.Empty;
|
||||||
|
|
||||||
|
detected = format switch
|
||||||
|
{
|
||||||
|
"NTFS" => FilesystemType.NTFS,
|
||||||
|
"FAT32" => FilesystemType.Fat,
|
||||||
|
"EXFAT" => FilesystemType.Exfat,
|
||||||
|
_ => FilesystemType.Unknown
|
||||||
|
};
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
detected = GetLinuxFilesystemType(filePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isWine || IsProbablyWine())
|
||||||
|
{
|
||||||
|
switch (detected)
|
||||||
|
{
|
||||||
|
case FilesystemType.NTFS:
|
||||||
|
case FilesystemType.Unknown:
|
||||||
|
{
|
||||||
|
var linuxDetected = GetLinuxFilesystemType(filePath);
|
||||||
|
if (linuxDetected != FilesystemType.Unknown)
|
||||||
|
{
|
||||||
|
detected = linuxDetected;
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_filesystemTypeCache[rootPath] = detected;
|
||||||
|
return detected;
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
return FilesystemType.Unknown;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string GetMountPoint(string filePath)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var path = Path.GetFullPath(filePath);
|
||||||
|
if (!File.Exists(_mountPath)) return "/";
|
||||||
|
var mounts = File.ReadAllLines(_mountPath);
|
||||||
|
|
||||||
|
string bestMount = "/";
|
||||||
|
foreach (var line in mounts)
|
||||||
|
{
|
||||||
|
var parts = line.Split(' ');
|
||||||
|
if (parts.Length < 3) continue;
|
||||||
|
var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal);
|
||||||
|
|
||||||
|
string normalizedMount;
|
||||||
|
try { normalizedMount = Path.GetFullPath(mountPoint); }
|
||||||
|
catch { normalizedMount = mountPoint; }
|
||||||
|
|
||||||
|
if (path.StartsWith(normalizedMount, StringComparison.Ordinal) &&
|
||||||
|
normalizedMount.Length > bestMount.Length)
|
||||||
|
{
|
||||||
|
bestMount = normalizedMount;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return bestMount;
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
return "/";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string GetMountOptionsForPath(string path)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var fullPath = Path.GetFullPath(path);
|
||||||
|
var mounts = File.ReadAllLines("/proc/mounts");
|
||||||
|
string bestMount = string.Empty;
|
||||||
|
string mountOptions = string.Empty;
|
||||||
|
|
||||||
|
foreach (var line in mounts)
|
||||||
|
{
|
||||||
|
var parts = line.Split(' ');
|
||||||
|
if (parts.Length < 4) continue;
|
||||||
|
var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal);
|
||||||
|
string normalized;
|
||||||
|
try { normalized = Path.GetFullPath(mountPoint); }
|
||||||
|
catch { normalized = mountPoint; }
|
||||||
|
|
||||||
|
if (fullPath.StartsWith(normalized, StringComparison.Ordinal) &&
|
||||||
|
normalized.Length > bestMount.Length)
|
||||||
|
{
|
||||||
|
bestMount = normalized;
|
||||||
|
mountOptions = parts[3];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return mountOptions;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
return string.Empty;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FilesystemType GetLinuxFilesystemType(string filePath)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var mountPoint = GetMountPoint(filePath);
|
||||||
|
var mounts = File.ReadAllLines(_mountPath);
|
||||||
|
|
||||||
|
foreach (var line in mounts)
|
||||||
|
{
|
||||||
|
var parts = line.Split(' ');
|
||||||
|
if (parts.Length < 3) continue;
|
||||||
|
var mount = parts[1].Replace("\\040", " ", StringComparison.Ordinal);
|
||||||
|
if (string.Equals(mount, mountPoint, StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
var fstype = parts[2].ToLowerInvariant();
|
||||||
|
return fstype switch
|
||||||
|
{
|
||||||
|
"btrfs" => FilesystemType.Btrfs,
|
||||||
|
"ext4" => FilesystemType.Ext4,
|
||||||
|
"xfs" => FilesystemType.Xfs,
|
||||||
|
"zfs" => FilesystemType.Zfs,
|
||||||
|
"apfs" => FilesystemType.Apfs,
|
||||||
|
"hfsplus" => FilesystemType.HfsPlus,
|
||||||
|
_ => FilesystemType.Unknown
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return FilesystemType.Unknown;
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
return FilesystemType.Unknown;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static int GetBlockSizeForPath(string path, ILogger? logger = null, bool isWine = false)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(path))
|
||||||
|
return _defaultBlockSize;
|
||||||
|
|
||||||
|
var fi = new FileInfo(path);
|
||||||
|
if (!fi.Exists)
|
||||||
|
return _defaultBlockSize;
|
||||||
|
|
||||||
|
var root = fi.Directory?.Root.FullName.ToLowerInvariant() ?? "/";
|
||||||
|
if (_blockSizeCache.TryGetValue(root, out int cached))
|
||||||
|
return cached;
|
||||||
|
|
||||||
|
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !isWine)
|
||||||
|
{
|
||||||
|
int result = GetDiskFreeSpaceW(root,
|
||||||
|
out uint sectorsPerCluster,
|
||||||
|
out uint bytesPerSector,
|
||||||
|
out _,
|
||||||
|
out _);
|
||||||
|
|
||||||
|
if (result == 0)
|
||||||
|
{
|
||||||
|
logger?.LogWarning("Failed to determine block size for {root}", root);
|
||||||
|
return _defaultBlockSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
int clusterSize = (int)(sectorsPerCluster * bytesPerSector);
|
||||||
|
_blockSizeCache[root] = clusterSize;
|
||||||
|
logger?.LogTrace("NTFS cluster size for {root}: {cluster}", root, clusterSize);
|
||||||
|
return clusterSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
string realPath = fi.FullName;
|
||||||
|
if (isWine && realPath.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
realPath = "/" + realPath.Substring(3).Replace('\\', '/');
|
||||||
|
}
|
||||||
|
|
||||||
|
var psi = new ProcessStartInfo
|
||||||
|
{
|
||||||
|
FileName = "/bin/bash",
|
||||||
|
Arguments = $"-c \"stat -f -c %s '{realPath.Replace("'", "'\\''")}'\"",
|
||||||
|
RedirectStandardOutput = true,
|
||||||
|
RedirectStandardError = true,
|
||||||
|
UseShellExecute = false,
|
||||||
|
CreateNoWindow = true,
|
||||||
|
WorkingDirectory = "/"
|
||||||
|
};
|
||||||
|
|
||||||
|
using var proc = Process.Start(psi);
|
||||||
|
string stdout = proc?.StandardOutput.ReadToEnd().Trim() ?? "";
|
||||||
|
proc?.WaitForExit();
|
||||||
|
|
||||||
|
if (int.TryParse(stdout, out int blockSize) && blockSize > 0)
|
||||||
|
{
|
||||||
|
_blockSizeCache[root] = blockSize;
|
||||||
|
logger?.LogTrace("Filesystem block size via stat for {root}: {block}", root, blockSize);
|
||||||
|
return blockSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger?.LogTrace("stat did not return valid block size for {file}, output: {out}", fi.FullName, stdout);
|
||||||
|
_blockSizeCache[root] = _defaultBlockSize;
|
||||||
|
return _defaultBlockSize;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
logger?.LogTrace(ex, "Error determining block size for {path}", path);
|
||||||
|
return _defaultBlockSize;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)]
|
||||||
|
private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters, out uint lpTotalNumberOfClusters);
|
||||||
|
|
||||||
|
//Extra check on
|
||||||
|
public static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -70,13 +70,6 @@ public class HubFactory : MediatorSubscriberBase
|
|||||||
_ => HttpTransportType.WebSockets | HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling
|
_ => HttpTransportType.WebSockets | HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling
|
||||||
};
|
};
|
||||||
|
|
||||||
if (_isWine && !_serverConfigurationManager.CurrentServer.ForceWebSockets
|
|
||||||
&& transportType.HasFlag(HttpTransportType.WebSockets))
|
|
||||||
{
|
|
||||||
Logger.LogDebug("Wine detected, falling back to ServerSentEvents / LongPolling");
|
|
||||||
transportType = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling;
|
|
||||||
}
|
|
||||||
|
|
||||||
Logger.LogDebug("Building new HubConnection using transport {transport}", transportType);
|
Logger.LogDebug("Building new HubConnection using transport {transport}", transportType);
|
||||||
|
|
||||||
_instance = new HubConnectionBuilder()
|
_instance = new HubConnectionBuilder()
|
||||||
|
|||||||
Reference in New Issue
Block a user