Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 51 Wiki Activity

Changes in file compression for windows, redone linux side because wine issues.

Browse Source
This commit is contained in:
cake
2025-10-31 23:47:41 +01:00
parent 5feb74c1c0
commit 6e3c60f627
3 changed files with 627 additions and 508 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
LightlessSync/FileCache/CacheMonitor.cs
View File

@@ -137,7 +137,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
if (fsType == FileSystemHelper.FilesystemType.Btrfs)
{
StorageIsBtrfs = true;
Logger.LogInformation("Lightless Storage is on BTRFS drive: {isNtfs}", StorageIsBtrfs);
Logger.LogInformation("Lightless Storage is on BTRFS drive: {isBtrfs}", StorageIsBtrfs);
}
Logger.LogDebug("Initializing Lightless FSW on {path}", lightlessPath);
@@ -661,44 +661,44 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
if (ct.IsCancellationRequested) return;
// scan new files
if (allScannedFiles.Any(c => !c.Value))
var newFiles = allScannedFiles.Where(c => !c.Value).Select(c => c.Key).ToList();
foreach (var cachePath in newFiles)
{
Parallel.ForEach(allScannedFiles.Where(c => !c.Value).Select(c => c.Key),
new ParallelOptions()
{
MaxDegreeOfParallelism = threadCount,
CancellationToken = ct
}, (cachePath) =>
{
if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null)
{
Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}", _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null);
return;
}
if (ct.IsCancellationRequested) break;
ProcessOne(cachePath);
Interlocked.Increment(ref _currentFileProgress);
}
if (ct.IsCancellationRequested) return;
Logger.LogTrace("Scanner added {count} new files to db", newFiles.Count);
if (!_ipcManager.Penumbra.APIAvailable)
{
Logger.LogWarning("Penumbra not available");
return;
}
void ProcessOne(string? cachePath)
{
if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null)
{
Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}",
_fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null);
return;
}
try
{
var entry = _fileDbManager.CreateFileEntry(cachePath);
if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath);
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Failed adding {file}", cachePath);
}
if (!_ipcManager.Penumbra.APIAvailable)
{
Logger.LogWarning("Penumbra not available");
return;
}
Interlocked.Increment(ref _currentFileProgress);
});
Logger.LogTrace("Scanner added {notScanned} new files to db", allScannedFiles.Count(c => !c.Value));
try
{
var entry = _fileDbManager.CreateFileEntry(cachePath);
if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath);
}
catch (IOException ioex)
{
Logger.LogDebug(ioex, "File busy or locked: {file}", cachePath);
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Failed adding {file}", cachePath);
}
}
Logger.LogDebug("Scan complete");

960
LightlessSync/FileCache/FileCompactor.cs
View File

File diff suppressed because it is too large Load Diff

107
LightlessSync/Utils/FileSystemHelper.cs
View File

@@ -1,4 +1,6 @@
using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Runtime.InteropServices;
namespace LightlessSync.Utils
@@ -20,6 +22,8 @@ namespace LightlessSync.Utils
}
private const string _mountPath = "/proc/mounts";
private const int _defaultBlockSize = 4096;
private static readonly Dictionary<string, int> _blockSizeCache = new(StringComparer.OrdinalIgnoreCase);
private static readonly ConcurrentDictionary<string, FilesystemType> _filesystemTypeCache = new(StringComparer.OrdinalIgnoreCase);
public static FilesystemType GetFilesystemType(string filePath, bool isWine = false)
@@ -27,6 +31,7 @@ namespace LightlessSync.Utils
try
{
string rootPath;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (!IsProbablyWine() || !isWine))
{
var info = new FileInfo(filePath);
@@ -49,6 +54,7 @@ namespace LightlessSync.Utils
{
var root = new DriveInfo(rootPath);
var format = root.DriveFormat?.ToUpperInvariant() ?? string.Empty;
detected = format switch
{
"NTFS" => FilesystemType.NTFS,
@@ -62,10 +68,28 @@ namespace LightlessSync.Utils
detected = GetLinuxFilesystemType(filePath);
}
if (isWine || IsProbablyWine())
{
switch (detected)
{
case FilesystemType.NTFS:
case FilesystemType.Unknown:
{
var linuxDetected = GetLinuxFilesystemType(filePath);
if (linuxDetected != FilesystemType.Unknown)
{
detected = linuxDetected;
}
break;
}
}
}
_filesystemTypeCache[rootPath] = detected;
return detected;
}
catch (Exception ex)
catch
{
return FilesystemType.Unknown;
}
@@ -175,7 +199,84 @@ namespace LightlessSync.Utils
}
}
public static int GetBlockSizeForPath(string path, ILogger? logger = null, bool isWine = false)
{
try
{
if (string.IsNullOrWhiteSpace(path))
return _defaultBlockSize;
var fi = new FileInfo(path);
if (!fi.Exists)
return _defaultBlockSize;
var root = fi.Directory?.Root.FullName.ToLowerInvariant() ?? "/";
if (_blockSizeCache.TryGetValue(root, out int cached))
return cached;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !isWine)
{
int result = GetDiskFreeSpaceW(root,
out uint sectorsPerCluster,
out uint bytesPerSector,
out _,
out _);
if (result == 0)
{
logger?.LogWarning("Failed to determine block size for {root}", root);
return _defaultBlockSize;
}
int clusterSize = (int)(sectorsPerCluster * bytesPerSector);
_blockSizeCache[root] = clusterSize;
logger?.LogTrace("NTFS cluster size for {root}: {cluster}", root, clusterSize);
return clusterSize;
}
string realPath = fi.FullName;
if (isWine && realPath.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase))
{
realPath = "/" + realPath.Substring(3).Replace('\\', '/');
}
var psi = new ProcessStartInfo
{
FileName = "/bin/bash",
Arguments = $"-c \"stat -f -c %s '{realPath.Replace("'", "'\\''")}'\"",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true,
WorkingDirectory = "/"
};
using var proc = Process.Start(psi);
string stdout = proc?.StandardOutput.ReadToEnd().Trim() ?? "";
proc?.WaitForExit();
if (int.TryParse(stdout, out int blockSize) && blockSize > 0)
{
_blockSizeCache[root] = blockSize;
logger?.LogTrace("Filesystem block size via stat for {root}: {block}", root, blockSize);
return blockSize;
}
logger?.LogTrace("stat did not return valid block size for {file}, output: {out}", fi.FullName, stdout);
_blockSizeCache[root] = _defaultBlockSize;
return _defaultBlockSize;
}
catch (Exception ex)
{
logger?.LogTrace(ex, "Error determining block size for {path}", path);
return _defaultBlockSize;
}
}
[DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)]
private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters, out uint lpTotalNumberOfClusters);
//Extra check on
private static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts");
public static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts");
}
}