913 lines
32 KiB
C#
913 lines
32 KiB
C#
using LightlessSync.Interop.Ipc;
|
|
using LightlessSync.LightlessConfiguration;
|
|
using LightlessSync.Services;
|
|
using LightlessSync.Services.Mediator;
|
|
using LightlessSync.Utils;
|
|
using Microsoft.Extensions.Logging;
|
|
using System.Collections.Concurrent;
|
|
using System.Collections.Immutable;
|
|
using System.IO;
|
|
|
|
namespace LightlessSync.FileCache;
|
|
|
|
public sealed class CacheMonitor : DisposableMediatorSubscriberBase
|
|
{
|
|
private readonly LightlessConfigService _configService;
|
|
private readonly DalamudUtilService _dalamudUtil;
|
|
private readonly FileCompactor _fileCompactor;
|
|
private readonly FileCacheManager _fileDbManager;
|
|
private readonly IpcManager _ipcManager;
|
|
private readonly PerformanceCollectorService _performanceCollector;
|
|
private long _currentFileProgress = 0;
|
|
private CancellationTokenSource _scanCancellationTokenSource = new();
|
|
private readonly CancellationTokenSource _periodicCalculationTokenSource = new();
|
|
private readonly SemaphoreSlim _dbGate = new(1, 1);
|
|
public static readonly IImmutableList<string> AllowedFileExtensions = [".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".pbd", ".scd", ".skp", ".shpk", ".kdb"];
|
|
private static readonly HashSet<string> AllowedFileExtensionSet = new(AllowedFileExtensions, StringComparer.OrdinalIgnoreCase);
|
|
|
|
public CacheMonitor(ILogger<CacheMonitor> logger, IpcManager ipcManager, LightlessConfigService configService,
|
|
FileCacheManager fileDbManager, LightlessMediator mediator, PerformanceCollectorService performanceCollector, DalamudUtilService dalamudUtil,
|
|
FileCompactor fileCompactor) : base(logger, mediator)
|
|
{
|
|
_ipcManager = ipcManager;
|
|
_configService = configService;
|
|
_fileDbManager = fileDbManager;
|
|
_performanceCollector = performanceCollector;
|
|
_dalamudUtil = dalamudUtil;
|
|
_fileCompactor = fileCompactor;
|
|
Mediator.Subscribe<PenumbraInitializedMessage>(this, (_) =>
|
|
{
|
|
StartPenumbraWatcher(_ipcManager.Penumbra.ModDirectory);
|
|
StartLightlessWatcher(configService.Current.CacheFolder);
|
|
InvokeScan();
|
|
});
|
|
Mediator.Subscribe<HaltScanMessage>(this, (msg) => HaltScan(msg.Source));
|
|
Mediator.Subscribe<ResumeScanMessage>(this, (msg) => ResumeScan(msg.Source));
|
|
Mediator.Subscribe<DalamudLoginMessage>(this, (_) =>
|
|
{
|
|
StartLightlessWatcher(configService.Current.CacheFolder);
|
|
StartPenumbraWatcher(_ipcManager.Penumbra.ModDirectory);
|
|
InvokeScan();
|
|
});
|
|
Mediator.Subscribe<PenumbraDirectoryChangedMessage>(this, (msg) =>
|
|
{
|
|
StartPenumbraWatcher(msg.ModDirectory);
|
|
InvokeScan();
|
|
});
|
|
if (_ipcManager.Penumbra.APIAvailable && !string.IsNullOrEmpty(_ipcManager.Penumbra.ModDirectory))
|
|
{
|
|
StartPenumbraWatcher(_ipcManager.Penumbra.ModDirectory);
|
|
}
|
|
if (configService.Current.HasValidSetup())
|
|
{
|
|
StartLightlessWatcher(configService.Current.CacheFolder);
|
|
InvokeScan();
|
|
}
|
|
|
|
var token = _periodicCalculationTokenSource.Token;
|
|
_ = Task.Run(async () =>
|
|
{
|
|
Logger.LogInformation("Starting Periodic Storage Directory Calculation Task");
|
|
var token = _periodicCalculationTokenSource.Token;
|
|
while (IsHalted() && !token.IsCancellationRequested)
|
|
await Task.Delay(TimeSpan.FromSeconds(5), token).ConfigureAwait(false);
|
|
|
|
while (!token.IsCancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
while (_dalamudUtil.IsOnFrameworkThread && !token.IsCancellationRequested)
|
|
{
|
|
await Task.Delay(1, token).ConfigureAwait(false);
|
|
}
|
|
|
|
RecalculateFileCacheSize(token);
|
|
}
|
|
catch
|
|
{
|
|
// ignore
|
|
}
|
|
await Task.Delay(TimeSpan.FromMinutes(1), token).ConfigureAwait(false);
|
|
}
|
|
}, token);
|
|
}
|
|
|
|
public long CurrentFileProgress => _currentFileProgress;
|
|
public long FileCacheSize { get; set; }
|
|
public long FileCacheDriveFree { get; set; }
|
|
|
|
private int _haltCount;
|
|
private bool IsHalted() => Volatile.Read(ref _haltCount) > 0;
|
|
public ConcurrentDictionary<string, int> HaltScanLocks { get; set; } = new(StringComparer.Ordinal);
|
|
public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
|
|
public long TotalFiles { get; private set; }
|
|
public long TotalFilesStorage { get; private set; }
|
|
|
|
public void HaltScan(string source)
|
|
{
|
|
HaltScanLocks.AddOrUpdate(source, 1, (_, v) => v + 1);
|
|
Interlocked.Increment(ref _haltCount);
|
|
}
|
|
|
|
record WatcherChange(WatcherChangeTypes ChangeType, string? OldPath = null);
|
|
private readonly record struct CacheEvictionCandidate(string FullPath, long Size, DateTime LastAccessTime);
|
|
private readonly object _penumbraGate = new();
|
|
private Dictionary<string, WatcherChange> _watcherChanges = new(StringComparer.OrdinalIgnoreCase);
|
|
|
|
private readonly object _lightlessGate = new();
|
|
private Dictionary<string, WatcherChange> _lightlessChanges = new(StringComparer.OrdinalIgnoreCase);
|
|
private Dictionary<string, WatcherChange> DrainPenumbraChanges()
|
|
{
|
|
lock (_penumbraGate)
|
|
{
|
|
var snapshot = _watcherChanges;
|
|
_watcherChanges = new(StringComparer.OrdinalIgnoreCase);
|
|
return snapshot;
|
|
}
|
|
}
|
|
|
|
private Dictionary<string, WatcherChange> DrainLightlessChanges()
|
|
{
|
|
lock (_lightlessGate)
|
|
{
|
|
var snapshot = _lightlessChanges;
|
|
_lightlessChanges = new(StringComparer.OrdinalIgnoreCase);
|
|
return snapshot;
|
|
}
|
|
}
|
|
|
|
public void StopMonitoring()
|
|
{
|
|
Logger.LogInformation("Stopping monitoring of Penumbra and Lightless storage folders");
|
|
LightlessWatcher?.Dispose();
|
|
PenumbraWatcher?.Dispose();
|
|
LightlessWatcher = null;
|
|
PenumbraWatcher = null;
|
|
}
|
|
|
|
public bool StorageisNTFS { get; private set; } = false;
|
|
|
|
public bool StorageIsBtrfs { get ; private set; } = false;
|
|
|
|
public void StartLightlessWatcher(string? lightlessPath)
|
|
{
|
|
LightlessWatcher?.Dispose();
|
|
if (string.IsNullOrEmpty(lightlessPath) || !Directory.Exists(lightlessPath))
|
|
{
|
|
LightlessWatcher = null;
|
|
Logger.LogWarning("Lightless file path is not set, cannot start the FSW for Lightless.");
|
|
return;
|
|
}
|
|
var fsType = FileSystemHelper.GetFilesystemType(_configService.Current.CacheFolder, _dalamudUtil.IsWine);
|
|
|
|
if (fsType == FileSystemHelper.FilesystemType.NTFS && !_dalamudUtil.IsWine)
|
|
{
|
|
StorageisNTFS = true;
|
|
Logger.LogInformation("Lightless Storage is on NTFS drive: {isNtfs}", StorageisNTFS);
|
|
}
|
|
|
|
if (fsType == FileSystemHelper.FilesystemType.Btrfs)
|
|
{
|
|
StorageIsBtrfs = true;
|
|
Logger.LogInformation("Lightless Storage is on BTRFS drive: {isBtrfs}", StorageIsBtrfs);
|
|
}
|
|
|
|
Logger.LogDebug("Initializing Lightless FSW on {path}", lightlessPath);
|
|
LightlessWatcher = new()
|
|
{
|
|
Path = lightlessPath,
|
|
InternalBufferSize = 8388608,
|
|
NotifyFilter = NotifyFilters.CreationTime
|
|
| NotifyFilters.LastWrite
|
|
| NotifyFilters.FileName
|
|
| NotifyFilters.DirectoryName
|
|
| NotifyFilters.Size,
|
|
Filter = "*.*",
|
|
IncludeSubdirectories = false,
|
|
};
|
|
|
|
LightlessWatcher.Deleted += LightlessWatcher_FileChanged;
|
|
LightlessWatcher.Created += LightlessWatcher_FileChanged;
|
|
LightlessWatcher.EnableRaisingEvents = true;
|
|
}
|
|
|
|
private void LightlessWatcher_FileChanged(object sender, FileSystemEventArgs e)
|
|
{
|
|
Logger.LogTrace("Lightless FSW: FileChanged: {change} => {path}", e.ChangeType, e.FullPath);
|
|
|
|
if (!HasAllowedExtension(e.FullPath)) return;
|
|
|
|
lock (_lightlessChanges)
|
|
{
|
|
_lightlessChanges[e.FullPath] = new(e.ChangeType);
|
|
}
|
|
|
|
_ = LightlessWatcherExecution();
|
|
}
|
|
|
|
public void StartPenumbraWatcher(string? penumbraPath)
|
|
{
|
|
PenumbraWatcher?.Dispose();
|
|
if (string.IsNullOrEmpty(penumbraPath))
|
|
{
|
|
PenumbraWatcher = null;
|
|
Logger.LogWarning("Penumbra is not connected or the path is not set, cannot start FSW for Penumbra.");
|
|
return;
|
|
}
|
|
|
|
Logger.LogDebug("Initializing Penumbra FSW on {path}", penumbraPath);
|
|
PenumbraWatcher = new()
|
|
{
|
|
Path = penumbraPath,
|
|
InternalBufferSize = 8388608,
|
|
NotifyFilter = NotifyFilters.CreationTime
|
|
| NotifyFilters.LastWrite
|
|
| NotifyFilters.FileName
|
|
| NotifyFilters.DirectoryName
|
|
| NotifyFilters.Size,
|
|
Filter = "*.*",
|
|
IncludeSubdirectories = true
|
|
};
|
|
|
|
PenumbraWatcher.Deleted += Fs_Changed;
|
|
PenumbraWatcher.Created += Fs_Changed;
|
|
PenumbraWatcher.Changed += Fs_Changed;
|
|
PenumbraWatcher.Renamed += Fs_Renamed;
|
|
PenumbraWatcher.EnableRaisingEvents = true;
|
|
}
|
|
|
|
private void Fs_Changed(object sender, FileSystemEventArgs e)
|
|
{
|
|
if (Directory.Exists(e.FullPath)) return;
|
|
if (!HasAllowedExtension(e.FullPath)) return;
|
|
|
|
if (e.ChangeType is not (WatcherChangeTypes.Changed or WatcherChangeTypes.Deleted or WatcherChangeTypes.Created))
|
|
return;
|
|
|
|
lock (_watcherChanges)
|
|
{
|
|
_watcherChanges[e.FullPath] = new(e.ChangeType);
|
|
}
|
|
|
|
Logger.LogTrace("FSW {event}: {path}", e.ChangeType, e.FullPath);
|
|
|
|
_ = PenumbraWatcherExecution();
|
|
}
|
|
|
|
private void Fs_Renamed(object sender, RenamedEventArgs e)
|
|
{
|
|
if (Directory.Exists(e.FullPath))
|
|
{
|
|
var directoryFiles = Directory.GetFiles(e.FullPath, "*.*", SearchOption.AllDirectories);
|
|
lock (_watcherChanges)
|
|
{
|
|
foreach (var file in directoryFiles)
|
|
{
|
|
if (!HasAllowedExtension(file)) continue;
|
|
var oldPath = file.Replace(e.FullPath, e.OldFullPath, StringComparison.OrdinalIgnoreCase);
|
|
|
|
_watcherChanges.Remove(oldPath);
|
|
_watcherChanges[file] = new(WatcherChangeTypes.Renamed, oldPath);
|
|
Logger.LogTrace("FSW Renamed: {path} -> {new}", oldPath, file);
|
|
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (!HasAllowedExtension(e.FullPath)) return;
|
|
|
|
lock (_watcherChanges)
|
|
{
|
|
_watcherChanges.Remove(e.OldFullPath);
|
|
_watcherChanges[e.FullPath] = new(WatcherChangeTypes.Renamed, e.OldFullPath);
|
|
}
|
|
|
|
Logger.LogTrace("FSW Renamed: {path} -> {new}", e.OldFullPath, e.FullPath);
|
|
}
|
|
|
|
_ = PenumbraWatcherExecution();
|
|
}
|
|
|
|
private CancellationTokenSource _penumbraFswCts = new();
|
|
private CancellationTokenSource _lightlessFswCts = new();
|
|
|
|
public FileSystemWatcher? PenumbraWatcher { get; private set; }
|
|
public FileSystemWatcher? LightlessWatcher { get; private set; }
|
|
|
|
private static bool HasAllowedExtension(string path)
|
|
{
|
|
if (string.IsNullOrEmpty(path))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var extension = Path.GetExtension(path);
|
|
return !string.IsNullOrEmpty(extension) && AllowedFileExtensionSet.Contains(extension);
|
|
}
|
|
|
|
private async Task LightlessWatcherExecution()
|
|
{
|
|
_lightlessFswCts = _lightlessFswCts.CancelRecreate();
|
|
var token = _lightlessFswCts.Token;
|
|
|
|
try
|
|
{
|
|
await Task.Delay(TimeSpan.FromSeconds(10), token).ConfigureAwait(false);
|
|
while (IsHalted() && !token.IsCancellationRequested)
|
|
await Task.Delay(250, token).ConfigureAwait(false);
|
|
}
|
|
catch (TaskCanceledException) { return; }
|
|
|
|
var changes = DrainLightlessChanges();
|
|
if (changes.Count > 0)
|
|
_ = HandleChangesAsync(changes, token);
|
|
}
|
|
private async Task HandleChangesAsync(Dictionary<string, WatcherChange> changes, CancellationToken token)
|
|
{
|
|
await _dbGate.WaitAsync(token).ConfigureAwait(false);
|
|
try
|
|
{
|
|
var deleted = changes.Where(c => c.Value.ChangeType == WatcherChangeTypes.Deleted).Select(c => c.Key);
|
|
var renamed = changes.Where(c => c.Value.ChangeType == WatcherChangeTypes.Renamed);
|
|
var remaining = changes.Where(c => c.Value.ChangeType != WatcherChangeTypes.Deleted).Select(c => c.Key);
|
|
|
|
foreach (var entry in deleted)
|
|
{
|
|
Logger.LogDebug("FSW Change: Deletion - {val}", entry);
|
|
}
|
|
|
|
foreach (var entry in renamed)
|
|
{
|
|
Logger.LogDebug("FSW Change: Renamed - {oldVal} => {val}", entry.Value.OldPath, entry.Key);
|
|
}
|
|
|
|
foreach (var entry in remaining)
|
|
{
|
|
Logger.LogDebug("FSW Change: Creation or Change - {val}", entry);
|
|
}
|
|
|
|
var allChanges = deleted
|
|
.Concat(renamed.Select(c => c.Value.OldPath!))
|
|
.Concat(renamed.Select(c => c.Key))
|
|
.Concat(remaining)
|
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
.ToArray();
|
|
|
|
_ = _fileDbManager.GetFileCachesByPaths(allChanges);
|
|
await _fileDbManager.WriteOutFullCsvAsync(token).ConfigureAwait(false);
|
|
}
|
|
finally
|
|
{
|
|
_dbGate.Release();
|
|
}
|
|
}
|
|
|
|
private async Task PenumbraWatcherExecution()
|
|
{
|
|
_penumbraFswCts = _penumbraFswCts.CancelRecreate();
|
|
var token = _penumbraFswCts.Token;
|
|
|
|
try
|
|
{
|
|
await Task.Delay(TimeSpan.FromSeconds(10), token).ConfigureAwait(false);
|
|
while (IsHalted() && !token.IsCancellationRequested)
|
|
await Task.Delay(250, token).ConfigureAwait(false);
|
|
}
|
|
catch (TaskCanceledException) { return; }
|
|
|
|
var changes = DrainPenumbraChanges();
|
|
if (changes.Count > 0)
|
|
_ = HandleChangesAsync(changes, token);
|
|
}
|
|
|
|
public void InvokeScan()
|
|
{
|
|
TotalFiles = 0;
|
|
TotalFilesStorage = 0;
|
|
Interlocked.Exchange(ref _currentFileProgress, 0);
|
|
|
|
_scanCancellationTokenSource = _scanCancellationTokenSource?.CancelRecreate() ?? new CancellationTokenSource();
|
|
var token = _scanCancellationTokenSource.Token;
|
|
|
|
_ = Task.Run(async () =>
|
|
{
|
|
TaskCompletionSource scanTcs = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
try
|
|
{
|
|
Logger.LogDebug("Starting Full File Scan");
|
|
|
|
while (IsHalted() && !token.IsCancellationRequested)
|
|
{
|
|
Logger.LogDebug("Scan is halted, waiting...");
|
|
await Task.Delay(250, token).ConfigureAwait(false);
|
|
}
|
|
|
|
var scanThread = new Thread(() =>
|
|
{
|
|
try
|
|
{
|
|
token.ThrowIfCancellationRequested();
|
|
|
|
_performanceCollector.LogPerformance(this, $"FullFileScan",
|
|
() => FullFileScan(token));
|
|
|
|
scanTcs.TrySetResult();
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
scanTcs.TrySetCanceled(token);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logger.LogError(ex, "Error during Full File Scan");
|
|
scanTcs.TrySetException(ex);
|
|
}
|
|
})
|
|
{
|
|
Priority = ThreadPriority.Lowest,
|
|
IsBackground = true,
|
|
Name = "LightlessSync.FullFileScan"
|
|
};
|
|
|
|
scanThread.Start();
|
|
|
|
using var _ = token.Register(() => scanTcs.TrySetCanceled(token));
|
|
|
|
await scanTcs.Task.ConfigureAwait(false);
|
|
}
|
|
catch (TaskCanceledException)
|
|
{
|
|
Logger.LogInformation("Full File Scan was canceled.");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logger.LogError(ex, "Unexpected error in InvokeScan task");
|
|
}
|
|
finally
|
|
{
|
|
TotalFiles = 0;
|
|
TotalFilesStorage = 0;
|
|
Interlocked.Exchange(ref _currentFileProgress, 0);
|
|
}
|
|
}, token);
|
|
}
|
|
|
|
public void RecalculateFileCacheSize(CancellationToken token)
|
|
{
|
|
if (IsHalted()) return;
|
|
|
|
if (string.IsNullOrEmpty(_configService.Current.CacheFolder) ||
|
|
!Directory.Exists(_configService.Current.CacheFolder))
|
|
{
|
|
FileCacheSize = 0;
|
|
return;
|
|
}
|
|
|
|
FileCacheSize = -1;
|
|
bool isWine = _dalamudUtil?.IsWine ?? false;
|
|
|
|
try
|
|
{
|
|
var drive = DriveInfo.GetDrives()
|
|
.FirstOrDefault(d => _configService.Current.CacheFolder
|
|
.StartsWith(d.Name, StringComparison.OrdinalIgnoreCase));
|
|
|
|
if (drive != null)
|
|
FileCacheDriveFree = drive.AvailableFreeSpace;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logger.LogWarning(ex, "Could not determine drive size for storage folder {folder}", _configService.Current.CacheFolder);
|
|
}
|
|
|
|
var cacheFolder = _configService.Current.CacheFolder;
|
|
var candidates = new List<CacheEvictionCandidate>();
|
|
long totalSize = 0;
|
|
totalSize += AddFolderCandidates(cacheFolder, candidates, token, isWine);
|
|
totalSize += AddFolderCandidates(Path.Combine(cacheFolder, "downscaled"), candidates, token, isWine);
|
|
totalSize += AddFolderCandidates(Path.Combine(cacheFolder, "decimated"), candidates, token, isWine);
|
|
|
|
FileCacheSize = totalSize;
|
|
|
|
var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d);
|
|
if (FileCacheSize < maxCacheInBytes)
|
|
return;
|
|
|
|
var maxCacheBuffer = maxCacheInBytes * 0.05d;
|
|
|
|
candidates.Sort(static (a, b) => a.LastAccessTime.CompareTo(b.LastAccessTime));
|
|
|
|
var evictionTarget = maxCacheInBytes - (long)maxCacheBuffer;
|
|
var index = 0;
|
|
while (FileCacheSize > evictionTarget && index < candidates.Count)
|
|
{
|
|
var oldestFile = candidates[index];
|
|
|
|
try
|
|
{
|
|
EvictCacheCandidate(oldestFile, cacheFolder);
|
|
FileCacheSize -= oldestFile.Size;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullPath);
|
|
}
|
|
|
|
index++;
|
|
}
|
|
}
|
|
|
|
public void ResetLocks()
|
|
{
|
|
HaltScanLocks.Clear();
|
|
}
|
|
|
|
private long AddFolderCandidates(string directory, List<CacheEvictionCandidate> candidates, CancellationToken token, bool isWine)
|
|
{
|
|
if (!Directory.Exists(directory))
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
long totalSize = 0;
|
|
foreach (var path in Directory.EnumerateFiles(directory))
|
|
{
|
|
token.ThrowIfCancellationRequested();
|
|
|
|
try
|
|
{
|
|
var file = new FileInfo(path);
|
|
var size = GetFileSizeOnDisk(file, isWine);
|
|
totalSize += size;
|
|
candidates.Add(new CacheEvictionCandidate(file.FullName, size, file.LastAccessTime));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logger.LogTrace(ex, "Error getting size for {file}", path);
|
|
}
|
|
}
|
|
|
|
return totalSize;
|
|
}
|
|
|
|
private long GetFileSizeOnDisk(FileInfo file, bool isWine)
|
|
{
|
|
if (isWine)
|
|
{
|
|
return file.Length;
|
|
}
|
|
|
|
try
|
|
{
|
|
return _fileCompactor.GetFileSizeOnDisk(file);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", file.FullName);
|
|
return file.Length;
|
|
}
|
|
}
|
|
|
|
private void EvictCacheCandidate(CacheEvictionCandidate candidate, string cacheFolder)
|
|
{
|
|
if (TryGetCacheHashAndPrefixedPath(candidate.FullPath, cacheFolder, out var hash, out var prefixedPath))
|
|
{
|
|
_fileDbManager.RemoveHashedFile(hash, prefixedPath);
|
|
}
|
|
|
|
try
|
|
{
|
|
if (File.Exists(candidate.FullPath))
|
|
{
|
|
File.Delete(candidate.FullPath);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logger.LogTrace(ex, "Failed to delete old file {file}", candidate.FullPath);
|
|
}
|
|
}
|
|
|
|
private static bool TryGetCacheHashAndPrefixedPath(string filePath, string cacheFolder, out string hash, out string prefixedPath)
|
|
{
|
|
hash = string.Empty;
|
|
prefixedPath = string.Empty;
|
|
|
|
if (string.IsNullOrEmpty(cacheFolder))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var fileName = Path.GetFileNameWithoutExtension(filePath);
|
|
if (string.IsNullOrEmpty(fileName) || !IsSha1Hash(fileName))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var relative = Path.GetRelativePath(cacheFolder, filePath)
|
|
.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
|
|
var sanitizedRelative = relative.TrimStart(Path.DirectorySeparatorChar);
|
|
prefixedPath = Path.Combine(FileCacheManager.CachePrefix, sanitizedRelative);
|
|
hash = fileName;
|
|
return true;
|
|
}
|
|
|
|
private static bool IsSha1Hash(string value)
|
|
{
|
|
if (value.Length != 40)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
foreach (var ch in value)
|
|
{
|
|
if (!Uri.IsHexDigit(ch))
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
public void ResumeScan(string source)
|
|
{
|
|
int delta = 0;
|
|
|
|
HaltScanLocks.AddOrUpdate(source,
|
|
addValueFactory: _ => 0,
|
|
updateValueFactory: (_, v) =>
|
|
{
|
|
ArgumentException.ThrowIfNullOrEmpty(_);
|
|
if (v <= 0) return 0;
|
|
delta = 1;
|
|
return v - 1;
|
|
});
|
|
|
|
if (delta == 1)
|
|
Interlocked.Decrement(ref _haltCount);
|
|
}
|
|
|
|
protected override void Dispose(bool disposing)
|
|
{
|
|
base.Dispose(disposing);
|
|
// Disposing of file system watchers
|
|
PenumbraWatcher?.Dispose();
|
|
LightlessWatcher?.Dispose();
|
|
|
|
// Disposing of cancellation token sources
|
|
_scanCancellationTokenSource?.CancelDispose();
|
|
_scanCancellationTokenSource?.Dispose();
|
|
_penumbraFswCts?.CancelDispose();
|
|
_penumbraFswCts?.Dispose();
|
|
_lightlessFswCts?.CancelDispose();
|
|
_lightlessFswCts?.Dispose();
|
|
_periodicCalculationTokenSource?.CancelDispose();
|
|
_periodicCalculationTokenSource?.Dispose();
|
|
}
|
|
|
|
private void FullFileScan(CancellationToken ct)
|
|
{
|
|
TotalFiles = 1;
|
|
_currentFileProgress = 0;
|
|
|
|
var penumbraDir = _ipcManager.Penumbra.ModDirectory;
|
|
var cacheFolder = _configService.Current.CacheFolder;
|
|
|
|
if (string.IsNullOrEmpty(penumbraDir) || !Directory.Exists(penumbraDir))
|
|
{
|
|
Logger.LogWarning("Penumbra directory is not set or does not exist.");
|
|
return;
|
|
}
|
|
|
|
if (string.IsNullOrEmpty(cacheFolder) || !Directory.Exists(cacheFolder))
|
|
{
|
|
Logger.LogWarning("Lightless Cache directory is not set or does not exist.");
|
|
return;
|
|
}
|
|
|
|
var prevPriority = Thread.CurrentThread.Priority;
|
|
Thread.CurrentThread.Priority = ThreadPriority.Lowest;
|
|
|
|
try
|
|
{
|
|
Logger.LogDebug("Getting files from {penumbra} and {storage}", penumbraDir, cacheFolder);
|
|
|
|
var onDiskPaths = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
|
|
static bool IsExcludedPenumbraPath(string path)
|
|
=> path.Contains(@"\bg\", StringComparison.OrdinalIgnoreCase)
|
|
|| path.Contains(@"\bgcommon\", StringComparison.OrdinalIgnoreCase)
|
|
|| path.Contains(@"\ui\", StringComparison.OrdinalIgnoreCase);
|
|
|
|
foreach (var folder in Directory.EnumerateDirectories(penumbraDir))
|
|
{
|
|
ct.ThrowIfCancellationRequested();
|
|
|
|
try
|
|
{
|
|
foreach (var file in Directory.EnumerateFiles(folder, "*.*", SearchOption.AllDirectories))
|
|
{
|
|
ct.ThrowIfCancellationRequested();
|
|
|
|
if (!HasAllowedExtension(file)) continue;
|
|
if (IsExcludedPenumbraPath(file)) continue;
|
|
|
|
onDiskPaths.Add(file);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logger.LogWarning(ex, "Could not enumerate path {path}", folder);
|
|
}
|
|
}
|
|
|
|
foreach (var file in Directory.EnumerateFiles(cacheFolder, "*.*", SearchOption.TopDirectoryOnly))
|
|
{
|
|
ct.ThrowIfCancellationRequested();
|
|
|
|
var name = Path.GetFileName(file);
|
|
var stem = Path.GetFileNameWithoutExtension(file);
|
|
|
|
if (name.Length == 40 || stem.Length == 40)
|
|
onDiskPaths.Add(file);
|
|
}
|
|
|
|
var threadCount = Math.Clamp((int)(Environment.ProcessorCount / 2.0f), 2, 8);
|
|
|
|
var fileCacheList = _fileDbManager.GetAllFileCaches();
|
|
var fileCaches = new ConcurrentQueue<FileCacheEntity>(fileCacheList);
|
|
|
|
TotalFilesStorage = fileCaches.Count;
|
|
TotalFiles = onDiskPaths.Count + TotalFilesStorage;
|
|
|
|
var validOrPresentInDb = new ConcurrentDictionary<string, byte>(StringComparer.OrdinalIgnoreCase);
|
|
var entitiesToUpdate = new ConcurrentBag<FileCacheEntity>();
|
|
var entitiesToRemove = new ConcurrentBag<FileCacheEntity>();
|
|
|
|
if (!_ipcManager.Penumbra.APIAvailable)
|
|
{
|
|
Logger.LogWarning("Penumbra not available");
|
|
return;
|
|
}
|
|
|
|
Thread[] workerThreads = new Thread[threadCount];
|
|
for (int i = 0; i < threadCount; i++)
|
|
{
|
|
workerThreads[i] = new Thread(tcounter =>
|
|
{
|
|
var threadNr = (int)tcounter!;
|
|
Logger.LogTrace("Spawning Worker Thread {i}", threadNr);
|
|
|
|
while (!ct.IsCancellationRequested && fileCaches.TryDequeue(out var workload))
|
|
{
|
|
try
|
|
{
|
|
if (ct.IsCancellationRequested) break;
|
|
|
|
if (!_ipcManager.Penumbra.APIAvailable)
|
|
break;
|
|
|
|
var validated = _fileDbManager.ValidateFileCacheEntity(workload);
|
|
|
|
if (validated.State != FileState.RequireDeletion)
|
|
{
|
|
validOrPresentInDb.TryAdd(validated.FileCache.ResolvedFilepath, 0);
|
|
}
|
|
|
|
if (validated.State == FileState.RequireUpdate)
|
|
{
|
|
Logger.LogTrace("To update: {path}", validated.FileCache.ResolvedFilepath);
|
|
entitiesToUpdate.Add(validated.FileCache);
|
|
}
|
|
else if (validated.State == FileState.RequireDeletion)
|
|
{
|
|
Logger.LogTrace("To delete: {path}", validated.FileCache.ResolvedFilepath);
|
|
entitiesToRemove.Add(validated.FileCache);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (workload != null)
|
|
Logger.LogWarning(ex, "Failed validating {path}", workload.ResolvedFilepath);
|
|
else
|
|
Logger.LogWarning(ex, "Failed validating unknown workload");
|
|
}
|
|
finally
|
|
{
|
|
Interlocked.Increment(ref _currentFileProgress);
|
|
}
|
|
}
|
|
|
|
Logger.LogTrace("Ending Worker Thread {i}", threadNr);
|
|
})
|
|
{
|
|
Priority = ThreadPriority.Lowest,
|
|
IsBackground = true
|
|
};
|
|
|
|
workerThreads[i].Start(i);
|
|
}
|
|
|
|
while (!ct.IsCancellationRequested && workerThreads.Any(t => t.IsAlive))
|
|
{
|
|
ct.WaitHandle.WaitOne(250);
|
|
}
|
|
|
|
if (ct.IsCancellationRequested) return;
|
|
|
|
Logger.LogTrace("Scanner validated existing db files");
|
|
|
|
if (!_ipcManager.Penumbra.APIAvailable)
|
|
{
|
|
Logger.LogWarning("Penumbra not available");
|
|
return;
|
|
}
|
|
|
|
var didMutateDb = false;
|
|
|
|
foreach (var entity in entitiesToUpdate)
|
|
{
|
|
didMutateDb = true;
|
|
_fileDbManager.UpdateHashedFile(entity);
|
|
}
|
|
|
|
foreach (var entity in entitiesToRemove)
|
|
{
|
|
didMutateDb = true;
|
|
_fileDbManager.RemoveHashedFile(entity.Hash, entity.PrefixedFilePath);
|
|
}
|
|
|
|
if (didMutateDb)
|
|
_fileDbManager.WriteOutFullCsv();
|
|
|
|
if (ct.IsCancellationRequested) return;
|
|
|
|
var newFiles = onDiskPaths.Where(p => !validOrPresentInDb.ContainsKey(p)).ToList();
|
|
|
|
foreach (var path in newFiles)
|
|
{
|
|
if (ct.IsCancellationRequested) break;
|
|
ProcessOne(path);
|
|
Interlocked.Increment(ref _currentFileProgress);
|
|
}
|
|
|
|
Logger.LogTrace("Scanner added {count} new files to db", newFiles.Count);
|
|
|
|
void ProcessOne(string? filePath)
|
|
{
|
|
if (filePath == null)
|
|
return;
|
|
|
|
if (!_ipcManager.Penumbra.APIAvailable)
|
|
{
|
|
Logger.LogWarning("Penumbra not available");
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
var entry = _fileDbManager.CreateFileEntry(filePath);
|
|
if (entry == null)
|
|
_ = _fileDbManager.CreateCacheEntry(filePath);
|
|
}
|
|
catch (IOException ioex)
|
|
{
|
|
Logger.LogDebug(ioex, "File busy or locked: {file}", filePath);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logger.LogWarning(ex, "Failed adding {file}", filePath);
|
|
}
|
|
}
|
|
|
|
Logger.LogDebug("Scan complete");
|
|
|
|
TotalFiles = 0;
|
|
_currentFileProgress = 0;
|
|
|
|
if (!_configService.Current.InitialScanComplete)
|
|
{
|
|
_configService.Current.InitialScanComplete = true;
|
|
_configService.Save();
|
|
|
|
StartLightlessWatcher(cacheFolder);
|
|
StartPenumbraWatcher(penumbraDir);
|
|
}
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
// normal cancellation
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logger.LogError(ex, "Error during Full File Scan");
|
|
}
|
|
finally
|
|
{
|
|
Thread.CurrentThread.Priority = prevPriority;
|
|
}
|
|
}
|
|
} |