Initial
This commit is contained in:
Zurazan
@@ -0,0 +1,214 @@
|
||||
using LightlessSyncShared.Metrics;
|
||||
using LightlessSyncShared.Services;
|
||||
using LightlessSyncStaticFilesServer.Utils;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net.Http.Headers;
|
||||
using LightlessSyncShared.Utils;
|
||||
using LightlessSync.API.Routes;
|
||||
using LightlessSyncShared.Utils.Configuration;
|
||||
|
||||
namespace LightlessSyncStaticFilesServer.Services;
|
||||
|
||||
public sealed class CachedFileProvider : IDisposable
|
||||
{
|
||||
private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
|
||||
private readonly ILogger<CachedFileProvider> _logger;
|
||||
private readonly FileStatisticsService _fileStatisticsService;
|
||||
private readonly LightlessMetrics _metrics;
|
||||
private readonly ServerTokenGenerator _generator;
|
||||
private readonly Uri _remoteCacheSourceUri;
|
||||
private readonly string _hotStoragePath;
|
||||
private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly SemaphoreSlim _downloadSemaphore = new(1, 1);
|
||||
private bool _disposed;
|
||||
|
||||
private bool IsMainServer => _remoteCacheSourceUri == null && _isDistributionServer;
|
||||
private bool _isDistributionServer;
|
||||
|
||||
public CachedFileProvider(IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger,
|
||||
FileStatisticsService fileStatisticsService, LightlessMetrics metrics, ServerTokenGenerator generator)
|
||||
{
|
||||
_configuration = configuration;
|
||||
_logger = logger;
|
||||
_fileStatisticsService = fileStatisticsService;
|
||||
_metrics = metrics;
|
||||
_generator = generator;
|
||||
_remoteCacheSourceUri = configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.DistributionFileServerAddress), null);
|
||||
_isDistributionServer = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false);
|
||||
_hotStoragePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
|
||||
_httpClient = new();
|
||||
_httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("LightlessSyncServer", "1.0.0.0"));
|
||||
_httpClient.Timeout = TimeSpan.FromSeconds(300);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
_httpClient?.Dispose();
|
||||
}
|
||||
|
||||
private async Task DownloadTask(string hash)
|
||||
{
|
||||
var destinationFilePath = FilePathUtil.GetFilePath(_hotStoragePath, hash);
|
||||
|
||||
// first check cold storage
|
||||
if (TryCopyFromColdStorage(hash, destinationFilePath)) return;
|
||||
|
||||
// if cold storage is not configured or file not found or error is present try to download file from remote
|
||||
var downloadUrl = LightlessFiles.DistributionGetFullPath(_remoteCacheSourceUri, hash);
|
||||
_logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl);
|
||||
|
||||
using var requestMessage = new HttpRequestMessage(HttpMethod.Get, downloadUrl);
|
||||
requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _generator.Token);
|
||||
HttpResponseMessage? response = null;
|
||||
|
||||
try
|
||||
{
|
||||
response = await _httpClient.SendAsync(requestMessage).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to download {url}", downloadUrl);
|
||||
response?.Dispose();
|
||||
return;
|
||||
}
|
||||
|
||||
var tempFileName = destinationFilePath + ".dl";
|
||||
var fileStream = new FileStream(tempFileName, FileMode.Create, FileAccess.ReadWrite);
|
||||
var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024;
|
||||
var buffer = new byte[bufferSize];
|
||||
|
||||
var bytesRead = 0;
|
||||
using var content = await response.Content.ReadAsStreamAsync().ConfigureAwait(false);
|
||||
while ((bytesRead = await content.ReadAsync(buffer).ConfigureAwait(false)) > 0)
|
||||
{
|
||||
await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead)).ConfigureAwait(false);
|
||||
}
|
||||
await fileStream.FlushAsync().ConfigureAwait(false);
|
||||
await fileStream.DisposeAsync().ConfigureAwait(false);
|
||||
File.Move(tempFileName, destinationFilePath, true);
|
||||
|
||||
_metrics.IncGauge(MetricsAPI.GaugeFilesTotal);
|
||||
_metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash).Length);
|
||||
response.Dispose();
|
||||
}
|
||||
|
||||
private bool TryCopyFromColdStorage(string hash, string destinationFilePath)
|
||||
{
|
||||
if (!_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false)) return false;
|
||||
|
||||
string coldStorageDir = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageDirectory), string.Empty);
|
||||
if (string.IsNullOrEmpty(coldStorageDir)) return false;
|
||||
|
||||
var coldStorageFilePath = FilePathUtil.GetFileInfoForHash(coldStorageDir, hash);
|
||||
if (coldStorageFilePath == null) return false;
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogDebug("Copying {hash} from cold storage: {path}", hash, coldStorageFilePath);
|
||||
var tempFileName = destinationFilePath + ".dl";
|
||||
File.Copy(coldStorageFilePath.FullName, tempFileName, true);
|
||||
File.Move(tempFileName, destinationFilePath, true);
|
||||
coldStorageFilePath.LastAccessTimeUtc = DateTime.UtcNow;
|
||||
var destinationFile = new FileInfo(destinationFilePath);
|
||||
destinationFile.LastAccessTimeUtc = DateTime.UtcNow;
|
||||
destinationFile.CreationTimeUtc = DateTime.UtcNow;
|
||||
destinationFile.LastWriteTimeUtc = DateTime.UtcNow;
|
||||
_metrics.IncGauge(MetricsAPI.GaugeFilesTotal);
|
||||
_metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, new FileInfo(destinationFilePath).Length);
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Could not copy {coldStoragePath} from cold storage", coldStorageFilePath);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public async Task DownloadFileWhenRequired(string hash)
|
||||
{
|
||||
var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash);
|
||||
if (fi == null && IsMainServer)
|
||||
{
|
||||
TryCopyFromColdStorage(hash, FilePathUtil.GetFilePath(_hotStoragePath, hash));
|
||||
return;
|
||||
}
|
||||
|
||||
await _downloadSemaphore.WaitAsync().ConfigureAwait(false);
|
||||
if ((fi == null || (fi?.Length ?? 0) == 0)
|
||||
&& (!_currentTransfers.TryGetValue(hash, out var downloadTask)
|
||||
|| (downloadTask?.IsCompleted ?? true)))
|
||||
{
|
||||
_currentTransfers[hash] = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
_metrics.IncGauge(MetricsAPI.GaugeFilesDownloadingFromCache);
|
||||
await DownloadTask(hash).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error during Download Task for {hash}", hash);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_metrics.DecGauge(MetricsAPI.GaugeFilesDownloadingFromCache);
|
||||
_currentTransfers.Remove(hash, out _);
|
||||
}
|
||||
});
|
||||
}
|
||||
_downloadSemaphore.Release();
|
||||
}
|
||||
|
||||
public FileInfo? GetLocalFilePath(string hash)
|
||||
{
|
||||
var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash);
|
||||
if (fi == null) return null;
|
||||
|
||||
fi.LastAccessTimeUtc = DateTime.UtcNow;
|
||||
|
||||
_fileStatisticsService.LogFile(hash, fi.Length);
|
||||
|
||||
return new FileInfo(fi.FullName);
|
||||
}
|
||||
|
||||
public async Task<FileInfo?> DownloadAndGetLocalFileInfo(string hash)
|
||||
{
|
||||
await DownloadFileWhenRequired(hash).ConfigureAwait(false);
|
||||
|
||||
if (_currentTransfers.TryGetValue(hash, out var downloadTask))
|
||||
{
|
||||
try
|
||||
{
|
||||
using CancellationTokenSource cts = new();
|
||||
cts.CancelAfter(TimeSpan.FromSeconds(300));
|
||||
_metrics.IncGauge(MetricsAPI.GaugeFilesTasksWaitingForDownloadFromCache);
|
||||
await downloadTask.WaitAsync(cts.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogWarning(e, "Failed while waiting for download task for {hash}", hash);
|
||||
return null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_metrics.DecGauge(MetricsAPI.GaugeFilesTasksWaitingForDownloadFromCache);
|
||||
}
|
||||
}
|
||||
|
||||
return GetLocalFilePath(hash);
|
||||
}
|
||||
|
||||
public bool AnyFilesDownloading(List<string> hashes)
|
||||
{
|
||||
return hashes.Exists(_currentTransfers.Keys.Contains);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
using LightlessSyncShared.Metrics;
|
||||
using System.Collections.Concurrent;
|
||||
|
||||
namespace LightlessSyncStaticFilesServer.Services;
|
||||
|
||||
public class FileStatisticsService : IHostedService
|
||||
{
|
||||
private readonly LightlessMetrics _metrics;
|
||||
private readonly ILogger<FileStatisticsService> _logger;
|
||||
private CancellationTokenSource _resetCancellationTokenSource;
|
||||
private ConcurrentDictionary<string, long> _pastHourFiles = new(StringComparer.Ordinal);
|
||||
private ConcurrentDictionary<string, long> _pastDayFiles = new(StringComparer.Ordinal);
|
||||
|
||||
public FileStatisticsService(LightlessMetrics metrics, ILogger<FileStatisticsService> logger)
|
||||
{
|
||||
_metrics = metrics;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public void LogFile(string fileHash, long length)
|
||||
{
|
||||
if (!_pastHourFiles.ContainsKey(fileHash))
|
||||
{
|
||||
_pastHourFiles[fileHash] = length;
|
||||
_metrics.IncGauge(MetricsAPI.GaugeFilesUniquePastHour);
|
||||
_metrics.IncGauge(MetricsAPI.GaugeFilesUniquePastHourSize, length);
|
||||
}
|
||||
if (!_pastDayFiles.ContainsKey(fileHash))
|
||||
{
|
||||
_pastDayFiles[fileHash] = length;
|
||||
_metrics.IncGauge(MetricsAPI.GaugeFilesUniquePastDay);
|
||||
_metrics.IncGauge(MetricsAPI.GaugeFilesUniquePastDaySize, length);
|
||||
}
|
||||
}
|
||||
|
||||
public void LogRequest(long requestSize)
|
||||
{
|
||||
_metrics.IncCounter(MetricsAPI.CounterFileRequests, 1);
|
||||
_metrics.IncCounter(MetricsAPI.CounterFileRequestSize, requestSize);
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("Starting FileStatisticsService");
|
||||
_resetCancellationTokenSource = new();
|
||||
_ = ResetHourlyFileData();
|
||||
_ = ResetDailyFileData();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async Task ResetHourlyFileData()
|
||||
{
|
||||
while (!_resetCancellationTokenSource.Token.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogInformation("Resetting 1h Data");
|
||||
|
||||
_pastHourFiles = new(StringComparer.Ordinal);
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHour, 0);
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHourSize, 0);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
|
||||
TimeOnly futureTime = new(now.Hour, 0, 0);
|
||||
var span = futureTime.AddHours(1) - currentTime;
|
||||
|
||||
await Task.Delay(span, _resetCancellationTokenSource.Token).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task ResetDailyFileData()
|
||||
{
|
||||
while (!_resetCancellationTokenSource.Token.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogInformation("Resetting 24h Data");
|
||||
|
||||
_pastDayFiles = new(StringComparer.Ordinal);
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDay, 0);
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDaySize, 0);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
DateTime midnight = new(new DateOnly(now.Date.Year, now.Date.Month, now.Date.Day), new(0, 0, 0));
|
||||
var span = midnight.AddDays(1) - now;
|
||||
|
||||
await Task.Delay(span, _resetCancellationTokenSource.Token).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_resetCancellationTokenSource.Cancel();
|
||||
_logger.LogInformation("Stopping FileStatisticsService");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace LightlessSyncStaticFilesServer.Services;
|
||||
|
||||
public interface IClientReadyMessageService
|
||||
{
|
||||
Task SendDownloadReady(string uid, Guid requestId);
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using LightlessSync.API.SignalR;
|
||||
using LightlessSyncServer.Hubs;
|
||||
|
||||
namespace LightlessSyncStaticFilesServer.Services;
|
||||
|
||||
public class MainClientReadyMessageService : IClientReadyMessageService
|
||||
{
|
||||
private readonly ILogger<MainClientReadyMessageService> _logger;
|
||||
private readonly IHubContext<LightlessHub> _lightlessHub;
|
||||
|
||||
public MainClientReadyMessageService(ILogger<MainClientReadyMessageService> logger, IHubContext<LightlessHub> lightlessHub)
|
||||
{
|
||||
_logger = logger;
|
||||
_lightlessHub = lightlessHub;
|
||||
}
|
||||
|
||||
public async Task SendDownloadReady(string uid, Guid requestId)
|
||||
{
|
||||
_logger.LogInformation("Sending Client Ready for {uid}:{requestId} to SignalR", uid, requestId);
|
||||
await _lightlessHub.Clients.User(uid).SendAsync(nameof(ILightlessHub.Client_DownloadReady), requestId).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
using ByteSizeLib;
|
||||
using K4os.Compression.LZ4.Legacy;
|
||||
using LightlessSyncShared.Data;
|
||||
using LightlessSyncShared.Metrics;
|
||||
using LightlessSyncShared.Models;
|
||||
using LightlessSyncShared.Services;
|
||||
using LightlessSyncShared.Utils.Configuration;
|
||||
using LightlessSyncStaticFilesServer.Utils;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace LightlessSyncStaticFilesServer.Services;
|
||||
|
||||
public class MainFileCleanupService : IHostedService
|
||||
{
|
||||
private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
|
||||
private readonly IDbContextFactory<LightlessDbContext> _dbContextFactory;
|
||||
private readonly ILogger<MainFileCleanupService> _logger;
|
||||
private readonly LightlessMetrics _metrics;
|
||||
private CancellationTokenSource _cleanupCts;
|
||||
|
||||
public MainFileCleanupService(LightlessMetrics metrics, ILogger<MainFileCleanupService> logger,
|
||||
IConfigurationService<StaticFilesServerConfiguration> configuration,
|
||||
IDbContextFactory<LightlessDbContext> dbContextFactory)
|
||||
{
|
||||
_metrics = metrics;
|
||||
_logger = logger;
|
||||
_configuration = configuration;
|
||||
_dbContextFactory = dbContextFactory;
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("Cleanup Service started");
|
||||
|
||||
_cleanupCts = new();
|
||||
|
||||
_ = Task.Run(() => CleanUpTask(_cleanupCts.Token)).ConfigureAwait(false);
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_cleanupCts.Cancel();
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private List<FileInfo> CleanUpFilesBeyondSizeLimit(List<FileInfo> files, double sizeLimit, bool deleteFromDb, LightlessDbContext dbContext, CancellationToken ct)
|
||||
{
|
||||
if (sizeLimit <= 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", sizeLimit);
|
||||
var allLocalFiles = files
|
||||
.OrderBy(f => f.LastAccessTimeUtc).ToList();
|
||||
var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
|
||||
long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes;
|
||||
while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Count != 0 && !ct.IsCancellationRequested)
|
||||
{
|
||||
var oldestFile = allLocalFiles[0];
|
||||
allLocalFiles.RemoveAt(0);
|
||||
totalCacheSizeInBytes -= oldestFile.Length;
|
||||
_logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes);
|
||||
oldestFile.Delete();
|
||||
FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() };
|
||||
if (deleteFromDb)
|
||||
dbContext.Entry(f).State = EntityState.Deleted;
|
||||
}
|
||||
|
||||
return allLocalFiles;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error during cache size limit cleanup");
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
private List<FileInfo> CleanUpOrphanedFiles(List<FileCache> allFiles, List<FileInfo> allPhysicalFiles, CancellationToken ct)
|
||||
{
|
||||
var allFilesHashes = new HashSet<string>(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal);
|
||||
foreach (var file in allPhysicalFiles.ToList())
|
||||
{
|
||||
if (!allFilesHashes.Contains(file.Name.ToUpperInvariant()))
|
||||
{
|
||||
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
|
||||
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
|
||||
file.Delete();
|
||||
_logger.LogInformation("File not in DB, deleting: {fileName}", file.Name);
|
||||
allPhysicalFiles.Remove(file);
|
||||
}
|
||||
|
||||
ct.ThrowIfCancellationRequested();
|
||||
}
|
||||
|
||||
return allPhysicalFiles;
|
||||
}
|
||||
|
||||
private async Task<List<FileInfo>> CleanUpOutdatedFiles(string dir, List<FileInfo> allFilesInDir, int unusedRetention, int forcedDeletionAfterHours,
|
||||
bool deleteFromDb, LightlessDbContext dbContext, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", unusedRetention);
|
||||
if (forcedDeletionAfterHours > 0)
|
||||
{
|
||||
_logger.LogInformation("Cleaning up files written to longer than {hours}h ago", forcedDeletionAfterHours);
|
||||
}
|
||||
|
||||
// clean up files in DB but not on disk or last access is expired
|
||||
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention));
|
||||
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
|
||||
List<FileCache> allDbFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false);
|
||||
List<string> removedFileHashes;
|
||||
|
||||
if (!deleteFromDb)
|
||||
{
|
||||
removedFileHashes = CleanupViaFiles(allFilesInDir, forcedDeletionAfterHours, prevTime, prevTimeForcedDeletion, ct);
|
||||
}
|
||||
else
|
||||
{
|
||||
removedFileHashes = await CleanupViaDb(dir, forcedDeletionAfterHours, dbContext, prevTime, prevTimeForcedDeletion, allDbFiles, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// clean up files that are on disk but not in DB anymore
|
||||
return CleanUpOrphanedFiles(allDbFiles, allFilesInDir.Where(c => !removedFileHashes.Contains(c.Name, StringComparer.OrdinalIgnoreCase)).ToList(), ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error during file cleanup of old files");
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
private void CleanUpStuckUploads(LightlessDbContext dbContext)
|
||||
{
|
||||
var pastTime = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(20));
|
||||
var stuckUploads = dbContext.Files.Where(f => !f.Uploaded && f.UploadDate < pastTime);
|
||||
dbContext.Files.RemoveRange(stuckUploads);
|
||||
}
|
||||
|
||||
private async Task CleanUpTask(CancellationToken ct)
|
||||
{
|
||||
InitializeGauges();
|
||||
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
var cleanupCheckMinutes = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CleanupCheckInMinutes), 15);
|
||||
bool useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
|
||||
var hotStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
|
||||
var coldStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
|
||||
using var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation("Running File Cleanup Task");
|
||||
|
||||
try
|
||||
{
|
||||
using CancellationTokenSource timedCts = new();
|
||||
timedCts.CancelAfter(TimeSpan.FromMinutes(cleanupCheckMinutes - 1));
|
||||
using var linkedTokenCts = CancellationTokenSource.CreateLinkedTokenSource(timedCts.Token, ct);
|
||||
var linkedToken = linkedTokenCts.Token;
|
||||
|
||||
DirectoryInfo dirHotStorage = new(hotStorageDir);
|
||||
_logger.LogInformation("File Cleanup Task gathering hot storage files");
|
||||
var allFilesInHotStorage = dirHotStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
|
||||
|
||||
var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
|
||||
var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
|
||||
var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);
|
||||
|
||||
_logger.LogInformation("File Cleanup Task cleaning up outdated hot storage files");
|
||||
var remainingHotFiles = await CleanUpOutdatedFiles(hotStorageDir, allFilesInHotStorage, unusedRetention, forcedDeletionAfterHours,
|
||||
deleteFromDb: !useColdStorage, dbContext: dbContext,
|
||||
ct: linkedToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation("File Cleanup Task cleaning up hot storage file beyond size limit");
|
||||
var finalRemainingHotFiles = CleanUpFilesBeyondSizeLimit(remainingHotFiles, sizeLimit,
|
||||
deleteFromDb: !useColdStorage, dbContext: dbContext,
|
||||
ct: linkedToken);
|
||||
|
||||
if (useColdStorage)
|
||||
{
|
||||
DirectoryInfo dirColdStorage = new(coldStorageDir);
|
||||
_logger.LogInformation("File Cleanup Task gathering cold storage files");
|
||||
var allFilesInColdStorageDir = dirColdStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
|
||||
|
||||
var coldStorageRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60);
|
||||
var coldStorageSize = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1);
|
||||
|
||||
// clean up cold storage
|
||||
_logger.LogInformation("File Cleanup Task cleaning up outdated cold storage files");
|
||||
var remainingColdFiles = await CleanUpOutdatedFiles(coldStorageDir, allFilesInColdStorageDir, coldStorageRetention, forcedDeletionAfterHours: -1,
|
||||
deleteFromDb: true, dbContext: dbContext,
|
||||
ct: linkedToken).ConfigureAwait(false);
|
||||
_logger.LogInformation("File Cleanup Task cleaning up cold storage file beyond size limit");
|
||||
var finalRemainingColdFiles = CleanUpFilesBeyondSizeLimit(remainingColdFiles, coldStorageSize,
|
||||
deleteFromDb: true, dbContext: dbContext,
|
||||
ct: linkedToken);
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "Error during cleanup task");
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanUpStuckUploads(dbContext);
|
||||
|
||||
await dbContext.SaveChangesAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (useColdStorage)
|
||||
{
|
||||
DirectoryInfo dirColdStorageAfterCleanup = new(coldStorageDir);
|
||||
var allFilesInColdStorageAfterCleanup = dirColdStorageAfterCleanup.GetFiles("*", SearchOption.AllDirectories).ToList();
|
||||
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, allFilesInColdStorageAfterCleanup.Sum(f => { try { return f.Length; } catch { return 0; } }));
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, allFilesInColdStorageAfterCleanup.Count);
|
||||
}
|
||||
|
||||
DirectoryInfo dirHotStorageAfterCleanup = new(hotStorageDir);
|
||||
var allFilesInHotStorageAfterCleanup = dirHotStorageAfterCleanup.GetFiles("*", SearchOption.AllDirectories).ToList();
|
||||
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFilesInHotStorageAfterCleanup.Sum(f => { try { return f.Length; } catch { return 0; } }));
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFilesInHotStorageAfterCleanup.Count);
|
||||
|
||||
var now = DateTime.Now;
|
||||
TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
|
||||
TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % cleanupCheckMinutes, 0);
|
||||
var span = futureTime.AddMinutes(cleanupCheckMinutes) - currentTime;
|
||||
|
||||
_logger.LogInformation("File Cleanup Complete, next run at {date}", now.Add(span));
|
||||
await Task.Delay(span, ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
private async Task<List<string>> CleanupViaDb(string dir, int forcedDeletionAfterHours,
|
||||
LightlessDbContext dbContext, DateTime lastAccessCutoffTime, DateTime forcedDeletionCutoffTime, List<FileCache> allDbFiles, CancellationToken ct)
|
||||
{
|
||||
int fileCounter = 0;
|
||||
List<string> removedFileHashes = new();
|
||||
foreach (var fileCache in allDbFiles.Where(f => f.Uploaded))
|
||||
{
|
||||
bool deleteCurrentFile = false;
|
||||
var file = FilePathUtil.GetFileInfoForHash(dir, fileCache.Hash);
|
||||
if (file == null)
|
||||
{
|
||||
_logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash);
|
||||
deleteCurrentFile = true;
|
||||
}
|
||||
else if (file != null && file.LastAccessTime < lastAccessCutoffTime)
|
||||
{
|
||||
_logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
|
||||
deleteCurrentFile = true;
|
||||
}
|
||||
else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < forcedDeletionCutoffTime)
|
||||
{
|
||||
_logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
|
||||
deleteCurrentFile = true;
|
||||
}
|
||||
|
||||
// only used if file in db has no raw size for whatever reason
|
||||
if (!deleteCurrentFile && file != null && fileCache.RawSize == 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
var length = LZ4Wrapper.Unwrap(File.ReadAllBytes(file.FullName)).LongLength;
|
||||
_logger.LogInformation("Setting Raw File Size of " + fileCache.Hash + " to " + length);
|
||||
fileCache.RawSize = length;
|
||||
if (fileCounter % 1000 == 0)
|
||||
await dbContext.SaveChangesAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Could not unpack {fileName}", file.FullName);
|
||||
}
|
||||
}
|
||||
|
||||
// do actual deletion of file and remove also from db if needed
|
||||
if (deleteCurrentFile)
|
||||
{
|
||||
if (file != null) file.Delete();
|
||||
|
||||
removedFileHashes.Add(fileCache.Hash);
|
||||
|
||||
dbContext.Files.Remove(fileCache);
|
||||
}
|
||||
|
||||
// only used if file in db has no size for whatever reason
|
||||
if (!deleteCurrentFile && file != null && fileCache.Size == 0)
|
||||
{
|
||||
_logger.LogInformation("Setting File Size of " + fileCache.Hash + " to " + file.Length);
|
||||
fileCache.Size = file.Length;
|
||||
// commit every 1000 files to db
|
||||
if (fileCounter % 1000 == 0)
|
||||
await dbContext.SaveChangesAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
fileCounter++;
|
||||
|
||||
ct.ThrowIfCancellationRequested();
|
||||
}
|
||||
|
||||
return removedFileHashes;
|
||||
}
|
||||
|
||||
private List<string> CleanupViaFiles(List<FileInfo> allFilesInDir, int forcedDeletionAfterHours,
|
||||
DateTime lastAccessCutoffTime, DateTime forcedDeletionCutoffTime, CancellationToken ct)
|
||||
{
|
||||
List<string> removedFileHashes = new List<string>();
|
||||
|
||||
foreach (var file in allFilesInDir)
|
||||
{
|
||||
bool deleteCurrentFile = false;
|
||||
if (file != null && file.LastAccessTime < lastAccessCutoffTime)
|
||||
{
|
||||
_logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
|
||||
deleteCurrentFile = true;
|
||||
}
|
||||
else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < forcedDeletionCutoffTime)
|
||||
{
|
||||
_logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
|
||||
deleteCurrentFile = true;
|
||||
}
|
||||
|
||||
if (deleteCurrentFile)
|
||||
{
|
||||
if (file != null) file.Delete();
|
||||
|
||||
removedFileHashes.Add(file.Name);
|
||||
}
|
||||
|
||||
ct.ThrowIfCancellationRequested();
|
||||
}
|
||||
|
||||
return removedFileHashes;
|
||||
}
|
||||
|
||||
private void InitializeGauges()
|
||||
{
|
||||
bool useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
|
||||
|
||||
if (useColdStorage)
|
||||
{
|
||||
var coldStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
|
||||
|
||||
DirectoryInfo dirColdStorage = new(coldStorageDir);
|
||||
var allFilesInColdStorageDir = dirColdStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
|
||||
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, allFilesInColdStorageDir.Sum(f => f.Length));
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, allFilesInColdStorageDir.Count);
|
||||
}
|
||||
|
||||
var hotStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
|
||||
DirectoryInfo dirHotStorage = new(hotStorageDir);
|
||||
var allFilesInHotStorage = dirHotStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
|
||||
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFilesInHotStorage.Sum(f => { try { return f.Length; } catch { return 0; } }));
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFilesInHotStorage.Count);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
using LightlessSyncShared.Services;
|
||||
using LightlessSyncShared.Utils.Configuration;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Frozen;
|
||||
|
||||
namespace LightlessSyncStaticFilesServer.Services;
|
||||
|
||||
public class MainServerShardRegistrationService : IHostedService
|
||||
{
|
||||
private readonly ILogger<MainServerShardRegistrationService> _logger;
|
||||
private readonly IConfigurationService<StaticFilesServerConfiguration> _configurationService;
|
||||
private readonly ConcurrentDictionary<string, ShardConfiguration> _shardConfigs = new(StringComparer.Ordinal);
|
||||
private readonly ConcurrentDictionary<string, DateTime> _shardHeartbeats = new(StringComparer.Ordinal);
|
||||
private readonly CancellationTokenSource _periodicCheckCts = new();
|
||||
|
||||
public MainServerShardRegistrationService(ILogger<MainServerShardRegistrationService> logger,
|
||||
IConfigurationService<StaticFilesServerConfiguration> configurationService)
|
||||
{
|
||||
_logger = logger;
|
||||
_configurationService = configurationService;
|
||||
}
|
||||
|
||||
public void RegisterShard(string shardName, ShardConfiguration shardConfiguration)
|
||||
{
|
||||
if (shardConfiguration == null || shardConfiguration == default)
|
||||
throw new InvalidOperationException("Empty configuration provided");
|
||||
|
||||
if (_shardConfigs.ContainsKey(shardName))
|
||||
_logger.LogInformation("Re-Registering Shard {name}", shardName);
|
||||
else
|
||||
_logger.LogInformation("Registering Shard {name}", shardName);
|
||||
|
||||
_shardHeartbeats[shardName] = DateTime.UtcNow;
|
||||
_shardConfigs[shardName] = shardConfiguration;
|
||||
}
|
||||
|
||||
public void UnregisterShard(string shardName)
|
||||
{
|
||||
_logger.LogInformation("Unregistering Shard {name}", shardName);
|
||||
|
||||
_shardHeartbeats.TryRemove(shardName, out _);
|
||||
_shardConfigs.TryRemove(shardName, out _);
|
||||
}
|
||||
|
||||
public List<ShardConfiguration> GetConfigurationsByContinent(string continent)
|
||||
{
|
||||
var shardConfigs = _shardConfigs.Values.Where(v => v.Continents.Contains(continent, StringComparer.OrdinalIgnoreCase)).ToList();
|
||||
if (shardConfigs.Any()) return shardConfigs;
|
||||
shardConfigs = _shardConfigs.Values.Where(v => v.Continents.Contains("*", StringComparer.OrdinalIgnoreCase)).ToList();
|
||||
if (shardConfigs.Any()) return shardConfigs;
|
||||
return [new ShardConfiguration() {
|
||||
Continents = ["*"],
|
||||
FileMatch = ".*",
|
||||
RegionUris = new(StringComparer.Ordinal) {
|
||||
{ "Central", _configurationService.GetValue<Uri>(nameof(StaticFilesServerConfiguration.CdnFullUrl)) }
|
||||
} }];
|
||||
}
|
||||
|
||||
public void ShardHeartbeat(string shardName)
|
||||
{
|
||||
if (!_shardConfigs.ContainsKey(shardName))
|
||||
throw new InvalidOperationException("Shard not registered");
|
||||
|
||||
_logger.LogInformation("Heartbeat from {name}", shardName);
|
||||
_shardHeartbeats[shardName] = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_ = Task.Run(() => PeriodicHeartbeatCleanup(_periodicCheckCts.Token), cancellationToken).ConfigureAwait(false);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await _periodicCheckCts.CancelAsync().ConfigureAwait(false);
|
||||
_periodicCheckCts.Dispose();
|
||||
}
|
||||
|
||||
private async Task PeriodicHeartbeatCleanup(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
foreach (var kvp in _shardHeartbeats.ToFrozenDictionary())
|
||||
{
|
||||
if (DateTime.UtcNow.Subtract(kvp.Value) > TimeSpan.FromMinutes(1))
|
||||
{
|
||||
_shardHeartbeats.TryRemove(kvp.Key, out _);
|
||||
_shardConfigs.TryRemove(kvp.Key, out _);
|
||||
}
|
||||
}
|
||||
|
||||
await Task.Delay(5000, ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
using LightlessSyncShared.Metrics;
|
||||
using LightlessSyncShared.Services;
|
||||
using LightlessSyncShared.Utils.Configuration;
|
||||
using LightlessSyncStaticFilesServer.Utils;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Timers;
|
||||
|
||||
namespace LightlessSyncStaticFilesServer.Services;
|
||||
|
||||
public class RequestQueueService : IHostedService
|
||||
{
|
||||
private readonly IClientReadyMessageService _clientReadyMessageService;
|
||||
private readonly CachedFileProvider _cachedFileProvider;
|
||||
private readonly ILogger<RequestQueueService> _logger;
|
||||
private readonly LightlessMetrics _metrics;
|
||||
private readonly ConcurrentQueue<UserRequest> _queue = new();
|
||||
private readonly ConcurrentQueue<UserRequest> _priorityQueue = new();
|
||||
private readonly int _queueExpirationSeconds;
|
||||
private readonly SemaphoreSlim _queueProcessingSemaphore = new(1);
|
||||
private readonly UserQueueEntry[] _userQueueRequests;
|
||||
private int _queueLimitForReset;
|
||||
private readonly int _queueReleaseSeconds;
|
||||
private System.Timers.Timer _queueTimer;
|
||||
|
||||
public RequestQueueService(LightlessMetrics metrics, IConfigurationService<StaticFilesServerConfiguration> configurationService,
|
||||
ILogger<RequestQueueService> logger, IClientReadyMessageService hubContext, CachedFileProvider cachedFileProvider)
|
||||
{
|
||||
_userQueueRequests = new UserQueueEntry[configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueSize), 50)];
|
||||
_queueExpirationSeconds = configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadTimeoutSeconds), 5);
|
||||
_queueLimitForReset = configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueClearLimit), 15000);
|
||||
_queueReleaseSeconds = configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueReleaseSeconds), 15);
|
||||
_metrics = metrics;
|
||||
_logger = logger;
|
||||
_clientReadyMessageService = hubContext;
|
||||
_cachedFileProvider = cachedFileProvider;
|
||||
}
|
||||
|
||||
public void ActivateRequest(Guid request)
|
||||
{
|
||||
_logger.LogDebug("Activating request {guid}", request);
|
||||
var req = _userQueueRequests.First(f => f != null && f.UserRequest.RequestId == request);
|
||||
req.MarkActive();
|
||||
}
|
||||
|
||||
public async Task EnqueueUser(UserRequest request, bool isPriority, CancellationToken token)
|
||||
{
|
||||
while (_queueProcessingSemaphore.CurrentCount == 0)
|
||||
{
|
||||
await Task.Delay(50, token).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_logger.LogDebug("Enqueueing req {guid} from {user} for {file}", request.RequestId, request.User, string.Join(", ", request.FileIds));
|
||||
|
||||
GetQueue(isPriority).Enqueue(request);
|
||||
}
|
||||
|
||||
public void FinishRequest(Guid request)
|
||||
{
|
||||
var req = _userQueueRequests.FirstOrDefault(f => f != null && f.UserRequest.RequestId == request);
|
||||
if (req != null)
|
||||
{
|
||||
var idx = Array.IndexOf(_userQueueRequests, req);
|
||||
_logger.LogDebug("Finishing Request {guid}, clearing slot {idx}", request, idx);
|
||||
_userQueueRequests[idx] = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("Request {guid} already cleared", request);
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsActiveProcessing(Guid request, string user, out UserRequest userRequest)
|
||||
{
|
||||
var userQueueRequest = _userQueueRequests.FirstOrDefault(u => u != null && u.UserRequest.RequestId == request && string.Equals(u.UserRequest.User, user, StringComparison.Ordinal));
|
||||
userRequest = userQueueRequest?.UserRequest ?? null;
|
||||
return userQueueRequest != null && userRequest != null && userQueueRequest.ExpirationDate > DateTime.UtcNow;
|
||||
}
|
||||
|
||||
public void RemoveFromQueue(Guid requestId, string user, bool isPriority)
|
||||
{
|
||||
var existingRequest = GetQueue(isPriority).FirstOrDefault(f => f.RequestId == requestId && string.Equals(f.User, user, StringComparison.Ordinal));
|
||||
if (existingRequest == null)
|
||||
{
|
||||
var activeSlot = _userQueueRequests.FirstOrDefault(r => r != null && string.Equals(r.UserRequest.User, user, StringComparison.Ordinal) && r.UserRequest.RequestId == requestId);
|
||||
if (activeSlot != null)
|
||||
{
|
||||
var idx = Array.IndexOf(_userQueueRequests, activeSlot);
|
||||
if (idx >= 0)
|
||||
{
|
||||
_userQueueRequests[idx] = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
existingRequest.IsCancelled = true;
|
||||
}
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_queueTimer = new System.Timers.Timer(500);
|
||||
_queueTimer.Elapsed += ProcessQueue;
|
||||
_queueTimer.AutoReset = true;
|
||||
_queueTimer.Start();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private ConcurrentQueue<UserRequest> GetQueue(bool isPriority) => isPriority ? _priorityQueue : _queue;
|
||||
|
||||
public bool StillEnqueued(Guid request, string user, bool isPriority)
|
||||
{
|
||||
return GetQueue(isPriority).Any(c => c.RequestId == request && string.Equals(c.User, user, StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_queueTimer.Stop();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private void DequeueIntoSlot(UserRequest userRequest, int slot)
|
||||
{
|
||||
_logger.LogDebug("Dequeueing {req} into {i}: {user} with {file}", userRequest.RequestId, slot, userRequest.User, string.Join(", ", userRequest.FileIds));
|
||||
_userQueueRequests[slot] = new(userRequest, DateTime.UtcNow.AddSeconds(_queueExpirationSeconds));
|
||||
_clientReadyMessageService.SendDownloadReady(userRequest.User, userRequest.RequestId);
|
||||
}
|
||||
|
||||
private void ProcessQueue(object src, ElapsedEventArgs e)
|
||||
{
|
||||
_logger.LogDebug("Periodic Processing Queue Start");
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeQueueFree, _userQueueRequests.Count(c => c == null));
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeQueueActive, _userQueueRequests.Count(c => c != null && c.IsActive));
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeQueueInactive, _userQueueRequests.Count(c => c != null && !c.IsActive));
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeDownloadQueue, _queue.Count(q => !q.IsCancelled));
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeDownloadQueueCancelled, _queue.Count(q => q.IsCancelled));
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeDownloadPriorityQueue, _priorityQueue.Count(q => !q.IsCancelled));
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeDownloadPriorityQueueCancelled, _priorityQueue.Count(q => q.IsCancelled));
|
||||
|
||||
if (_queueProcessingSemaphore.CurrentCount == 0)
|
||||
{
|
||||
_logger.LogDebug("Aborting Periodic Processing Queue, processing still running");
|
||||
return;
|
||||
}
|
||||
|
||||
_queueProcessingSemaphore.Wait();
|
||||
|
||||
try
|
||||
{
|
||||
if (_queue.Count(c => !c.IsCancelled) > _queueLimitForReset)
|
||||
{
|
||||
_queue.Clear();
|
||||
return;
|
||||
}
|
||||
|
||||
for (int i = 0; i < _userQueueRequests.Length; i++)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (_userQueueRequests[i] != null
|
||||
&& (((!_userQueueRequests[i].IsActive && _userQueueRequests[i].ExpirationDate < DateTime.UtcNow))
|
||||
|| (_userQueueRequests[i].IsActive && _userQueueRequests[i].ActivationDate < DateTime.UtcNow.Subtract(TimeSpan.FromSeconds(_queueReleaseSeconds))))
|
||||
)
|
||||
{
|
||||
_logger.LogDebug("Expiring request {guid} slot {slot}", _userQueueRequests[i].UserRequest.RequestId, i);
|
||||
_userQueueRequests[i] = null;
|
||||
}
|
||||
|
||||
if (_userQueueRequests[i] != null) continue;
|
||||
|
||||
while (true)
|
||||
{
|
||||
if (!_priorityQueue.All(u => _cachedFileProvider.AnyFilesDownloading(u.FileIds))
|
||||
&& _priorityQueue.TryDequeue(out var prioRequest))
|
||||
{
|
||||
if (prioRequest.IsCancelled)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (_cachedFileProvider.AnyFilesDownloading(prioRequest.FileIds))
|
||||
{
|
||||
_priorityQueue.Enqueue(prioRequest);
|
||||
continue;
|
||||
}
|
||||
|
||||
DequeueIntoSlot(prioRequest, i);
|
||||
break;
|
||||
}
|
||||
|
||||
if (!_queue.All(u => _cachedFileProvider.AnyFilesDownloading(u.FileIds))
|
||||
&& _queue.TryDequeue(out var request))
|
||||
{
|
||||
if (request.IsCancelled)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (_cachedFileProvider.AnyFilesDownloading(request.FileIds))
|
||||
{
|
||||
_queue.Enqueue(request);
|
||||
continue;
|
||||
}
|
||||
|
||||
DequeueIntoSlot(request, i);
|
||||
break;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error during inside queue processing");
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error during Queue processing");
|
||||
}
|
||||
finally
|
||||
{
|
||||
_queueProcessingSemaphore.Release();
|
||||
_logger.LogDebug("Periodic Processing Queue End");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
using LightlessSync.API.Routes;
|
||||
using LightlessSyncShared.Services;
|
||||
using LightlessSyncShared.Utils;
|
||||
using LightlessSyncShared.Utils.Configuration;
|
||||
using System.Net.Http.Headers;
|
||||
|
||||
namespace LightlessSyncStaticFilesServer.Services;
|
||||
|
||||
public class ShardClientReadyMessageService : IClientReadyMessageService
|
||||
{
|
||||
private readonly ILogger<ShardClientReadyMessageService> _logger;
|
||||
private readonly ServerTokenGenerator _tokenGenerator;
|
||||
private readonly IConfigurationService<StaticFilesServerConfiguration> _configurationService;
|
||||
private readonly HttpClient _httpClient;
|
||||
|
||||
public ShardClientReadyMessageService(ILogger<ShardClientReadyMessageService> logger, ServerTokenGenerator tokenGenerator, IConfigurationService<StaticFilesServerConfiguration> configurationService)
|
||||
{
|
||||
_logger = logger;
|
||||
_tokenGenerator = tokenGenerator;
|
||||
_configurationService = configurationService;
|
||||
_httpClient = new();
|
||||
_httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("LightlessSyncServer", "1.0.0.0"));
|
||||
}
|
||||
|
||||
public async Task SendDownloadReady(string uid, Guid requestId)
|
||||
{
|
||||
var mainUrl = _configurationService.GetValue<Uri>(nameof(StaticFilesServerConfiguration.MainFileServerAddress));
|
||||
var path = LightlessFiles.MainSendReadyFullPath(mainUrl, uid, requestId);
|
||||
using HttpRequestMessage msg = new()
|
||||
{
|
||||
RequestUri = path
|
||||
};
|
||||
msg.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _tokenGenerator.Token);
|
||||
|
||||
_logger.LogInformation("Sending Client Ready for {uid}:{requestId} to {path}", uid, requestId, path);
|
||||
try
|
||||
{
|
||||
using var result = await _httpClient.SendAsync(msg).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failure to send for {uid}:{requestId}", uid, requestId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
using ByteSizeLib;
|
||||
using LightlessSyncShared.Metrics;
|
||||
using LightlessSyncShared.Services;
|
||||
using LightlessSyncShared.Utils.Configuration;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace LightlessSyncStaticFilesServer.Services;
|
||||
|
||||
public class ShardFileCleanupService : IHostedService
|
||||
{
|
||||
private readonly string _cacheDir;
|
||||
private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
|
||||
private readonly ILogger<MainFileCleanupService> _logger;
|
||||
private readonly LightlessMetrics _metrics;
|
||||
private CancellationTokenSource _cleanupCts;
|
||||
|
||||
public ShardFileCleanupService(LightlessMetrics metrics, ILogger<MainFileCleanupService> logger, IConfigurationService<StaticFilesServerConfiguration> configuration)
|
||||
{
|
||||
_metrics = metrics;
|
||||
_logger = logger;
|
||||
_configuration = configuration;
|
||||
_cacheDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
|
||||
}
|
||||
|
||||
public async Task CleanUpTask(CancellationToken ct)
|
||||
{
|
||||
_logger.LogInformation("Starting periodic cleanup task");
|
||||
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
DirectoryInfo dir = new(_cacheDir);
|
||||
var allFiles = dir.GetFiles("*", SearchOption.AllDirectories);
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFiles.Sum(f => f.Length));
|
||||
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFiles.Length);
|
||||
|
||||
CleanUpOutdatedFiles(ct);
|
||||
|
||||
CleanUpFilesBeyondSizeLimit(ct);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "Error during cleanup task");
|
||||
}
|
||||
|
||||
var cleanupCheckMinutes = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CleanupCheckInMinutes), 15);
|
||||
|
||||
var now = DateTime.Now;
|
||||
TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
|
||||
TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % cleanupCheckMinutes, 0);
|
||||
var span = futureTime.AddMinutes(cleanupCheckMinutes) - currentTime;
|
||||
|
||||
_logger.LogInformation("File Cleanup Complete, next run at {date}", now.Add(span));
|
||||
await Task.Delay(span, ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("Cleanup Service started");
|
||||
|
||||
_cleanupCts = new();
|
||||
|
||||
_ = CleanUpTask(_cleanupCts.Token);
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_cleanupCts.Cancel();
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private void CleanUpFilesBeyondSizeLimit(CancellationToken ct)
|
||||
{
|
||||
var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);
|
||||
if (sizeLimit <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", sizeLimit);
|
||||
var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories)
|
||||
.Where(f => !f.EndsWith("dl", StringComparison.OrdinalIgnoreCase))
|
||||
.Select(f => new FileInfo(f)).ToList()
|
||||
.OrderBy(f => f.LastAccessTimeUtc).ToList();
|
||||
var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
|
||||
long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes;
|
||||
while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested)
|
||||
{
|
||||
var oldestFile = allLocalFiles[0];
|
||||
allLocalFiles.Remove(oldestFile);
|
||||
totalCacheSizeInBytes -= oldestFile.Length;
|
||||
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, oldestFile.Length);
|
||||
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
|
||||
_logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes);
|
||||
oldestFile.Delete();
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error during cache size limit cleanup");
|
||||
}
|
||||
}
|
||||
|
||||
private void CleanUpOutdatedFiles(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
|
||||
var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
|
||||
|
||||
_logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", unusedRetention);
|
||||
if (forcedDeletionAfterHours > 0)
|
||||
{
|
||||
_logger.LogInformation("Cleaning up files written to longer than {hours}h ago", forcedDeletionAfterHours);
|
||||
}
|
||||
|
||||
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention));
|
||||
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
|
||||
DirectoryInfo dir = new(_cacheDir);
|
||||
var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories)
|
||||
.Where(f => !f.Name.EndsWith("dl", StringComparison.OrdinalIgnoreCase))
|
||||
.ToList();
|
||||
|
||||
foreach (var file in allFilesInDir)
|
||||
{
|
||||
if (file.LastAccessTime < prevTime)
|
||||
{
|
||||
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
|
||||
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
|
||||
_logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
|
||||
file.Delete();
|
||||
}
|
||||
else if (forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
|
||||
{
|
||||
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
|
||||
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
|
||||
_logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
|
||||
file.Delete();
|
||||
}
|
||||
else if (file.Length == 0 && !string.Equals(file.Extension, ".dl", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
|
||||
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
|
||||
_logger.LogInformation("File with size 0 deleted: {filename}", file.Name);
|
||||
file.Delete();
|
||||
}
|
||||
|
||||
ct.ThrowIfCancellationRequested();
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error during file cleanup of old files");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
using LightlessSync.API.Routes;
|
||||
using LightlessSyncShared.Services;
|
||||
using LightlessSyncShared.Utils;
|
||||
using LightlessSyncShared.Utils.Configuration;
|
||||
|
||||
namespace LightlessSyncStaticFilesServer.Services;
|
||||
|
||||
public class ShardRegistrationService : IHostedService
|
||||
{
|
||||
private readonly ILogger<ShardRegistrationService> _logger;
|
||||
private readonly IConfigurationService<StaticFilesServerConfiguration> _configurationService;
|
||||
private readonly HttpClient _httpClient = new();
|
||||
private readonly CancellationTokenSource _heartBeatCts = new();
|
||||
private bool _isRegistered = false;
|
||||
|
||||
public ShardRegistrationService(ILogger<ShardRegistrationService> logger,
|
||||
IConfigurationService<StaticFilesServerConfiguration> configurationService,
|
||||
ServerTokenGenerator serverTokenGenerator)
|
||||
{
|
||||
_logger = logger;
|
||||
_configurationService = configurationService;
|
||||
_httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", serverTokenGenerator.Token);
|
||||
}
|
||||
|
||||
private void OnConfigChanged(object sender, EventArgs e)
|
||||
{
|
||||
_isRegistered = false;
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("Starting");
|
||||
_configurationService.ConfigChangedEvent += OnConfigChanged;
|
||||
_ = Task.Run(() => HeartbeatLoop(_heartBeatCts.Token));
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("Stopping");
|
||||
|
||||
_configurationService.ConfigChangedEvent -= OnConfigChanged;
|
||||
_heartBeatCts.Cancel();
|
||||
_heartBeatCts.Dispose();
|
||||
// call unregister
|
||||
await UnregisterShard().ConfigureAwait(false);
|
||||
_httpClient.Dispose();
|
||||
}
|
||||
|
||||
private async Task HeartbeatLoop(CancellationToken ct)
|
||||
{
|
||||
while (!_heartBeatCts.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await ProcessHeartbeat(ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Issue during Heartbeat");
|
||||
_isRegistered = false;
|
||||
}
|
||||
|
||||
await Task.Delay(TimeSpan.FromSeconds(30), ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessHeartbeat(CancellationToken ct)
|
||||
{
|
||||
if (!_isRegistered)
|
||||
{
|
||||
await TryRegisterShard(ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await ShardHeartbeat(ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task ShardHeartbeat(CancellationToken ct)
|
||||
{
|
||||
Uri mainServer = _configurationService.GetValue<Uri>(nameof(StaticFilesServerConfiguration.MainFileServerAddress));
|
||||
_logger.LogInformation("Running heartbeat against Main {server}", mainServer);
|
||||
|
||||
using var heartBeat = await _httpClient.PostAsync(new Uri(mainServer, LightlessFiles.Main + "/shardHeartbeat"), null, ct).ConfigureAwait(false);
|
||||
heartBeat.EnsureSuccessStatusCode();
|
||||
}
|
||||
|
||||
private async Task TryRegisterShard(CancellationToken ct)
|
||||
{
|
||||
Uri mainServer = _configurationService.GetValue<Uri>(nameof(StaticFilesServerConfiguration.MainFileServerAddress));
|
||||
_logger.LogInformation("Registering Shard with Main {server}", mainServer);
|
||||
var config = _configurationService.GetValue<ShardConfiguration>(nameof(StaticFilesServerConfiguration.ShardConfiguration));
|
||||
_logger.LogInformation("Config Value {varName}: {value}", nameof(ShardConfiguration.Continents), string.Join(", ", config.Continents));
|
||||
_logger.LogInformation("Config Value {varName}: {value}", nameof(ShardConfiguration.FileMatch), config.FileMatch);
|
||||
_logger.LogInformation("Config Value {varName}: {value}", nameof(ShardConfiguration.RegionUris), string.Join("; ", config.RegionUris.Select(k => k.Key + ":" + k.Value)));
|
||||
|
||||
using var register = await _httpClient.PostAsJsonAsync(new Uri(mainServer, LightlessFiles.Main + "/shardRegister"), config, ct).ConfigureAwait(false);
|
||||
register.EnsureSuccessStatusCode();
|
||||
_isRegistered = true;
|
||||
}
|
||||
|
||||
private async Task UnregisterShard()
|
||||
{
|
||||
Uri mainServer = _configurationService.GetValue<Uri>(nameof(StaticFilesServerConfiguration.MainFileServerAddress));
|
||||
_logger.LogInformation("Unregistering Shard with Main {server}", mainServer);
|
||||
using var heartBeat = await _httpClient.PostAsync(new Uri(mainServer, LightlessFiles.Main + "/shardUnregister"), null).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user