From b6907a2704ae6aa369a28fdf5a47a81fc104ac93 Mon Sep 17 00:00:00 2001 From: azyges <229218900+azyges@users.noreply.github.com> Date: Fri, 10 Oct 2025 07:37:33 +0900 Subject: [PATCH] cdn downloads support --- LightlessAPI | 2 +- .../StaticFilesServerConfiguration.cs | 4 + .../Controllers/ServerFilesController.cs | 42 ++++++- .../Services/CDNDownloadUrlService.cs | 108 ++++++++++++++++++ .../LightlessSyncStaticFilesServer/Startup.cs | 1 + .../appsettings.json | 4 +- 6 files changed, 157 insertions(+), 4 deletions(-) create mode 100644 LightlessSyncServer/LightlessSyncStaticFilesServer/Services/CDNDownloadUrlService.cs diff --git a/LightlessAPI b/LightlessAPI index f3c6064..89ac342 160000 --- a/LightlessAPI +++ b/LightlessAPI @@ -1 +1 @@ -Subproject commit f3c60648921abab03c3a6cc6142543f06ba02c45 +Subproject commit 89ac34235d297655643684c425311cd68f0702f2 diff --git a/LightlessSyncServer/LightlessSyncShared/Utils/Configuration/StaticFilesServerConfiguration.cs b/LightlessSyncServer/LightlessSyncShared/Utils/Configuration/StaticFilesServerConfiguration.cs index 85b9585..af4b3b6 100644 --- a/LightlessSyncServer/LightlessSyncShared/Utils/Configuration/StaticFilesServerConfiguration.cs +++ b/LightlessSyncServer/LightlessSyncShared/Utils/Configuration/StaticFilesServerConfiguration.cs @@ -20,6 +20,8 @@ public class StaticFilesServerConfiguration : LightlessConfigurationBase public string ColdStorageDirectory { get; set; } = null; public double ColdStorageSizeHardLimitInGiB { get; set; } = -1; public int ColdStorageUnusedFileRetentionPeriodInDays { get; set; } = 30; + public bool EnableDirectDownloads { get; set; } = true; + public int DirectDownloadTokenLifetimeSeconds { get; set; } = 300; [RemoteConfiguration] public double SpeedTestHoursRateLimit { get; set; } = 0.5; [RemoteConfiguration] @@ -40,6 +42,8 @@ public class StaticFilesServerConfiguration : LightlessConfigurationBase sb.AppendLine($"{nameof(CacheDirectory)} => {CacheDirectory}"); sb.AppendLine($"{nameof(DownloadQueueSize)} => {DownloadQueueSize}"); sb.AppendLine($"{nameof(DownloadQueueReleaseSeconds)} => {DownloadQueueReleaseSeconds}"); + sb.AppendLine($"{nameof(EnableDirectDownloads)} => {EnableDirectDownloads}"); + sb.AppendLine($"{nameof(DirectDownloadTokenLifetimeSeconds)} => {DirectDownloadTokenLifetimeSeconds}"); return sb.ToString(); } } diff --git a/LightlessSyncServer/LightlessSyncStaticFilesServer/Controllers/ServerFilesController.cs b/LightlessSyncServer/LightlessSyncStaticFilesServer/Controllers/ServerFilesController.cs index a81ca17..6ec924b 100644 --- a/LightlessSyncServer/LightlessSyncStaticFilesServer/Controllers/ServerFilesController.cs +++ b/LightlessSyncServer/LightlessSyncStaticFilesServer/Controllers/ServerFilesController.cs @@ -10,6 +10,7 @@ using LightlessSyncShared.Services; using LightlessSyncShared.Utils.Configuration; using LightlessSyncStaticFilesServer.Services; using LightlessSyncStaticFilesServer.Utils; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; @@ -32,12 +33,13 @@ public class ServerFilesController : ControllerBase private readonly IDbContextFactory _lightlessDbContext; private readonly LightlessMetrics _metricsClient; private readonly MainServerShardRegistrationService _shardRegistrationService; + private readonly CDNDownloadUrlService _cdnDownloadUrlService; public ServerFilesController(ILogger logger, CachedFileProvider cachedFileProvider, IConfigurationService configuration, IHubContext hubContext, IDbContextFactory lightlessDbContext, LightlessMetrics metricsClient, - MainServerShardRegistrationService shardRegistrationService) : base(logger) + MainServerShardRegistrationService shardRegistrationService, CDNDownloadUrlService cdnDownloadUrlService) : base(logger) { _basePath = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false) ? configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)) @@ -48,6 +50,7 @@ public class ServerFilesController : ControllerBase _lightlessDbContext = lightlessDbContext; _metricsClient = metricsClient; _shardRegistrationService = shardRegistrationService; + _cdnDownloadUrlService = cdnDownloadUrlService; } [HttpPost(LightlessFiles.ServerFiles_DeleteAll)] @@ -105,6 +108,16 @@ public class ServerFilesController : ControllerBase baseUrl = shard.Value ?? _configuration.GetValue(nameof(StaticFilesServerConfiguration.CdnFullUrl)); } + var cdnDownloadUrl = string.Empty; + if (forbiddenFile == null) + { + var directUri = _cdnDownloadUrlService.TryCreateDirectDownloadUri(baseUrl, file.Hash); + if (directUri != null) + { + cdnDownloadUrl = directUri.ToString(); + } + } + response.Add(new DownloadFileDto { FileExists = file.Size > 0, @@ -113,6 +126,7 @@ public class ServerFilesController : ControllerBase Hash = file.Hash, Size = file.Size, Url = baseUrl?.ToString() ?? string.Empty, + CDNDownloadUrl = cdnDownloadUrl, RawSize = file.RawSize }); } @@ -127,6 +141,30 @@ public class ServerFilesController : ControllerBase return Ok(JsonSerializer.Serialize(allFileShards.SelectMany(t => t.RegionUris.Select(v => v.Value.ToString())))); } + [HttpGet(LightlessFiles.ServerFiles_DirectDownload + "/{hash}")] + [AllowAnonymous] + public async Task DownloadFileDirect(string hash, [FromQuery] long expires, [FromQuery] string signature) + { + if (!_cdnDownloadUrlService.DirectDownloadsEnabled) + { + return NotFound(); + } + + hash = hash.ToUpperInvariant(); + if (!_cdnDownloadUrlService.TryValidateSignature(hash, expires, signature)) + { + return Unauthorized(); + } + + var fileInfo = await _cachedFileProvider.DownloadAndGetLocalFileInfo(hash).ConfigureAwait(false); + if (fileInfo == null) + { + return NotFound(); + } + + return PhysicalFile(fileInfo.FullName, "application/octet-stream"); + } + [HttpPost(LightlessFiles.ServerFiles_FilesSend)] public async Task FilesSend([FromBody] FilesSendDto filesSendDto) { @@ -360,4 +398,4 @@ public class ServerFilesController : ControllerBase buffer[i] ^= 42; } } -} \ No newline at end of file +} diff --git a/LightlessSyncServer/LightlessSyncStaticFilesServer/Services/CDNDownloadUrlService.cs b/LightlessSyncServer/LightlessSyncStaticFilesServer/Services/CDNDownloadUrlService.cs new file mode 100644 index 0000000..4bbc32b --- /dev/null +++ b/LightlessSyncServer/LightlessSyncStaticFilesServer/Services/CDNDownloadUrlService.cs @@ -0,0 +1,108 @@ +using LightlessSync.API.Routes; +using LightlessSyncShared.Services; +using LightlessSyncShared.Utils.Configuration; +using Microsoft.AspNetCore.Http.Extensions; +using Microsoft.AspNetCore.WebUtilities; +using System.Security.Cryptography; +using System.Text; +using System.Globalization; +using System.Linq; + +namespace LightlessSyncStaticFilesServer.Services; + +public class CDNDownloadUrlService +{ + private readonly IConfigurationService _staticConfig; + private readonly IConfigurationService _globalConfig; + private readonly ILogger _logger; + + public CDNDownloadUrlService(IConfigurationService staticConfig, + IConfigurationService globalConfig, ILogger logger) + { + _staticConfig = staticConfig; + _globalConfig = globalConfig; + _logger = logger; + } + + public bool DirectDownloadsEnabled => + _staticConfig.GetValueOrDefault(nameof(StaticFilesServerConfiguration.EnableDirectDownloads), false); + + public Uri? TryCreateDirectDownloadUri(Uri? baseUri, string hash) + { + if (!DirectDownloadsEnabled || baseUri == null) + { + return null; + } + + if (!IsSupportedHash(hash)) + { + _logger.LogDebug("Skipping direct download link generation for invalid hash {hash}", hash); + return null; + } + + var normalizedHash = hash.ToUpperInvariant(); + + var lifetimeSeconds = Math.Max(5, + _staticConfig.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DirectDownloadTokenLifetimeSeconds), 300)); + var expiresAt = DateTimeOffset.UtcNow.AddSeconds(lifetimeSeconds); + var signature = CreateSignature(normalizedHash, expiresAt.ToUnixTimeSeconds()); + + var directPath = $"{LightlessFiles.ServerFiles}/{LightlessFiles.ServerFiles_DirectDownload}/{normalizedHash}"; + var builder = new UriBuilder(new Uri(baseUri, directPath)); + var query = new QueryBuilder + { + { "expires", expiresAt.ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture) }, + { "signature", signature } + }; + builder.Query = query.ToQueryString().Value!.TrimStart('?'); + return builder.Uri; + } + + public bool TryValidateSignature(string hash, long expiresUnixSeconds, string signature) + { + if (!DirectDownloadsEnabled) + { + return false; + } + + if (string.IsNullOrEmpty(signature) || !IsSupportedHash(hash)) + { + return false; + } + + var normalizedHash = hash.ToUpperInvariant(); + + DateTimeOffset expiresAt; + try + { + expiresAt = DateTimeOffset.FromUnixTimeSeconds(expiresUnixSeconds); + } + catch (ArgumentOutOfRangeException) + { + return false; + } + + if (expiresAt < DateTimeOffset.UtcNow) + { + return false; + } + + var expected = CreateSignature(normalizedHash, expiresAt.ToUnixTimeSeconds()); + return CryptographicOperations.FixedTimeEquals( + Encoding.UTF8.GetBytes(expected), + Encoding.UTF8.GetBytes(signature)); + } + + private string CreateSignature(string hash, long expiresUnixSeconds) + { + var signingKey = _globalConfig.GetValue(nameof(LightlessConfigurationBase.Jwt)); + using var hmac = new HMACSHA256(Encoding.UTF8.GetBytes(signingKey)); + var payload = Encoding.UTF8.GetBytes($"{hash}:{expiresUnixSeconds}"); + return WebEncoders.Base64UrlEncode(hmac.ComputeHash(payload)); + } + + private static bool IsSupportedHash(string hash) + { + return hash.Length == 40 && hash.All(char.IsAsciiLetterOrDigit); + } +} diff --git a/LightlessSyncServer/LightlessSyncStaticFilesServer/Startup.cs b/LightlessSyncServer/LightlessSyncStaticFilesServer/Startup.cs index 1ed3ec3..f4034a5 100644 --- a/LightlessSyncServer/LightlessSyncStaticFilesServer/Startup.cs +++ b/LightlessSyncServer/LightlessSyncStaticFilesServer/Startup.cs @@ -87,6 +87,7 @@ public class Startup services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddHostedService(p => p.GetService()); services.AddHostedService(m => m.GetService()); services.AddSingleton, LightlessConfigurationServiceClient>(); diff --git a/LightlessSyncServer/LightlessSyncStaticFilesServer/appsettings.json b/LightlessSyncServer/LightlessSyncStaticFilesServer/appsettings.json index dc53c47..35f6258 100644 --- a/LightlessSyncServer/LightlessSyncStaticFilesServer/appsettings.json +++ b/LightlessSyncServer/LightlessSyncStaticFilesServer/appsettings.json @@ -25,7 +25,9 @@ "UnusedFileRetentionPeriodInDays": 7, "CacheDirectory": "G:\\ServerTest", "ServiceAddress": "http://localhost:5002", - "RemoteCacheSourceUri": "" + "RemoteCacheSourceUri": "", + "EnableDirectDownloads": true, + "DirectDownloadTokenLifetimeSeconds": 300 }, "AllowedHosts": "*" }