adds the ability for shards to sync with the main server regarding their caches, when pulling files from main server the shard can stream it to the client directly while downloading and add info for server to report to client regarding file locations across shards
This commit is contained in:
Abelfreyja
@@ -82,7 +82,9 @@ public class ServerFilesController : ControllerBase
|
||||
}
|
||||
|
||||
[HttpGet(LightlessFiles.ServerFiles_GetSizes)]
|
||||
public async Task<IActionResult> FilesGetSizes([FromBody] List<string> hashes)
|
||||
public async Task<IActionResult> FilesGetSizes(
|
||||
[FromBody] List<string> hashes,
|
||||
[FromQuery(Name = "avoidHost")] List<string>? avoidHosts = null)
|
||||
{
|
||||
using var dbContext = await _lightlessDbContext.CreateDbContextAsync();
|
||||
var forbiddenFiles = await dbContext.ForbiddenUploadEntries.
|
||||
@@ -94,27 +96,97 @@ public class ServerFilesController : ControllerBase
|
||||
.Select(k => new { k.Hash, k.Size, k.RawSize })
|
||||
.ToListAsync().ConfigureAwait(false);
|
||||
|
||||
var allFileShards = _shardRegistrationService.GetConfigurationsByContinent(Continent);
|
||||
var avoidHostSet = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
if (avoidHosts != null)
|
||||
{
|
||||
foreach (var host in avoidHosts)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(host))
|
||||
{
|
||||
avoidHostSet.Add(host);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var allFileShards = _shardRegistrationService.GetShardEntriesByContinent(Continent);
|
||||
var shardContexts = new List<ShardSelectionContext>(allFileShards.Count);
|
||||
foreach (var shard in allFileShards)
|
||||
{
|
||||
shardContexts.Add(new ShardSelectionContext(
|
||||
shard.ShardName,
|
||||
shard.Config,
|
||||
new Regex(shard.Config.FileMatch, RegexOptions.Compiled)));
|
||||
}
|
||||
|
||||
foreach (var file in cacheFile)
|
||||
{
|
||||
var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
|
||||
Uri? baseUrl = null;
|
||||
Uri? queuedBaseUrl = null;
|
||||
Uri? directBaseUrl = null;
|
||||
var queuedUrls = new List<string>();
|
||||
var hasFileUrls = new List<string>();
|
||||
var hasFileDirectUrls = new List<string>();
|
||||
var pullThroughUrls = new List<string>();
|
||||
var pullThroughDirectUrls = new List<string>();
|
||||
|
||||
if (forbiddenFile == null)
|
||||
{
|
||||
var matchingShards = allFileShards.Where(f => new Regex(f.FileMatch).IsMatch(file.Hash)).ToList();
|
||||
var matchingShards = shardContexts
|
||||
.Where(f => f.FileMatchRegex.IsMatch(file.Hash))
|
||||
.ToList();
|
||||
|
||||
var shard = matchingShards.SelectMany(g => g.RegionUris)
|
||||
.OrderBy(g => Guid.NewGuid()).FirstOrDefault();
|
||||
foreach (var shardEntry in matchingShards)
|
||||
{
|
||||
var regionUris = shardEntry.GetRegionUris(avoidHostSet);
|
||||
|
||||
baseUrl = shard.Value ?? _configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.CdnFullUrl));
|
||||
if (regionUris.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var uri in regionUris)
|
||||
{
|
||||
AddBaseUrl(queuedUrls, uri);
|
||||
}
|
||||
|
||||
var hasFile = !string.IsNullOrEmpty(shardEntry.ShardName)
|
||||
&& _shardRegistrationService.ShardHasFile(shardEntry.ShardName, file.Hash);
|
||||
|
||||
var baseList = hasFile ? hasFileUrls : pullThroughUrls;
|
||||
var directList = hasFile ? hasFileDirectUrls : pullThroughDirectUrls;
|
||||
|
||||
foreach (var uri in regionUris)
|
||||
{
|
||||
AddCandidate(baseList, directList, uri, file.Hash);
|
||||
}
|
||||
}
|
||||
|
||||
if (queuedUrls.Count == 0)
|
||||
{
|
||||
var fallback = _configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.CdnFullUrl));
|
||||
if (fallback != null && (avoidHostSet.Count == 0 || !IsAvoidedHost(fallback, avoidHostSet)))
|
||||
{
|
||||
AddBaseUrl(queuedUrls, fallback);
|
||||
}
|
||||
}
|
||||
|
||||
if (hasFileUrls.Count == 0 && pullThroughUrls.Count == 0)
|
||||
{
|
||||
var fallback = _configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.CdnFullUrl));
|
||||
if (fallback != null && (avoidHostSet.Count == 0 || !IsAvoidedHost(fallback, avoidHostSet)))
|
||||
{
|
||||
AddCandidate(pullThroughUrls, pullThroughDirectUrls, fallback, file.Hash);
|
||||
}
|
||||
}
|
||||
|
||||
queuedBaseUrl = SelectPreferredBase(queuedUrls);
|
||||
directBaseUrl = SelectPreferredBase(hasFileUrls, pullThroughUrls);
|
||||
}
|
||||
|
||||
var cdnDownloadUrl = string.Empty;
|
||||
if (forbiddenFile == null)
|
||||
{
|
||||
var directUri = _cdnDownloadUrlService.TryCreateDirectDownloadUri(baseUrl, file.Hash);
|
||||
var directUri = _cdnDownloadUrlService.TryCreateDirectDownloadUri(directBaseUrl, file.Hash);
|
||||
if (directUri != null)
|
||||
{
|
||||
cdnDownloadUrl = directUri.ToString();
|
||||
@@ -128,8 +200,10 @@ public class ServerFilesController : ControllerBase
|
||||
IsForbidden = forbiddenFile != null,
|
||||
Hash = file.Hash,
|
||||
Size = file.Size,
|
||||
Url = baseUrl?.ToString() ?? string.Empty,
|
||||
Url = queuedBaseUrl?.ToString() ?? string.Empty,
|
||||
CDNDownloadUrl = cdnDownloadUrl,
|
||||
HasFileDirectUrls = hasFileDirectUrls,
|
||||
PullThroughDirectUrls = pullThroughDirectUrls,
|
||||
RawSize = file.RawSize
|
||||
});
|
||||
}
|
||||
@@ -144,22 +218,127 @@ public class ServerFilesController : ControllerBase
|
||||
return Ok(JsonSerializer.Serialize(allFileShards.SelectMany(t => t.RegionUris.Select(v => v.Value.ToString()))));
|
||||
}
|
||||
|
||||
private static bool IsAvoidedHost(Uri uri, HashSet<string> avoidHosts)
|
||||
{
|
||||
if (avoidHosts.Count == 0)
|
||||
return false;
|
||||
|
||||
var host = uri.Host;
|
||||
if (!string.IsNullOrWhiteSpace(host) && avoidHosts.Contains(host))
|
||||
return true;
|
||||
|
||||
var authority = uri.Authority;
|
||||
if (!string.IsNullOrWhiteSpace(authority) && avoidHosts.Contains(authority))
|
||||
return true;
|
||||
|
||||
var absolute = uri.ToString().TrimEnd('/');
|
||||
return avoidHosts.Contains(absolute);
|
||||
}
|
||||
|
||||
private sealed class ShardSelectionContext
|
||||
{
|
||||
private List<Uri>? _cachedUris;
|
||||
private List<Uri>? _cachedAvoidedUris;
|
||||
|
||||
public ShardSelectionContext(string shardName, ShardConfiguration config, Regex fileMatchRegex)
|
||||
{
|
||||
ShardName = shardName;
|
||||
Config = config;
|
||||
FileMatchRegex = fileMatchRegex;
|
||||
}
|
||||
|
||||
public string ShardName { get; }
|
||||
public ShardConfiguration Config { get; }
|
||||
public Regex FileMatchRegex { get; }
|
||||
|
||||
public List<Uri> GetRegionUris(HashSet<string> avoidHosts)
|
||||
{
|
||||
if (_cachedUris == null)
|
||||
{
|
||||
_cachedUris = Config.RegionUris.Values.ToList();
|
||||
}
|
||||
|
||||
if (avoidHosts.Count == 0)
|
||||
{
|
||||
return _cachedUris;
|
||||
}
|
||||
|
||||
_cachedAvoidedUris ??= _cachedUris.Where(u => !IsAvoidedHost(u, avoidHosts)).ToList();
|
||||
return _cachedAvoidedUris.Count > 0 ? _cachedAvoidedUris : _cachedUris;
|
||||
}
|
||||
}
|
||||
|
||||
private void AddCandidate(List<string> baseUrls, List<string> directUrls, Uri baseUri, string hash)
|
||||
{
|
||||
var baseUrl = baseUri.ToString();
|
||||
if (baseUrls.Any(u => string.Equals(u, baseUrl, StringComparison.OrdinalIgnoreCase)))
|
||||
return;
|
||||
|
||||
baseUrls.Add(baseUrl);
|
||||
|
||||
var direct = _cdnDownloadUrlService.TryCreateDirectDownloadUri(baseUri, hash);
|
||||
directUrls.Add(direct?.ToString() ?? string.Empty);
|
||||
}
|
||||
|
||||
private static void AddBaseUrl(List<string> baseUrls, Uri baseUri)
|
||||
{
|
||||
var baseUrl = baseUri.ToString();
|
||||
if (baseUrls.Any(u => string.Equals(u, baseUrl, StringComparison.OrdinalIgnoreCase)))
|
||||
return;
|
||||
|
||||
baseUrls.Add(baseUrl);
|
||||
}
|
||||
|
||||
private static Uri? SelectPreferredBase(List<string> urls)
|
||||
{
|
||||
if (urls.Count == 0)
|
||||
return null;
|
||||
|
||||
var selected = urls[Random.Shared.Next(urls.Count)];
|
||||
return Uri.TryCreate(selected, UriKind.Absolute, out var uri) ? uri : null;
|
||||
}
|
||||
|
||||
private static Uri? SelectPreferredBase(List<string> hasFileUrls, List<string> pullThroughUrls)
|
||||
{
|
||||
var list = hasFileUrls.Count > 0 ? hasFileUrls : pullThroughUrls;
|
||||
if (list.Count == 0)
|
||||
return null;
|
||||
|
||||
var selected = list[Random.Shared.Next(list.Count)];
|
||||
return Uri.TryCreate(selected, UriKind.Absolute, out var uri) ? uri : null;
|
||||
}
|
||||
|
||||
[HttpGet(LightlessFiles.ServerFiles_DirectDownload + "/{hash}")]
|
||||
[AllowAnonymous]
|
||||
public async Task<IActionResult> DownloadFileDirect(string hash, [FromQuery] long expires, [FromQuery] string signature)
|
||||
{
|
||||
var result = await _cdnDownloadsService.GetDownloadAsync(hash, expires, signature).ConfigureAwait(false);
|
||||
var result = await _cdnDownloadsService.GetDownloadAsync(hash, expires, signature, HttpContext.RequestAborted).ConfigureAwait(false);
|
||||
|
||||
return result.Status switch
|
||||
{
|
||||
CDNDownloadsService.ResultStatus.Disabled => NotFound(),
|
||||
CDNDownloadsService.ResultStatus.Unauthorized => Unauthorized(),
|
||||
CDNDownloadsService.ResultStatus.NotFound => NotFound(),
|
||||
CDNDownloadsService.ResultStatus.Success => PhysicalFile(result.File!.FullName, "application/octet-stream"),
|
||||
CDNDownloadsService.ResultStatus.Success => BuildDirectDownloadResult(result),
|
||||
_ => NotFound()
|
||||
};
|
||||
}
|
||||
|
||||
private IActionResult BuildDirectDownloadResult(CDNDownloadsService.Result result)
|
||||
{
|
||||
if (result.Stream != null)
|
||||
{
|
||||
if (result.ContentLength.HasValue)
|
||||
{
|
||||
Response.ContentLength = result.ContentLength.Value;
|
||||
}
|
||||
|
||||
return new FileStreamResult(result.Stream, "application/octet-stream");
|
||||
}
|
||||
|
||||
return PhysicalFile(result.File!.FullName, "application/octet-stream");
|
||||
}
|
||||
|
||||
[HttpPost(LightlessFiles.ServerFiles_FilesSend)]
|
||||
public async Task<IActionResult> FilesSend([FromBody] FilesSendDto filesSendDto)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user