Initial

LightlessSyncServer/LightlessSyncStaticFilesServer/Utils/BlockFileDataStream.cs

@@ -0,0 +1,108 @@
+
+namespace LightlessSyncStaticFilesServer.Utils;
+
+public sealed class BlockFileDataStream : Stream
+{
+    private readonly IEnumerable<BlockFileDataSubstream> _substreams;
+    private int _currentStreamIndex = 0;
+
+    public BlockFileDataStream(IEnumerable<BlockFileDataSubstream> substreams)
+    {
+        _substreams = substreams;
+    }
+
+    public override bool CanRead => true;
+    public override bool CanSeek => false;
+    public override bool CanWrite => false;
+    public override long Length => throw new NotSupportedException();
+    public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); }
+
+    public override async Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
+    {
+        int totalRead = 0;
+        int currentOffset = 0;
+        int remainingCount = count;
+        while (totalRead < count && _currentStreamIndex < _substreams.Count())
+        {
+            var lastReadBytes = await _substreams.ElementAt(_currentStreamIndex).ReadAsync(buffer, currentOffset, remainingCount, cancellationToken).ConfigureAwait(false);
+            if (lastReadBytes < remainingCount)
+            {
+                _substreams.ElementAt(_currentStreamIndex).Dispose();
+                _currentStreamIndex++;
+            }
+
+            totalRead += lastReadBytes;
+            currentOffset += lastReadBytes;
+            remainingCount -= lastReadBytes;
+        }
+
+        return totalRead;
+    }
+
+    public override int Read(byte[] buffer, int offset, int count)
+    {
+        int totalRead = 0;
+        int currentOffset = 0;
+        int remainingCount = count;
+        while (totalRead < count && _currentStreamIndex < _substreams.Count())
+        {
+            var lastReadBytes = _substreams.ElementAt(_currentStreamIndex).Read(buffer, currentOffset, remainingCount);
+            if (lastReadBytes < remainingCount)
+            {
+                _substreams.ElementAt(_currentStreamIndex).Dispose();
+                _currentStreamIndex++;
+            }
+
+            totalRead += lastReadBytes;
+            currentOffset += lastReadBytes;
+            remainingCount -= lastReadBytes;
+        }
+
+        return totalRead;
+    }
+
+    public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
+    {
+        int totalRead = 0;
+
+        while (totalRead < buffer.Length && _currentStreamIndex < _substreams.Count())
+        {
+            var substream = _substreams.ElementAt(_currentStreamIndex);
+            int bytesRead = await substream.ReadAsync(buffer.Slice(totalRead), cancellationToken).ConfigureAwait(false);
+
+            if (bytesRead == 0)
+            {
+                substream.Dispose();
+                _currentStreamIndex++;
+            }
+            else
+            {
+                totalRead += bytesRead;
+            }
+        }
+
+        return totalRead;
+    }
+
+    protected override void Dispose(bool disposing)
+    {
+        if (disposing)
+        {
+            foreach (var substream in _substreams)
+            {
+                // probably unnecessary but better safe than sorry
+                substream.Dispose();
+            }
+        }
+
+        base.Dispose(disposing);
+    }
+
+    public override void Flush() => throw new NotSupportedException();
+
+    public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
+
+    public override void SetLength(long value) => throw new NotSupportedException();
+
+    public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
+}

LightlessSyncServer/LightlessSyncStaticFilesServer/Utils/BlockFileDataSubstream.cs

@@ -0,0 +1,115 @@
+using System.Globalization;
+using System.Text;
+
+namespace LightlessSyncStaticFilesServer.Utils;
+
+public sealed class BlockFileDataSubstream : IDisposable
+{
+    private readonly MemoryStream _headerStream;
+    private bool _disposed = false;
+    private readonly Lazy<FileStream> _dataStreamLazy;
+    private FileStream DataStream => _dataStreamLazy.Value;
+
+    public BlockFileDataSubstream(FileInfo file)
+    {
+        _dataStreamLazy = new(() => File.Open(file.FullName, GetFileStreamOptions(file.Length)));
+        _headerStream = new MemoryStream(Encoding.ASCII.GetBytes("#" + file.Name + ":" + file.Length.ToString(CultureInfo.InvariantCulture) + "#"));
+    }
+
+    private static FileStreamOptions GetFileStreamOptions(long fileSize)
+    {
+        int bufferSize = fileSize switch
+        {
+            <= 128 * 1024 => 0,
+            <= 512 * 1024 => 4096,
+            <= 1 * 1024 * 1024 => 65536,
+            <= 10 * 1024 * 1024 => 131072,
+            <= 100 * 1024 * 1024 => 524288,
+            _ => 1048576
+        };
+
+        FileStreamOptions opts = new()
+        {
+            Mode = FileMode.Open,
+            Access = FileAccess.Read,
+            Share = FileShare.Read | FileShare.Inheritable,
+            BufferSize = bufferSize
+        };
+
+        return opts;
+    }
+
+    public int Read(byte[] inputBuffer, int offset, int count)
+    {
+        int bytesRead = 0;
+
+        // Read from header stream if it has remaining data
+        if (_headerStream.Position < _headerStream.Length)
+        {
+            int headerBytesToRead = (int)Math.Min(count, _headerStream.Length - _headerStream.Position);
+            bytesRead += _headerStream.Read(inputBuffer, offset, headerBytesToRead);
+            offset += bytesRead;
+            count -= bytesRead;
+        }
+
+        // Read from data stream if there is still space in buffer
+        if (count > 0 && DataStream.Position < DataStream.Length)
+        {
+            bytesRead += DataStream.Read(inputBuffer, offset, count);
+        }
+
+        return bytesRead;
+    }
+
+    public async Task<int> ReadAsync(byte[] inputBuffer, int offset, int count, CancellationToken cancellationToken = default)
+    {
+        int bytesRead = 0;
+
+        // Async read from header stream
+        if (_headerStream.Position < _headerStream.Length)
+        {
+            int headerBytesToRead = (int)Math.Min(count, _headerStream.Length - _headerStream.Position);
+            bytesRead += await _headerStream.ReadAsync(inputBuffer.AsMemory(offset, headerBytesToRead), cancellationToken).ConfigureAwait(false);
+            offset += bytesRead;
+            count -= bytesRead;
+        }
+
+        // Async read from data stream
+        if (count > 0 && DataStream.Position < DataStream.Length)
+        {
+            bytesRead += await DataStream.ReadAsync(inputBuffer.AsMemory(offset, count), cancellationToken).ConfigureAwait(false);
+        }
+
+        return bytesRead;
+    }
+
+    public async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
+    {
+        int bytesRead = 0;
+
+        // Async read from header stream
+        if (_headerStream.Position < _headerStream.Length)
+        {
+            int headerBytesToRead = (int)Math.Min(buffer.Length, _headerStream.Length - _headerStream.Position);
+            bytesRead += await _headerStream.ReadAsync(buffer.Slice(0, headerBytesToRead), cancellationToken).ConfigureAwait(false);
+            buffer = buffer.Slice(headerBytesToRead);
+        }
+
+        // Async read from data stream
+        if (buffer.Length > 0 && DataStream.Position < DataStream.Length)
+        {
+            bytesRead += await DataStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false);
+        }
+
+        return bytesRead;
+    }
+
+    public void Dispose()
+    {
+        if (_disposed) return;
+        _headerStream.Dispose();
+        if (_dataStreamLazy.IsValueCreated)
+            DataStream.Dispose();
+        _disposed = true;
+    }
+}

LightlessSyncServer/LightlessSyncStaticFilesServer/Utils/FilePathUtil.cs

@@ -0,0 +1,31 @@
+namespace LightlessSyncStaticFilesServer.Utils;
+
+public static partial class FilePathUtil
+{
+    public static FileInfo GetFileInfoForHash(string basePath, string hash)
+    {
+        if (hash.Length != 40 || !hash.All(char.IsAsciiLetterOrDigit)) throw new InvalidOperationException();
+
+        FileInfo fi = new(Path.Join(basePath, hash[0].ToString(), hash));
+        if (!fi.Exists)
+        {
+            fi = new FileInfo(Path.Join(basePath, hash));
+            if (!fi.Exists)
+            {
+                return null;
+            }
+        }
+
+        return fi;
+    }
+
+    public static string GetFilePath(string basePath, string hash)
+    {
+        if (hash.Length != 40 || !hash.All(char.IsAsciiLetterOrDigit)) throw new InvalidOperationException();
+
+        var dirPath = Path.Join(basePath, hash[0].ToString());
+        var path = Path.Join(dirPath, hash);
+        if (!Directory.Exists(dirPath)) Directory.CreateDirectory(dirPath);
+        return path;
+    }
+}

LightlessSyncServer/LightlessSyncStaticFilesServer/Utils/RequestFileStreamResult.cs

@@ -0,0 +1,58 @@
+using LightlessSyncShared.Metrics;
+using LightlessSyncStaticFilesServer.Services;
+using Microsoft.AspNetCore.Mvc;
+
+namespace LightlessSyncStaticFilesServer.Utils;
+
+public class RequestFileStreamResult : FileStreamResult
+{
+    private readonly Guid _requestId;
+    private readonly RequestQueueService _requestQueueService;
+    private readonly LightlessMetrics _lightlessMetrics;
+
+    public RequestFileStreamResult(Guid requestId, RequestQueueService requestQueueService, LightlessMetrics lightlessMetrics,
+        Stream fileStream, string contentType) : base(fileStream, contentType)
+    {
+        _requestId = requestId;
+        _requestQueueService = requestQueueService;
+        _lightlessMetrics = lightlessMetrics;
+        _lightlessMetrics.IncGauge(MetricsAPI.GaugeCurrentDownloads);
+    }
+
+    public override void ExecuteResult(ActionContext context)
+    {
+        try
+        {
+            base.ExecuteResult(context);
+        }
+        catch
+        {
+            throw;
+        }
+        finally
+        {
+            _requestQueueService.FinishRequest(_requestId);
+
+            _lightlessMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads);
+            FileStream?.Dispose();
+        }
+    }
+
+    public override async Task ExecuteResultAsync(ActionContext context)
+    {
+        try
+        {
+            await base.ExecuteResultAsync(context).ConfigureAwait(false);
+        }
+        catch
+        {
+            throw;
+        }
+        finally
+        {
+            _requestQueueService.FinishRequest(_requestId);
+            _lightlessMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads);
+            FileStream?.Dispose();
+        }
+    }
+}

LightlessSyncServer/LightlessSyncStaticFilesServer/Utils/RequestFileStreamResultFactory.cs

@@ -0,0 +1,26 @@
+using LightlessSyncShared.Metrics;
+using LightlessSyncShared.Services;
+using LightlessSyncShared.Utils.Configuration;
+using LightlessSyncStaticFilesServer.Services;
+
+namespace LightlessSyncStaticFilesServer.Utils;
+
+public class RequestFileStreamResultFactory
+{
+    private readonly LightlessMetrics _metrics;
+    private readonly RequestQueueService _requestQueueService;
+    private readonly IConfigurationService<StaticFilesServerConfiguration> _configurationService;
+
+    public RequestFileStreamResultFactory(LightlessMetrics metrics, RequestQueueService requestQueueService, IConfigurationService<StaticFilesServerConfiguration> configurationService)
+    {
+        _metrics = metrics;
+        _requestQueueService = requestQueueService;
+        _configurationService = configurationService;
+    }
+
+    public RequestFileStreamResult Create(Guid requestId, Stream stream)
+    {
+        return new RequestFileStreamResult(requestId, _requestQueueService,
+            _metrics, stream, "application/octet-stream");
+    }
+}

LightlessSyncServer/LightlessSyncStaticFilesServer/Utils/UserQueueEntry.cs

@@ -0,0 +1,21 @@
+namespace LightlessSyncStaticFilesServer.Utils;
+
+public class UserQueueEntry
+{
+    public UserQueueEntry(UserRequest userRequest, DateTime expirationDate)
+    {
+        UserRequest = userRequest;
+        ExpirationDate = expirationDate;
+    }
+
+    public void MarkActive()
+    {
+        IsActive = true;
+        ActivationDate = DateTime.UtcNow;
+    }
+
+    public UserRequest UserRequest { get; }
+    public DateTime ExpirationDate { get; }
+    public bool IsActive { get; private set; } = false;
+    public DateTime ActivationDate { get; private set; }
+}

LightlessSyncServer/LightlessSyncStaticFilesServer/Utils/UserRequest.cs

@@ -0,0 +1,6 @@
+namespace LightlessSyncStaticFilesServer.Utils;
+
+public record UserRequest(Guid RequestId, string User, List<string> FileIds)
+{
+    public bool IsCancelled { get; set; } = false;
+}