adds the ability for shards to sync with the main server regarding their caches, when pulling files from main server the shard can stream it to the client directly while downloading and add info for server to report to client regarding file locations across shards

LightlessSyncServer/LightlessSyncStaticFilesServer/Services/ShardFileCleanupService.cs

@@ -12,13 +12,16 @@ public class ShardFileCleanupService : IHostedService
     private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
     private readonly ILogger<MainFileCleanupService> _logger;
     private readonly LightlessMetrics _metrics;
+    private readonly IShardFileInventoryReporter _inventoryReporter;
     private CancellationTokenSource _cleanupCts;
 
-    public ShardFileCleanupService(LightlessMetrics metrics, ILogger<MainFileCleanupService> logger, IConfigurationService<StaticFilesServerConfiguration> configuration)
+    public ShardFileCleanupService(LightlessMetrics metrics, ILogger<MainFileCleanupService> logger, IConfigurationService<StaticFilesServerConfiguration> configuration,
+        IShardFileInventoryReporter inventoryReporter)
     {
         _metrics = metrics;
         _logger = logger;
         _configuration = configuration;
+        _inventoryReporter = inventoryReporter;
         _cacheDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
     }
 
@@ -99,6 +102,7 @@ public class ShardFileCleanupService : IHostedService
                 _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, oldestFile.Length);
                 _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
                 _logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes);
+                ReportRemoval(oldestFile.Name);
                 oldestFile.Delete();
             }
         }
@@ -135,6 +139,7 @@ public class ShardFileCleanupService : IHostedService
                     _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
                     _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
                     _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
+                    ReportRemoval(file.Name);
                     file.Delete();
                 }
                 else if (forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
@@ -142,6 +147,7 @@ public class ShardFileCleanupService : IHostedService
                     _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
                     _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
                     _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
+                    ReportRemoval(file.Name);
                     file.Delete();
                 }
                 else if (file.Length == 0 && !string.Equals(file.Extension, ".dl", StringComparison.OrdinalIgnoreCase))
@@ -149,6 +155,7 @@ public class ShardFileCleanupService : IHostedService
                     _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
                     _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
                     _logger.LogInformation("File with size 0 deleted: {filename}", file.Name);
+                    ReportRemoval(file.Name);
                     file.Delete();
                 }
 
@@ -160,4 +167,12 @@ public class ShardFileCleanupService : IHostedService
             _logger.LogWarning(ex, "Error during file cleanup of old files");
         }
     }
+
+    private void ReportRemoval(string fileName)
+    {
+        if (fileName.Length == 40 && fileName.All(char.IsAsciiLetterOrDigit))
+        {
+            _inventoryReporter.ReportRemoved(fileName);
+        }
+    }
 }