make cleanup check time configurable, add distribution file server address

handle download errors on shards better

flush filestream

oml

MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs

@@ -7,6 +7,7 @@ public class StaticFilesServerConfiguration : MareConfigurationBase
 {
     public bool IsDistributionNode { get; set; } = false;
     public Uri? MainFileServerAddress { get; set; } = null;
+    public Uri? DistributionFileServerAddress { get; set; } = null;
     public int ForcedDeletionOfFilesAfterHours { get; set; } = -1;
     public double CacheSizeHardLimitInGiB { get; set; } = -1;
     public int UnusedFileRetentionPeriodInDays { get; set; } = 14;
@@ -15,6 +16,7 @@ public class StaticFilesServerConfiguration : MareConfigurationBase
     public int DownloadTimeoutSeconds { get; set; } = 5;
     public int DownloadQueueReleaseSeconds { get; set; } = 15;
     public int DownloadQueueClearLimit { get; set; } = 15000;
+    public int CleanupCheckInMinutes { get; set; } = 15;
     [RemoteConfiguration]
     public Uri CdnFullUrl { get; set; } = null;
     [RemoteConfiguration]

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs

@@ -30,7 +30,7 @@ public sealed class CachedFileProvider : IDisposable
         _fileStatisticsService = fileStatisticsService;
         _metrics = metrics;
         _generator = generator;
-        _remoteCacheSourceUri = configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.MainFileServerAddress), null);
+        _remoteCacheSourceUri = configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.DistributionFileServerAddress), null);
         _isDistributionServer = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false);
         _basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
         _httpClient = new();
@@ -69,7 +69,8 @@ public sealed class CachedFileProvider : IDisposable
         }
 
         var fileName = FilePathUtil.GetFilePath(_basePath, hash);
-        using var fileStream = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite);
+        var tempFileName = fileName + ".dl";
+        var fileStream = new FileStream(tempFileName, FileMode.Create, FileAccess.ReadWrite);
         var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024;
         var buffer = new byte[bufferSize];
 
@@ -79,6 +80,9 @@ public sealed class CachedFileProvider : IDisposable
         {
             await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead)).ConfigureAwait(false);
         }
+        await fileStream.FlushAsync().ConfigureAwait(false);
+        await fileStream.DisposeAsync().ConfigureAwait(false);
+        File.Move(tempFileName, fileName, true);
 
         _metrics.IncGauge(MetricsAPI.GaugeFilesTotal);
         _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_basePath, hash).Length);
@@ -90,7 +94,7 @@ public sealed class CachedFileProvider : IDisposable
         if (fi == null && IsMainServer) return;
 
         await _downloadSemaphore.WaitAsync().ConfigureAwait(false);
-        if (fi == null && !_currentTransfers.ContainsKey(hash))
+        if ((fi == null || (fi?.Length ?? 0) == 0) && !_currentTransfers.ContainsKey(hash))
         {
             _currentTransfers[hash] = Task.Run(async () =>
             {

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs

@@ -17,7 +17,8 @@ public class MainFileCleanupService : IHostedService
     private readonly IServiceProvider _services;
     private CancellationTokenSource _cleanupCts;
 
-    public MainFileCleanupService(MareMetrics metrics, ILogger<MainFileCleanupService> logger, IServiceProvider services, IConfigurationService<StaticFilesServerConfiguration> configuration)
+    public MainFileCleanupService(MareMetrics metrics, ILogger<MainFileCleanupService> logger,
+        IServiceProvider services, IConfigurationService<StaticFilesServerConfiguration> configuration)
     {
         _metrics = metrics;
         _logger = logger;
@@ -54,11 +55,11 @@ public class MainFileCleanupService : IHostedService
             {
                 _logger.LogError(e, "Error during cleanup task");
             }
-
+            var cleanupCheckMinutes = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CleanupCheckInMinutes), 15);
             var now = DateTime.Now;
             TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
-            TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % 10, 0);
+            TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % cleanupCheckMinutes, 0);
-            var span = futureTime.AddMinutes(10) - currentTime;
+            var span = futureTime.AddMinutes(cleanupCheckMinutes) - currentTime;
 
             _logger.LogInformation("File Cleanup Complete, next run at {date}", now.Add(span));
             await Task.Delay(span, ct).ConfigureAwait(false);

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardFileCleanupService.cs

@@ -121,7 +121,6 @@ public class ShardFileCleanupService : IHostedService
             var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
             DirectoryInfo dir = new(_cacheDir);
             var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories);
-            int fileCounter = 0;
 
             foreach (var file in allFilesInDir)
             {
@@ -139,8 +138,13 @@ public class ShardFileCleanupService : IHostedService
                     _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
                     file.Delete();
                 }
-
+                else if (file.Length == 0 && !string.Equals(file.Extension, ".dl", StringComparison.OrdinalIgnoreCase))
-                fileCounter++;
+                {
+                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
+                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
+                    _logger.LogInformation("File with size 0 deleted: {filename}", file.Name);
+                    file.Delete();
+                }
 
                 ct.ThrowIfCancellationRequested();
             }