add some more metrics for files downloading from caches

MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs

@@ -13,6 +13,8 @@ public class MetricsAPI
     public const string GaugePairsPaused = "mare_pairs_paused";
     public const string GaugeFilesTotal = "mare_files";
     public const string GaugeFilesTotalSize = "mare_files_size";
+    public const string GaugeFilesDownloadingFromCache = "mare_files_downloading_from_cache";
+    public const string GaugeFilesTasksWaitingForDownloadFromCache = "mare_files_waiting_for_dl";
     public const string CounterUserPushData = "mare_user_push";
     public const string CounterUserPushDataTo = "mare_user_push_to";
     public const string CounterAuthenticationRequests = "mare_auth_requests";

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs

@@ -105,6 +105,7 @@ public sealed class CachedFileProvider : IDisposable
             {
                 try
                 {
+                    _metrics.IncGauge(MetricsAPI.GaugeFilesDownloadingFromCache);
                     await DownloadTask(hash).ConfigureAwait(false);
                 }
                 catch (Exception ex)
@@ -113,6 +114,7 @@ public sealed class CachedFileProvider : IDisposable
                 }
                 finally
                 {
+                    _metrics.DecGauge(MetricsAPI.GaugeFilesDownloadingFromCache);
                     _currentTransfers.Remove(hash, out _);
                 }
             });
@@ -136,9 +138,29 @@ public sealed class CachedFileProvider : IDisposable
 
         if (_currentTransfers.TryGetValue(hash, out var downloadTask))
         {
-            await downloadTask.ConfigureAwait(false);
+            try
+            {
+                using CancellationTokenSource cts = new();
+                cts.CancelAfter(TimeSpan.FromSeconds(15));
+                _metrics.IncGauge(MetricsAPI.GaugeFilesTasksWaitingForDownloadFromCache);
+                await downloadTask.WaitAsync(cts.Token).ConfigureAwait(false);
+            }
+            catch (Exception e)
+            {
+                _logger.LogWarning(e, "Failed while waiting for download task for {hash}", hash);
+                return null;
+            }
+            finally
+            {
+                _metrics.DecGauge(MetricsAPI.GaugeFilesTasksWaitingForDownloadFromCache);
+            }
         }
 
         return GetLocalFileStream(hash);
     }
+
+    public bool AnyFilesDownloading(List<string> hashes)
+    {
+        return hashes.TrueForAll(_currentTransfers.Keys.Contains);
+    }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs

@@ -9,6 +9,7 @@ namespace MareSynchronosStaticFilesServer.Services;
 public class RequestQueueService : IHostedService
 {
     private readonly IClientReadyMessageService _clientReadyMessageService;
+    private readonly CachedFileProvider _cachedFileProvider;
     private readonly ILogger<RequestQueueService> _logger;
     private readonly MareMetrics _metrics;
     private readonly ConcurrentQueue<UserRequest> _queue = new();
@@ -21,7 +22,7 @@ public class RequestQueueService : IHostedService
     private System.Timers.Timer _queueTimer;
 
     public RequestQueueService(MareMetrics metrics, IConfigurationService<StaticFilesServerConfiguration> configurationService,
-        ILogger<RequestQueueService> logger, IClientReadyMessageService hubContext)
+        ILogger<RequestQueueService> logger, IClientReadyMessageService hubContext, CachedFileProvider cachedFileProvider)
     {
         _userQueueRequests = new UserQueueEntry[configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueSize), 50)];
         _queueExpirationSeconds = configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadTimeoutSeconds), 5);
@@ -30,6 +31,7 @@ public class RequestQueueService : IHostedService
         _metrics = metrics;
         _logger = logger;
         _clientReadyMessageService = hubContext;
+        _cachedFileProvider = cachedFileProvider;
     }
 
     public void ActivateRequest(Guid request)
@@ -157,6 +159,7 @@ public class RequestQueueService : IHostedService
                         if (_priorityQueue.TryDequeue(out var prioRequest))
                         {
                             if (prioRequest.IsCancelled) continue;
+                            if (_cachedFileProvider.AnyFilesDownloading(prioRequest.FileIds)) continue;
 
                             DequeueIntoSlot(prioRequest, i);
                             break;
@@ -165,6 +168,7 @@ public class RequestQueueService : IHostedService
                         if (_queue.TryDequeue(out var request))
                         {
                             if (request.IsCancelled) continue;
+                            if (_cachedFileProvider.AnyFilesDownloading(request.FileIds)) continue;
 
                             DequeueIntoSlot(request, i);
                             break;

MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs

@@ -74,6 +74,8 @@ public class Startup
             MetricsAPI.GaugeQueueFree,
             MetricsAPI.GaugeQueueInactive,
             MetricsAPI.GaugeQueueActive,
+            MetricsAPI.GaugeFilesDownloadingFromCache,
+            MetricsAPI.GaugeFilesTasksWaitingForDownloadFromCache
         }));
 
         // generic services