From 012ae04dce1b21b208188880aa9c9258f4d9927f Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Fri, 1 Mar 2024 11:13:10 +0100 Subject: [PATCH] add some more metrics for files downloading from caches --- .../Metrics/MetricsAPI.cs | 2 ++ .../Services/CachedFileProvider.cs | 24 ++++++++++++++++++- .../Services/RequestQueueService.cs | 6 ++++- .../Startup.cs | 2 ++ 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs b/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs index c19caaf..cb78e67 100644 --- a/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs +++ b/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs @@ -13,6 +13,8 @@ public class MetricsAPI public const string GaugePairsPaused = "mare_pairs_paused"; public const string GaugeFilesTotal = "mare_files"; public const string GaugeFilesTotalSize = "mare_files_size"; + public const string GaugeFilesDownloadingFromCache = "mare_files_downloading_from_cache"; + public const string GaugeFilesTasksWaitingForDownloadFromCache = "mare_files_waiting_for_dl"; public const string CounterUserPushData = "mare_user_push"; public const string CounterUserPushDataTo = "mare_user_push_to"; public const string CounterAuthenticationRequests = "mare_auth_requests"; diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs index b7cf52e..3617cd3 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs @@ -105,6 +105,7 @@ public sealed class CachedFileProvider : IDisposable { try { + _metrics.IncGauge(MetricsAPI.GaugeFilesDownloadingFromCache); await DownloadTask(hash).ConfigureAwait(false); } catch (Exception ex) @@ -113,6 +114,7 @@ public sealed class CachedFileProvider : IDisposable } finally { + _metrics.DecGauge(MetricsAPI.GaugeFilesDownloadingFromCache); _currentTransfers.Remove(hash, out _); } }); @@ -136,9 +138,29 @@ public sealed class CachedFileProvider : IDisposable if (_currentTransfers.TryGetValue(hash, out var downloadTask)) { - await downloadTask.ConfigureAwait(false); + try + { + using CancellationTokenSource cts = new(); + cts.CancelAfter(TimeSpan.FromSeconds(15)); + _metrics.IncGauge(MetricsAPI.GaugeFilesTasksWaitingForDownloadFromCache); + await downloadTask.WaitAsync(cts.Token).ConfigureAwait(false); + } + catch (Exception e) + { + _logger.LogWarning(e, "Failed while waiting for download task for {hash}", hash); + return null; + } + finally + { + _metrics.DecGauge(MetricsAPI.GaugeFilesTasksWaitingForDownloadFromCache); + } } return GetLocalFileStream(hash); } + + public bool AnyFilesDownloading(List hashes) + { + return hashes.TrueForAll(_currentTransfers.Keys.Contains); + } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs index f7911d6..a408c42 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs @@ -9,6 +9,7 @@ namespace MareSynchronosStaticFilesServer.Services; public class RequestQueueService : IHostedService { private readonly IClientReadyMessageService _clientReadyMessageService; + private readonly CachedFileProvider _cachedFileProvider; private readonly ILogger _logger; private readonly MareMetrics _metrics; private readonly ConcurrentQueue _queue = new(); @@ -21,7 +22,7 @@ public class RequestQueueService : IHostedService private System.Timers.Timer _queueTimer; public RequestQueueService(MareMetrics metrics, IConfigurationService configurationService, - ILogger logger, IClientReadyMessageService hubContext) + ILogger logger, IClientReadyMessageService hubContext, CachedFileProvider cachedFileProvider) { _userQueueRequests = new UserQueueEntry[configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueSize), 50)]; _queueExpirationSeconds = configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadTimeoutSeconds), 5); @@ -30,6 +31,7 @@ public class RequestQueueService : IHostedService _metrics = metrics; _logger = logger; _clientReadyMessageService = hubContext; + _cachedFileProvider = cachedFileProvider; } public void ActivateRequest(Guid request) @@ -157,6 +159,7 @@ public class RequestQueueService : IHostedService if (_priorityQueue.TryDequeue(out var prioRequest)) { if (prioRequest.IsCancelled) continue; + if (_cachedFileProvider.AnyFilesDownloading(prioRequest.FileIds)) continue; DequeueIntoSlot(prioRequest, i); break; @@ -165,6 +168,7 @@ public class RequestQueueService : IHostedService if (_queue.TryDequeue(out var request)) { if (request.IsCancelled) continue; + if (_cachedFileProvider.AnyFilesDownloading(request.FileIds)) continue; DequeueIntoSlot(request, i); break; diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs index c915a6a..60602c0 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs @@ -74,6 +74,8 @@ public class Startup MetricsAPI.GaugeQueueFree, MetricsAPI.GaugeQueueInactive, MetricsAPI.GaugeQueueActive, + MetricsAPI.GaugeFilesDownloadingFromCache, + MetricsAPI.GaugeFilesTasksWaitingForDownloadFromCache })); // generic services