From 7d3297e1e87fe1215ae4adf16ec12a59c147b969 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Tue, 20 Dec 2022 05:02:03 +0100 Subject: [PATCH] add some file statistics metrics --- .../Metrics/MetricsAPI.cs | 8 +- .../FileStatisticsService.cs | 83 +++++++++++++++++++ .../FilesController.cs | 44 +++++----- .../Startup.cs | 11 ++- 4 files changed, 122 insertions(+), 24 deletions(-) create mode 100644 MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs diff --git a/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs b/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs index d46cc63..d573720 100644 --- a/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs +++ b/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs @@ -1,4 +1,6 @@ -namespace MareSynchronosShared.Metrics; +using StackExchange.Redis; + +namespace MareSynchronosShared.Metrics; public class MetricsAPI { @@ -22,4 +24,8 @@ public class MetricsAPI public const string GaugeGroups = "mare_groups"; public const string GaugeGroupPairs = "mare_groups_pairs"; public const string GaugeGroupPairsPaused = "mare_groups_pairs_paused"; + public const string GaugeFilesUniquePastHour = "mare_files_unique_past_hour"; + public const string GaugeFilesUniquePastHourSize = "mare_files_unique_past_hour_size"; + public const string GaugeFilesUniquePastDay = "mare_files_unique_past_day"; + public const string GaugeFilesUniquePastDaySize = "mare_files_unique_past_day_size"; } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs new file mode 100644 index 0000000..f6dd157 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs @@ -0,0 +1,83 @@ +using MareSynchronosShared.Metrics; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Concurrent; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MareSynchronosStaticFilesServer; + +public class FileStatisticsService : IHostedService +{ + private readonly MareMetrics _metrics; + private readonly ILogger _logger; + private CancellationTokenSource _resetCancellationTokenSource; + private ConcurrentDictionary _pastHourFiles = new(StringComparer.Ordinal); + private ConcurrentDictionary _pastDayFiles = new(StringComparer.Ordinal); + + public FileStatisticsService(MareMetrics metrics, ILogger logger) + { + _metrics = metrics; + _logger = logger; + } + + public void LogFile(string fileHash, long length) + { + if (!_pastHourFiles.ContainsKey(fileHash)) + { + _pastHourFiles[fileHash] = length; + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHour, _pastHourFiles.Count); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHourSize, _pastHourFiles.Sum(f => f.Value)); + } + if (!_pastDayFiles.ContainsKey(fileHash)) + { + _pastDayFiles[fileHash] = length; + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDay, _pastDayFiles.Count); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDaySize, _pastDayFiles.Sum(f => f.Value)); + } + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Starting FileStatisticsService"); + _resetCancellationTokenSource = new(); + _ = ResetHourlyFileData(); + _ = ResetDailyFileData(); + return Task.CompletedTask; + } + + public async Task ResetHourlyFileData() + { + while (!_resetCancellationTokenSource.Token.IsCancellationRequested) + { + _logger.LogInformation("Resetting 1h Data"); + + _pastHourFiles = new(StringComparer.Ordinal); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHour, 0); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHourSize, 0); + await Task.Delay(TimeSpan.FromHours(1), _resetCancellationTokenSource.Token).ConfigureAwait(false); + } + } + + public async Task ResetDailyFileData() + { + while (!_resetCancellationTokenSource.Token.IsCancellationRequested) + { + _logger.LogInformation("Resetting 24h Data"); + + _pastDayFiles = new(StringComparer.Ordinal); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDay, 0); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDaySize, 0); + await Task.Delay(TimeSpan.FromDays(1), _resetCancellationTokenSource.Token).ConfigureAwait(false); + } + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _resetCancellationTokenSource.Cancel(); + _logger.LogInformation("Stopping FileStatisticsService"); + return Task.CompletedTask; + } +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs index 53464af..b3f63af 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs @@ -4,34 +4,36 @@ using Microsoft.Extensions.Logging; using System.IO; using System.Linq; using System.Security.Claims; -using System.Threading.Tasks; -namespace MareSynchronosStaticFilesServer +namespace MareSynchronosStaticFilesServer; + +[Route("/cache")] +public class FilesController : Controller { - [Route("/cache")] - public class FilesController : Controller + private readonly ILogger _logger; + private readonly IConfiguration _configuration; + private readonly FileStatisticsService _fileStatisticsService; + + public FilesController(ILogger logger, IConfiguration configuration, FileStatisticsService fileStatisticsService) { - private readonly ILogger logger; - private readonly IConfiguration configuration; + _logger = logger; + _configuration = configuration; + _fileStatisticsService = fileStatisticsService; + } - public FilesController(ILogger logger, IConfiguration configuration) - { - this.logger = logger; - this.configuration = configuration; - } + [HttpGet("{fileId}")] + public IActionResult GetFile(string fileId) + { + var authedUser = HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, ClaimTypes.NameIdentifier, System.StringComparison.Ordinal))?.Value ?? "Unknown"; + _logger.LogInformation($"GetFile:{authedUser}:{fileId}"); - [HttpGet("{fileId}")] - public IActionResult GetFile(string fileId) - { - var authedUser = HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, ClaimTypes.NameIdentifier, System.StringComparison.Ordinal))?.Value ?? "Unknown"; - logger.LogInformation($"GetFile:{authedUser}:{fileId}"); + FileInfo fi = new(Path.Combine(_configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"], fileId)); + if (!fi.Exists) return NotFound(); - FileInfo fi = new(Path.Combine(configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"], fileId)); - if (!fi.Exists) return NotFound(); + _fileStatisticsService.LogFile(fileId, fi.Length); - var fileStream = new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read); + var fileStream = new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read); - return File(fileStream, "application/octet-stream"); - } + return File(fileStream, "application/octet-stream"); } } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs index 002b8d1..7507969 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs @@ -51,7 +51,7 @@ public class Startup } }; - services.AddSingleton(m => new MareMetrics(m.GetService>(), new List + services.AddSingleton(m => new MareMetrics(m.GetService>(), new List { MetricsAPI.CounterAuthenticationCacheHits, MetricsAPI.CounterAuthenticationFailures, @@ -60,8 +60,15 @@ public class Startup }, new List { MetricsAPI.GaugeFilesTotalSize, - MetricsAPI.GaugeFilesTotal + MetricsAPI.GaugeFilesTotal, + MetricsAPI.GaugeFilesUniquePastDay, + MetricsAPI.GaugeFilesUniquePastDaySize, + MetricsAPI.GaugeFilesUniquePastHour, + MetricsAPI.GaugeFilesUniquePastHourSize })); + services.AddSingleton(); + + services.AddHostedService(m => m.GetService()); services.AddHostedService(); services.AddSingleton();