add some file statistics metrics

MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs

@@ -1,4 +1,6 @@
-namespace MareSynchronosShared.Metrics;
+using StackExchange.Redis;
+
+namespace MareSynchronosShared.Metrics;
 
 public class MetricsAPI
 {
@@ -22,4 +24,8 @@ public class MetricsAPI
     public const string GaugeGroups = "mare_groups";
     public const string GaugeGroupPairs = "mare_groups_pairs";
     public const string GaugeGroupPairsPaused = "mare_groups_pairs_paused";
+    public const string GaugeFilesUniquePastHour = "mare_files_unique_past_hour";
+    public const string GaugeFilesUniquePastHourSize = "mare_files_unique_past_hour_size";
+    public const string GaugeFilesUniquePastDay = "mare_files_unique_past_day";
+    public const string GaugeFilesUniquePastDaySize = "mare_files_unique_past_day_size";
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs

@@ -0,0 +1,83 @@
+using MareSynchronosShared.Metrics;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using System;
+using System.Collections.Concurrent;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace MareSynchronosStaticFilesServer;
+
+public class FileStatisticsService : IHostedService
+{
+    private readonly MareMetrics _metrics;
+    private readonly ILogger<FileStatisticsService> _logger;
+    private CancellationTokenSource _resetCancellationTokenSource;
+    private ConcurrentDictionary<string, long> _pastHourFiles = new(StringComparer.Ordinal);
+    private ConcurrentDictionary<string, long> _pastDayFiles = new(StringComparer.Ordinal);
+
+    public FileStatisticsService(MareMetrics metrics, ILogger<FileStatisticsService> logger)
+    {
+        _metrics = metrics;
+        _logger = logger;
+    }
+
+    public void LogFile(string fileHash, long length)
+    {
+        if (!_pastHourFiles.ContainsKey(fileHash))
+        {
+            _pastHourFiles[fileHash] = length;
+            _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHour, _pastHourFiles.Count);
+            _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHourSize, _pastHourFiles.Sum(f => f.Value));
+        }
+        if (!_pastDayFiles.ContainsKey(fileHash))
+        {
+            _pastDayFiles[fileHash] = length;
+            _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDay, _pastDayFiles.Count);
+            _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDaySize, _pastDayFiles.Sum(f => f.Value));
+        }
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Starting FileStatisticsService");
+        _resetCancellationTokenSource = new();
+        _ = ResetHourlyFileData();
+        _ = ResetDailyFileData();
+        return Task.CompletedTask;
+    }
+
+    public async Task ResetHourlyFileData()
+    {
+        while (!_resetCancellationTokenSource.Token.IsCancellationRequested)
+        {
+            _logger.LogInformation("Resetting 1h Data");
+
+            _pastHourFiles = new(StringComparer.Ordinal);
+            _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHour, 0);
+            _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHourSize, 0);
+            await Task.Delay(TimeSpan.FromHours(1), _resetCancellationTokenSource.Token).ConfigureAwait(false);
+        }
+    }
+
+    public async Task ResetDailyFileData()
+    {
+        while (!_resetCancellationTokenSource.Token.IsCancellationRequested)
+        {
+            _logger.LogInformation("Resetting 24h Data");
+
+            _pastDayFiles = new(StringComparer.Ordinal);
+            _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDay, 0);
+            _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDaySize, 0);
+            await Task.Delay(TimeSpan.FromDays(1), _resetCancellationTokenSource.Token).ConfigureAwait(false);
+        }
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        _resetCancellationTokenSource.Cancel();
+        _logger.LogInformation("Stopping FileStatisticsService");
+        return Task.CompletedTask;
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs

@@ -4,34 +4,36 @@ using Microsoft.Extensions.Logging;
 using System.IO;
 using System.Linq;
 using System.Security.Claims;
-using System.Threading.Tasks;
 
-namespace MareSynchronosStaticFilesServer
+namespace MareSynchronosStaticFilesServer;
+
+[Route("/cache")]
+public class FilesController : Controller
 {
-    [Route("/cache")]
-    public class FilesController : Controller
+    private readonly ILogger<FilesController> _logger;
+    private readonly IConfiguration _configuration;
+    private readonly FileStatisticsService _fileStatisticsService;
+
+    public FilesController(ILogger<FilesController> logger, IConfiguration configuration, FileStatisticsService fileStatisticsService)
     {
-        private readonly ILogger<FilesController> logger;
-        private readonly IConfiguration configuration;
+        _logger = logger;
+        _configuration = configuration;
+        _fileStatisticsService = fileStatisticsService;
+    }
 
-        public FilesController(ILogger<FilesController> logger, IConfiguration configuration)
-        {
-            this.logger = logger;
-            this.configuration = configuration;
-        }
+    [HttpGet("{fileId}")]
+    public IActionResult GetFile(string fileId)
+    {
+        var authedUser = HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, ClaimTypes.NameIdentifier, System.StringComparison.Ordinal))?.Value ?? "Unknown";
+        _logger.LogInformation($"GetFile:{authedUser}:{fileId}");
 
-        [HttpGet("{fileId}")]
-        public IActionResult GetFile(string fileId)
-        {
-            var authedUser = HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, ClaimTypes.NameIdentifier, System.StringComparison.Ordinal))?.Value ?? "Unknown";
-            logger.LogInformation($"GetFile:{authedUser}:{fileId}");
+        FileInfo fi = new(Path.Combine(_configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"], fileId));
+        if (!fi.Exists) return NotFound();
 
-            FileInfo fi = new(Path.Combine(configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"], fileId));
-            if (!fi.Exists) return NotFound();
+        _fileStatisticsService.LogFile(fileId, fi.Length);
 
-            var fileStream = new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read);
+        var fileStream = new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read);
 
-            return File(fileStream, "application/octet-stream");
-        }
+        return File(fileStream, "application/octet-stream");
     }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs

@@ -51,7 +51,7 @@ public class Startup
             }
         };
 
-        services.AddSingleton<MareMetrics>(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string>
+        services.AddSingleton(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string>
         {
             MetricsAPI.CounterAuthenticationCacheHits,
             MetricsAPI.CounterAuthenticationFailures,
@@ -60,8 +60,15 @@ public class Startup
         }, new List<string>
         {
             MetricsAPI.GaugeFilesTotalSize,
-            MetricsAPI.GaugeFilesTotal
+            MetricsAPI.GaugeFilesTotal,
+            MetricsAPI.GaugeFilesUniquePastDay,
+            MetricsAPI.GaugeFilesUniquePastDaySize,
+            MetricsAPI.GaugeFilesUniquePastHour,
+            MetricsAPI.GaugeFilesUniquePastHourSize
         }));
+        services.AddSingleton<FileStatisticsService>();
+
+        services.AddHostedService(m => m.GetService<FileStatisticsService>());
         services.AddHostedService<CleanupService>();
 
         services.AddSingleton<SecretKeyAuthenticatorService>();