Add Server-Side Download Queue (#21)

* test add queueing to file service

* further adjustments to download queueing

* add check for whether the request is still in the queue to CheckQueue

* forcefully release slot if download didn't finish in 15s

* actually cancel the delay task

* add metrics and refactor some of the request queue service

* refactor pathing

* reuse httpclient

* add queue request dto to requestfile, enqueue users immediately if a slot is available

* change startup to include all controllers

* update server pathing

* update pathing, again

* several adjustments to auth, banning, jwt server tokens, renaming, authorization

* update api I guess

* adjust automated banning of charaident and reg

* generate jwt on servers for internal authentication

* remove mvcextensions

Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/FilePathUtil.cs

@@ -0,0 +1,27 @@
+namespace MareSynchronosStaticFilesServer.Utils;
+
+public static class FilePathUtil
+{
+    public static FileInfo GetFileInfoForHash(string basePath, string hash)
+    {
+        FileInfo fi = new(Path.Combine(basePath, hash[0].ToString(), hash));
+        if (!fi.Exists)
+        {
+            fi = new FileInfo(Path.Combine(basePath, hash));
+            if (!fi.Exists)
+            {
+                return null;
+            }
+        }
+
+        return fi;
+    }
+
+    public static string GetFilePath(string basePath, string hash)
+    {
+        var dirPath = Path.Combine(basePath, hash[0].ToString());
+        var path = Path.Combine(dirPath, hash);
+        if (!Directory.Exists(dirPath)) Directory.CreateDirectory(dirPath);
+        return path;
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs

@@ -0,0 +1,59 @@
+using MareSynchronosShared.Metrics;
+using MareSynchronosStaticFilesServer.Services;
+using Microsoft.AspNetCore.Mvc;
+
+namespace MareSynchronosStaticFilesServer.Utils;
+
+public class RequestFileStreamResult : FileStreamResult
+{
+    private readonly Guid _requestId;
+    private readonly RequestQueueService _requestQueueService;
+    private readonly MareMetrics _mareMetrics;
+    private readonly CancellationTokenSource _releaseCts = new();
+    private bool _releasedSlot = false;
+
+    public RequestFileStreamResult(Guid requestId, int secondsUntilRelease, RequestQueueService requestQueueService,
+        MareMetrics mareMetrics, Stream fileStream, string contentType) : base(fileStream, contentType)
+    {
+        _requestId = requestId;
+        _requestQueueService = requestQueueService;
+        _mareMetrics = mareMetrics;
+        _mareMetrics.IncGauge(MetricsAPI.GaugeCurrentDownloads);
+
+        // forcefully release slot after secondsUntilRelease
+        Task.Run(async () =>
+        {
+            try
+            {
+                await Task.Delay(TimeSpan.FromSeconds(secondsUntilRelease), _releaseCts.Token).ConfigureAwait(false);
+                _requestQueueService.FinishRequest(_requestId);
+                _releasedSlot = true;
+            }
+            catch { }
+        });
+    }
+
+    public override void ExecuteResult(ActionContext context)
+    {
+        base.ExecuteResult(context);
+
+        _releaseCts.Cancel();
+
+        if (!_releasedSlot)
+            _requestQueueService.FinishRequest(_requestId);
+
+        _mareMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads);
+    }
+
+    public override async Task ExecuteResultAsync(ActionContext context)
+    {
+        await base.ExecuteResultAsync(context).ConfigureAwait(false);
+
+        _releaseCts.Cancel();
+
+        if (!_releasedSlot)
+            _requestQueueService.FinishRequest(_requestId);
+
+        _mareMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads);
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResultFactory.cs

@@ -0,0 +1,25 @@
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Services;
+using MareSynchronosStaticFilesServer.Services;
+
+namespace MareSynchronosStaticFilesServer.Utils;
+
+public class RequestFileStreamResultFactory
+{
+    private readonly MareMetrics _metrics;
+    private readonly RequestQueueService _requestQueueService;
+    private readonly IConfigurationService<StaticFilesServerConfiguration> _configurationService;
+
+    public RequestFileStreamResultFactory(MareMetrics metrics, RequestQueueService requestQueueService, IConfigurationService<StaticFilesServerConfiguration> configurationService)
+    {
+        _metrics = metrics;
+        _requestQueueService = requestQueueService;
+        _configurationService = configurationService;
+    }
+
+    public RequestFileStreamResult Create(Guid requestId, FileStream fs)
+    {
+        return new RequestFileStreamResult(requestId, _configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueReleaseSeconds), 15),
+            _requestQueueService, _metrics, fs, "application/octet-stream");
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserQueueEntry.cs

@@ -0,0 +1,6 @@
+namespace MareSynchronosStaticFilesServer.Utils;
+
+public record UserQueueEntry(UserRequest UserRequest, DateTime ExpirationDate)
+{
+    public bool IsActive { get; set; } = false;
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserRequest.cs

@@ -0,0 +1,3 @@
+namespace MareSynchronosStaticFilesServer.Utils;
+
+public record UserRequest(Guid RequestId, string User, string FileId);