Add Server-Side Download Queue (#21)

* test add queueing to file service

* further adjustments to download queueing

* add check for whether the request is still in the queue to CheckQueue

* forcefully release slot if download didn't finish in 15s

* actually cancel the delay task

* add metrics and refactor some of the request queue service

* refactor pathing

* reuse httpclient

* add queue request dto to requestfile, enqueue users immediately if a slot is available

* change startup to include all controllers

* update server pathing

* update pathing, again

* several adjustments to auth, banning, jwt server tokens, renaming, authorization

* update api I guess

* adjust automated banning of charaident and reg

* generate jwt on servers for internal authentication

* remove mvcextensions

Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs

@@ -0,0 +1,42 @@
+using MareSynchronos.API;
+using MareSynchronosShared.Utils;
+using MareSynchronosStaticFilesServer.Services;
+using MareSynchronosStaticFilesServer.Utils;
+using Microsoft.AspNetCore.Mvc;
+
+namespace MareSynchronosStaticFilesServer.Controllers;
+
+[Route(MareFiles.Cache)]
+public class CacheController : ControllerBase
+{
+    private readonly RequestFileStreamResultFactory _requestFileStreamResultFactory;
+    private readonly CachedFileProvider _cachedFileProvider;
+    private readonly RequestQueueService _requestQueue;
+
+    public CacheController(ILogger<CacheController> logger, RequestFileStreamResultFactory requestFileStreamResultFactory,
+        CachedFileProvider cachedFileProvider, RequestQueueService requestQueue, ServerTokenGenerator generator) : base(logger, generator)
+    {
+        _requestFileStreamResultFactory = requestFileStreamResultFactory;
+        _cachedFileProvider = cachedFileProvider;
+        _requestQueue = requestQueue;
+    }
+
+    [HttpGet(MareFiles.Cache_Get)]
+    public async Task<IActionResult> GetFile(Guid requestId)
+    {
+        _logger.LogDebug($"GetFile:{MareUser}:{requestId}");
+
+        if (!_requestQueue.IsActiveProcessing(requestId, MareUser, out var request)) return BadRequest();
+
+        _requestQueue.ActivateRequest(requestId);
+
+        var fs = await _cachedFileProvider.GetAndDownloadFileStream(request.FileId, Authorization);
+        if (fs == null)
+        {
+            _requestQueue.FinishRequest(requestId);
+            return NotFound();
+        }
+
+        return _requestFileStreamResultFactory.Create(requestId, fs);
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs

@@ -0,0 +1,19 @@
+using MareSynchronosShared.Utils;
+using Microsoft.AspNetCore.Mvc;
+
+namespace MareSynchronosStaticFilesServer.Controllers;
+
+public class ControllerBase : Controller
+{
+    protected ILogger _logger;
+    private readonly ServerTokenGenerator _generator;
+
+    public ControllerBase(ILogger logger, ServerTokenGenerator generator)
+    {
+        _logger = logger;
+        _generator = generator;
+    }
+
+    protected string MareUser => HttpContext.User.Claims.First(f => string.Equals(f.Type, MareClaimTypes.Uid, StringComparison.Ordinal)).Value;
+    protected string Authorization => "Bearer " + _generator.Token;
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs

@@ -0,0 +1,54 @@
+using MareSynchronos.API;
+using MareSynchronosShared.Utils;
+using MareSynchronosStaticFilesServer.Services;
+using Microsoft.AspNetCore.Mvc;
+using System.Text.Json;
+
+namespace MareSynchronosStaticFilesServer.Controllers;
+
+[Route(MareFiles.Request)]
+public class RequestController : ControllerBase
+{
+    private readonly CachedFileProvider _cachedFileProvider;
+    private readonly RequestQueueService _requestQueue;
+
+    public RequestController(ILogger<RequestController> logger, CachedFileProvider cachedFileProvider, RequestQueueService requestQueue, 
+        ServerTokenGenerator generator) : base(logger, generator)
+    {
+        _cachedFileProvider = cachedFileProvider;
+        _requestQueue = requestQueue;
+    }
+
+    [HttpPost]
+    [Route(MareFiles.Request_Enqueue)]
+    public IActionResult PreRequestFiles([FromBody] List<string> files)
+    {
+        foreach (var file in files)
+        {
+            _cachedFileProvider.DownloadFileWhenRequired(file, Authorization);
+        }
+
+        return Ok();
+    }
+
+    [HttpGet]
+    [Route(MareFiles.Request_RequestFile)]
+    public async Task<IActionResult> RequestFile(string file)
+    {
+        Guid g = Guid.NewGuid();
+        _cachedFileProvider.DownloadFileWhenRequired(file, Authorization);
+        var queueStatus = await _requestQueue.EnqueueUser(new(g, MareUser, file));
+        return Ok(JsonSerializer.Serialize(new QueueRequestDto(g, queueStatus)));
+    }
+
+    [HttpGet]
+    [Route(MareFiles.Request_CheckQueue)]
+    public IActionResult CheckQueue(Guid requestId)
+    {
+        if (_requestQueue.IsActiveProcessing(requestId, MareUser, out _)) return Ok();
+
+        if (_requestQueue.StillEnqueued(requestId, MareUser, out int position)) return Conflict(position);
+
+        return BadRequest();
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs

@@ -0,0 +1,30 @@
+using MareSynchronos.API;
+using MareSynchronosShared.Utils;
+using MareSynchronosStaticFilesServer.Services;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+
+namespace MareSynchronosStaticFilesServer.Controllers;
+
+[Route(MareFiles.ServerFiles)]
+public class ServerFilesController : ControllerBase
+{
+    private readonly CachedFileProvider _cachedFileProvider;
+
+    public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider, ServerTokenGenerator generator) : base(logger, generator)
+    {
+        _cachedFileProvider = cachedFileProvider;
+    }
+
+    [HttpGet(MareFiles.ServerFiles_Get + "/{fileId}")]
+    [Authorize(Policy = "Internal")]
+    public async Task<IActionResult> GetFile(string fileId)
+    {
+        _logger.LogInformation($"GetFile:{MareUser}:{fileId}");
+
+        var fs = _cachedFileProvider.GetLocalFileStream(fileId);
+        if (fs == null) return NotFound();
+
+        return File(fs, "application/octet-stream");
+    }
+}