From 9b4e298b6685422df242764b80d81540dd1ae218 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Wed, 18 Jan 2023 10:20:24 +0100 Subject: [PATCH] refactor server auth on files server, add checking request queue --- MareAPI | 2 +- .../Controllers/CacheController.cs | 4 +-- .../Controllers/ControllerBase.cs | 5 +--- .../Controllers/RequestController.cs | 29 +++++++++++++++---- .../Controllers/ServerFilesController.cs | 2 +- .../Services/CachedFileProvider.cs | 17 ++++++----- 6 files changed, 38 insertions(+), 21 deletions(-) diff --git a/MareAPI b/MareAPI index 5ac8a75..18b93b8 160000 --- a/MareAPI +++ b/MareAPI @@ -1 +1 @@ -Subproject commit 5ac8a753dd1e54ab9ab321119ece462562981e82 +Subproject commit 18b93b889507733cbc775a263f49f7af989bc9cb diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs index 2b39f24..a8669cb 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs @@ -14,7 +14,7 @@ public class CacheController : ControllerBase private readonly RequestQueueService _requestQueue; public CacheController(ILogger logger, RequestFileStreamResultFactory requestFileStreamResultFactory, - CachedFileProvider cachedFileProvider, RequestQueueService requestQueue, ServerTokenGenerator generator) : base(logger, generator) + CachedFileProvider cachedFileProvider, RequestQueueService requestQueue) : base(logger) { _requestFileStreamResultFactory = requestFileStreamResultFactory; _cachedFileProvider = cachedFileProvider; @@ -30,7 +30,7 @@ public class CacheController : ControllerBase _requestQueue.ActivateRequest(requestId); - var fs = await _cachedFileProvider.GetAndDownloadFileStream(request.FileId, Authorization); + var fs = await _cachedFileProvider.GetAndDownloadFileStream(request.FileId); if (fs == null) { _requestQueue.FinishRequest(requestId); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs index edc6f2d..4ccbd64 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs @@ -6,14 +6,11 @@ namespace MareSynchronosStaticFilesServer.Controllers; public class ControllerBase : Controller { protected ILogger _logger; - private readonly ServerTokenGenerator _generator; - public ControllerBase(ILogger logger, ServerTokenGenerator generator) + public ControllerBase(ILogger logger) { _logger = logger; - _generator = generator; } protected string MareUser => HttpContext.User.Claims.First(f => string.Equals(f.Type, MareClaimTypes.Uid, StringComparison.Ordinal)).Value; - protected string Authorization => _generator.Token; } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs index 245c531..0bd3f1b 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs @@ -10,10 +10,9 @@ public class RequestController : ControllerBase { private readonly CachedFileProvider _cachedFileProvider; private readonly RequestQueueService _requestQueue; - private static SemaphoreSlim _parallelRequestSemaphore = new(500); + private static readonly SemaphoreSlim _parallelRequestSemaphore = new(500); - public RequestController(ILogger logger, CachedFileProvider cachedFileProvider, RequestQueueService requestQueue, - ServerTokenGenerator generator) : base(logger, generator) + public RequestController(ILogger logger, CachedFileProvider cachedFileProvider, RequestQueueService requestQueue) : base(logger) { _cachedFileProvider = cachedFileProvider; _requestQueue = requestQueue; @@ -46,7 +45,7 @@ public class RequestController : ControllerBase foreach (var file in files) { _logger.LogDebug("Prerequested file: " + file); - _cachedFileProvider.DownloadFileWhenRequired(file, Authorization); + _cachedFileProvider.DownloadFileWhenRequired(file); } return Ok(); @@ -66,7 +65,7 @@ public class RequestController : ControllerBase { await _parallelRequestSemaphore.WaitAsync(HttpContext.RequestAborted); Guid g = Guid.NewGuid(); - _cachedFileProvider.DownloadFileWhenRequired(file, Authorization); + _cachedFileProvider.DownloadFileWhenRequired(file); await _requestQueue.EnqueueUser(new(g, MareUser, file)); return Ok(g); } @@ -76,4 +75,22 @@ public class RequestController : ControllerBase _parallelRequestSemaphore.Release(); } } -} + + [HttpGet] + [Route(MareFiles.Request_Check)] + public async Task CheckQueueAsync(Guid requestId) + { + try + { + await _parallelRequestSemaphore.WaitAsync(HttpContext.RequestAborted); + if (_requestQueue.StillEnqueued(requestId, MareUser)) + return Ok(); + return BadRequest(); + } + catch (OperationCanceledException) { return BadRequest(); } + finally + { + _parallelRequestSemaphore.Release(); + } + } +} \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs index 5a19f33..313eb73 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs @@ -11,7 +11,7 @@ public class ServerFilesController : ControllerBase { private readonly CachedFileProvider _cachedFileProvider; - public ServerFilesController(ILogger logger, CachedFileProvider cachedFileProvider, ServerTokenGenerator generator) : base(logger, generator) + public ServerFilesController(ILogger logger, CachedFileProvider cachedFileProvider) : base(logger) { _cachedFileProvider = cachedFileProvider; } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs index 460b1e3..0508f12 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs @@ -4,6 +4,7 @@ using MareSynchronosStaticFilesServer.Utils; using System.Collections.Concurrent; using System.Net.Http.Headers; using MareSynchronos.API; +using MareSynchronosShared.Utils; namespace MareSynchronosStaticFilesServer.Services; @@ -12,30 +13,32 @@ public class CachedFileProvider private readonly ILogger _logger; private readonly FileStatisticsService _fileStatisticsService; private readonly MareMetrics _metrics; + private readonly ServerTokenGenerator _generator; private readonly Uri _remoteCacheSourceUri; private readonly string _basePath; private readonly ConcurrentDictionary _currentTransfers = new(StringComparer.Ordinal); private readonly HttpClient _httpClient; private bool IsMainServer => _remoteCacheSourceUri == null; - public CachedFileProvider(IConfigurationService configuration, ILogger logger, FileStatisticsService fileStatisticsService, MareMetrics metrics) + public CachedFileProvider(IConfigurationService configuration, ILogger logger, FileStatisticsService fileStatisticsService, MareMetrics metrics, ServerTokenGenerator generator) { _logger = logger; _fileStatisticsService = fileStatisticsService; _metrics = metrics; + _generator = generator; _remoteCacheSourceUri = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.RemoteCacheSourceUri), null); _basePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); _httpClient = new HttpClient(); } - private async Task DownloadTask(string hash, string auth) + private async Task DownloadTask(string hash) { // download file from remote var downloadUrl = MareFiles.ServerFilesGetFullPath(_remoteCacheSourceUri, hash); _logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl); using var requestMessage = new HttpRequestMessage(HttpMethod.Get, downloadUrl); - requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", auth); + requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _generator.Token); var response = await _httpClient.SendAsync(requestMessage).ConfigureAwait(false); try @@ -63,7 +66,7 @@ public class CachedFileProvider _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_basePath, hash).Length); } - public void DownloadFileWhenRequired(string hash, string auth) + public void DownloadFileWhenRequired(string hash) { var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash); if (fi == null && IsMainServer) return; @@ -72,7 +75,7 @@ public class CachedFileProvider { _currentTransfers[hash] = Task.Run(async () => { - await DownloadTask(hash, auth).ConfigureAwait(false); + await DownloadTask(hash).ConfigureAwait(false); _currentTransfers.Remove(hash, out _); }); } @@ -88,9 +91,9 @@ public class CachedFileProvider return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read); } - public async Task GetAndDownloadFileStream(string hash, string auth) + public async Task GetAndDownloadFileStream(string hash) { - DownloadFileWhenRequired(hash, auth); + DownloadFileWhenRequired(hash); if (_currentTransfers.TryGetValue(hash, out var downloadTask)) {