diff --git a/MareAPI b/MareAPI index a5373bc..4aacbb7 160000 --- a/MareAPI +++ b/MareAPI @@ -1 +1 @@ -Subproject commit a5373bca246ee3dab590a2eac68d974ba546ee8c +Subproject commit 4aacbb78bb2793ae4e3f0f1b809e491e2cae7567 diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs index a81b3c5..afca305 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs @@ -2,6 +2,8 @@ using MareSynchronosStaticFilesServer.Services; using MareSynchronosStaticFilesServer.Utils; using Microsoft.AspNetCore.Mvc; +using System.Globalization; +using System.Text; namespace MareSynchronosStaticFilesServer.Controllers; @@ -21,7 +23,7 @@ public class CacheController : ControllerBase } [HttpGet(MareFiles.Cache_Get)] - public async Task GetFile(Guid requestId) + public async Task GetFiles(Guid requestId) { _logger.LogDebug($"GetFile:{MareUser}:{requestId}"); @@ -29,13 +31,23 @@ public class CacheController : ControllerBase _requestQueue.ActivateRequest(requestId); - var fs = await _cachedFileProvider.GetAndDownloadFileStream(request.FileId); - if (fs == null) + Response.ContentType = "application/octet-stream"; + var memoryStream = new MemoryStream(); + var streamWriter = new BinaryWriter(memoryStream); + + foreach (var file in request.FileIds) { - _requestQueue.FinishRequest(requestId); - return NotFound(); + var fs = await _cachedFileProvider.GetAndDownloadFileStream(file); + if (fs == null) continue; + streamWriter.Write(Encoding.ASCII.GetBytes("#" + file + ":" + fs.Length.ToString(CultureInfo.InvariantCulture) + "#")); + byte[] buffer = new byte[fs.Length]; + _ = await fs.ReadAsync(buffer, HttpContext.RequestAborted); + streamWriter.Write(buffer); } - return _requestFileStreamResultFactory.Create(requestId, fs); + streamWriter.Flush(); + memoryStream.Position = 0; + + return _requestFileStreamResultFactory.Create(requestId, memoryStream); } } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs index fd52936..7d4609d 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs @@ -36,7 +36,7 @@ public class RequestController : ControllerBase [HttpPost] [Route(MareFiles.Request_Enqueue)] - public async Task PreRequestFilesAsync([FromBody] List files) + public async Task PreRequestFilesAsync([FromBody] IEnumerable files) { try { @@ -47,43 +47,26 @@ public class RequestController : ControllerBase _cachedFileProvider.DownloadFileWhenRequired(file); } - return Ok(); - } - catch (OperationCanceledException) { return BadRequest(); } - finally - { - _parallelRequestSemaphore.Release(); - } - } + Guid g = Guid.NewGuid(); + await _requestQueue.EnqueueUser(new(g, MareUser, files.ToList())); - [HttpGet] - [Route(MareFiles.Request_RequestFile)] - public async Task RequestFile(string file) - { - Guid g = Guid.NewGuid(); - - try - { - await _parallelRequestSemaphore.WaitAsync(HttpContext.RequestAborted); - _cachedFileProvider.DownloadFileWhenRequired(file); return Ok(g); } catch (OperationCanceledException) { return BadRequest(); } finally { _parallelRequestSemaphore.Release(); - await _requestQueue.EnqueueUser(new(g, MareUser, file)); } } [HttpGet] [Route(MareFiles.Request_Check)] - public async Task CheckQueueAsync(Guid requestId, string file) + public async Task CheckQueueAsync(Guid requestId, [FromBody] IEnumerable files) { try { if (!_requestQueue.StillEnqueued(requestId, MareUser)) - await _requestQueue.EnqueueUser(new(requestId, MareUser, file)); + await _requestQueue.EnqueueUser(new(requestId, MareUser, files.ToList())); return Ok(); } catch (OperationCanceledException) { return BadRequest(); } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs index 15d027b..5330fb3 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs @@ -40,7 +40,7 @@ public class RequestQueueService : IHostedService public async Task EnqueueUser(UserRequest request) { - _logger.LogDebug("Enqueueing req {guid} from {user} for {file}", request.RequestId, request.User, request.FileId); + _logger.LogDebug("Enqueueing req {guid} from {user} for {file}", request.RequestId, request.User, string.Join(", ", request.FileIds)); if (_queueProcessingSemaphore.CurrentCount == 0) { @@ -123,7 +123,7 @@ public class RequestQueueService : IHostedService private async Task DequeueIntoSlotAsync(UserRequest userRequest, int slot) { - _logger.LogDebug("Dequeueing {req} into {i}: {user} with {file}", userRequest.RequestId, slot, userRequest.User, userRequest.FileId); + _logger.LogDebug("Dequeueing {req} into {i}: {user} with {file}", userRequest.RequestId, slot, userRequest.User, string.Join(", ", userRequest.FileIds)); _userQueueRequests[slot] = new(userRequest, DateTime.UtcNow.AddSeconds(_queueExpirationSeconds)); await _hubContext.Clients.User(userRequest.User).SendAsync(nameof(IMareHub.Client_DownloadReady), userRequest.RequestId).ConfigureAwait(false); } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs index a470263..388b648 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs @@ -21,7 +21,7 @@ public class RequestFileStreamResult : FileStreamResult _mareMetrics.IncGauge(MetricsAPI.GaugeCurrentDownloads); // forcefully release slot after secondsUntilRelease - Task.Run(async () => + _ = Task.Run(async () => { try { diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResultFactory.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResultFactory.cs index 4e19e59..8ce225c 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResultFactory.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResultFactory.cs @@ -17,9 +17,9 @@ public class RequestFileStreamResultFactory _configurationService = configurationService; } - public RequestFileStreamResult Create(Guid requestId, FileStream fs) + public RequestFileStreamResult Create(Guid requestId, MemoryStream ms) { return new RequestFileStreamResult(requestId, _configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueReleaseSeconds), 15), - _requestQueueService, _metrics, fs, "application/octet-stream"); + _requestQueueService, _metrics, ms, "application/octet-stream"); } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserRequest.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserRequest.cs index 8452985..4640c09 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserRequest.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserRequest.cs @@ -1,3 +1,3 @@ namespace MareSynchronosStaticFilesServer.Utils; -public record UserRequest(Guid RequestId, string User, string FileId); +public record UserRequest(Guid RequestId, string User, List FileIds);