Single file transfer (#26)

* move to single file transfer for server

* fix cache get

---------

Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs

@@ -2,6 +2,8 @@
 using MareSynchronosStaticFilesServer.Services;
 using MareSynchronosStaticFilesServer.Utils;
 using Microsoft.AspNetCore.Mvc;
+using System.Globalization;
+using System.Text;
 
 namespace MareSynchronosStaticFilesServer.Controllers;
 
@@ -21,7 +23,7 @@ public class CacheController : ControllerBase
     }
 
     [HttpGet(MareFiles.Cache_Get)]
-    public async Task<IActionResult> GetFile(Guid requestId)
+    public async Task<IActionResult> GetFiles(Guid requestId)
     {
         _logger.LogDebug($"GetFile:{MareUser}:{requestId}");
 
@@ -29,13 +31,23 @@ public class CacheController : ControllerBase
 
         _requestQueue.ActivateRequest(requestId);
 
-        var fs = await _cachedFileProvider.GetAndDownloadFileStream(request.FileId);
+        Response.ContentType = "application/octet-stream";
-        if (fs == null)
+        var memoryStream = new MemoryStream();
+        var streamWriter = new BinaryWriter(memoryStream);
+
+        foreach (var file in request.FileIds)
         {
-            _requestQueue.FinishRequest(requestId);
+            var fs = await _cachedFileProvider.GetAndDownloadFileStream(file);
-            return NotFound();
+            if (fs == null) continue;
+            streamWriter.Write(Encoding.ASCII.GetBytes("#" + file + ":" + fs.Length.ToString(CultureInfo.InvariantCulture) + "#"));
+            byte[] buffer = new byte[fs.Length];
+            _ = await fs.ReadAsync(buffer, HttpContext.RequestAborted);
+            streamWriter.Write(buffer);
         }
 
-        return _requestFileStreamResultFactory.Create(requestId, fs);
+        streamWriter.Flush();
+        memoryStream.Position = 0;
+
+        return _requestFileStreamResultFactory.Create(requestId, memoryStream);
     }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs

@@ -36,7 +36,7 @@ public class RequestController : ControllerBase
 
     [HttpPost]
     [Route(MareFiles.Request_Enqueue)]
-    public async Task<IActionResult> PreRequestFilesAsync([FromBody] List<string> files)
+    public async Task<IActionResult> PreRequestFilesAsync([FromBody] IEnumerable<string> files)
     {
         try
         {
@@ -47,43 +47,26 @@ public class RequestController : ControllerBase
                 _cachedFileProvider.DownloadFileWhenRequired(file);
             }
 
-            return Ok();
-        }
-        catch (OperationCanceledException) { return BadRequest(); }
-        finally
-        {
-            _parallelRequestSemaphore.Release();
-        }
-    }
-
-    [HttpGet]
-    [Route(MareFiles.Request_RequestFile)]
-    public async Task<IActionResult> RequestFile(string file)
-    {
             Guid g = Guid.NewGuid();
+            await _requestQueue.EnqueueUser(new(g, MareUser, files.ToList()));
 
-        try
-        {
-            await _parallelRequestSemaphore.WaitAsync(HttpContext.RequestAborted);
-            _cachedFileProvider.DownloadFileWhenRequired(file);
             return Ok(g);
         }
         catch (OperationCanceledException) { return BadRequest(); }
         finally
         {
             _parallelRequestSemaphore.Release();
-            await _requestQueue.EnqueueUser(new(g, MareUser, file));
         }
     }
 
     [HttpGet]
     [Route(MareFiles.Request_Check)]
-    public async Task<IActionResult> CheckQueueAsync(Guid requestId, string file)
+    public async Task<IActionResult> CheckQueueAsync(Guid requestId, [FromBody] IEnumerable<string> files)
     {
         try
         {
             if (!_requestQueue.StillEnqueued(requestId, MareUser))
-                await _requestQueue.EnqueueUser(new(requestId, MareUser, file));
+                await _requestQueue.EnqueueUser(new(requestId, MareUser, files.ToList()));
             return Ok();
         }
         catch (OperationCanceledException) { return BadRequest(); }

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs

@@ -40,7 +40,7 @@ public class RequestQueueService : IHostedService
 
     public async Task EnqueueUser(UserRequest request)
     {
-        _logger.LogDebug("Enqueueing req {guid} from {user} for {file}", request.RequestId, request.User, request.FileId);
+        _logger.LogDebug("Enqueueing req {guid} from {user} for {file}", request.RequestId, request.User, string.Join(", ", request.FileIds));
 
         if (_queueProcessingSemaphore.CurrentCount == 0)
         {
@@ -123,7 +123,7 @@ public class RequestQueueService : IHostedService
 
     private async Task DequeueIntoSlotAsync(UserRequest userRequest, int slot)
     {
-        _logger.LogDebug("Dequeueing {req} into {i}: {user} with {file}", userRequest.RequestId, slot, userRequest.User, userRequest.FileId);
+        _logger.LogDebug("Dequeueing {req} into {i}: {user} with {file}", userRequest.RequestId, slot, userRequest.User, string.Join(", ", userRequest.FileIds));
         _userQueueRequests[slot] = new(userRequest, DateTime.UtcNow.AddSeconds(_queueExpirationSeconds));
         await _hubContext.Clients.User(userRequest.User).SendAsync(nameof(IMareHub.Client_DownloadReady), userRequest.RequestId).ConfigureAwait(false);
     }

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs

@@ -21,7 +21,7 @@ public class RequestFileStreamResult : FileStreamResult
         _mareMetrics.IncGauge(MetricsAPI.GaugeCurrentDownloads);
 
         // forcefully release slot after secondsUntilRelease
-        Task.Run(async () =>
+        _ = Task.Run(async () =>
         {
             try
             {

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResultFactory.cs

@@ -17,9 +17,9 @@ public class RequestFileStreamResultFactory
         _configurationService = configurationService;
     }
 
-    public RequestFileStreamResult Create(Guid requestId, FileStream fs)
+    public RequestFileStreamResult Create(Guid requestId, MemoryStream ms)
     {
         return new RequestFileStreamResult(requestId, _configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueReleaseSeconds), 15),
-            _requestQueueService, _metrics, fs, "application/octet-stream");
+            _requestQueueService, _metrics, ms, "application/octet-stream");
     }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserRequest.cs

@@ -1,3 +1,3 @@
 namespace MareSynchronosStaticFilesServer.Utils;
 
-public record UserRequest(Guid RequestId, string User, string FileId);
+public record UserRequest(Guid RequestId, string User, List<string> FileIds);