Use sendfile for client file get as well

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs

@@ -10,15 +10,15 @@ namespace MareSynchronosStaticFilesServer.Controllers;
 [Route(MareFiles.Cache)]
 public class CacheController : ControllerBase
 {
-    private readonly RequestFileStreamResultFactory _requestFileStreamResultFactory;
+    private readonly RequestBlockFileListResultFactory _requestBlockFileListResultFactory;
     private readonly CachedFileProvider _cachedFileProvider;
     private readonly RequestQueueService _requestQueue;
     private readonly FileStatisticsService _fileStatisticsService;
 
-    public CacheController(ILogger<CacheController> logger, RequestFileStreamResultFactory requestFileStreamResultFactory,
+    public CacheController(ILogger<CacheController> logger, RequestBlockFileListResultFactory requestBlockFileListResultFactory,
         CachedFileProvider cachedFileProvider, RequestQueueService requestQueue, FileStatisticsService fileStatisticsService) : base(logger)
     {
-        _requestFileStreamResultFactory = requestFileStreamResultFactory;
+        _requestBlockFileListResultFactory = requestBlockFileListResultFactory;
         _cachedFileProvider = cachedFileProvider;
         _requestQueue = requestQueue;
         _fileStatisticsService = fileStatisticsService;
@@ -33,23 +33,19 @@ public class CacheController : ControllerBase
 
         _requestQueue.ActivateRequest(requestId);
 
-        Response.ContentType = "application/octet-stream";
-
         long requestSize = 0;
-        var streamList = new List<Stream>();
+        var fileList = new List<FileInfo>(request.FileIds.Count);
 
         foreach (var file in request.FileIds)
         {
-            var fs = await _cachedFileProvider.GetAndDownloadFileStream(file);
-            if (fs == null) continue;
-            var headerBytes = Encoding.ASCII.GetBytes("#" + file + ":" + fs.Length.ToString(CultureInfo.InvariantCulture) + "#");
-            streamList.Add(new MemoryStream(headerBytes));
-            streamList.Add(fs);
-            requestSize += fs.Length;
+            var fi = await _cachedFileProvider.GetAndDownloadFile(file);
+            if (fi == null) continue;
+            requestSize += fi.Length;
+            fileList.Add(fi);
         }
 
         _fileStatisticsService.LogRequest(requestSize);
 
-        return _requestFileStreamResultFactory.Create(requestId, new ConcatenatedStreamReader(streamList));
+        return _requestBlockFileListResultFactory.Create(requestId, fileList);
     }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs

@@ -86,7 +86,7 @@ public class Startup
         services.AddSingleton<CachedFileProvider>();
         services.AddHostedService<FileCleanupService>();
         services.AddSingleton<FileStatisticsService>();
-        services.AddSingleton<RequestFileStreamResultFactory>();
+        services.AddSingleton<RequestBlockFileListResultFactory>();
         services.AddSingleton<ServerTokenGenerator>();
         services.AddSingleton<RequestQueueService>();
         services.AddHostedService(p => p.GetService<RequestQueueService>());

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/ConcatenatedStreamReader.cs

@@ -1,84 +0,0 @@
-namespace MareSynchronosStaticFilesServer.Utils;
-
-// Concatenates the content of multiple readable streams
-public class ConcatenatedStreamReader : Stream
-{
-    private IEnumerable<Stream> _streams;
-    private IEnumerator<Stream> _iter;
-    private bool _finished;
-    public bool DisposeUnderlying = true;
-
-    public ConcatenatedStreamReader(IEnumerable<Stream> streams)
-    {
-        _streams = streams;
-        _iter = streams.GetEnumerator();
-        _finished = !_iter.MoveNext();
-    }
-
-    protected override void Dispose(bool disposing)
-    {
-        if (!DisposeUnderlying)
-            return;
-        foreach (var stream in _streams)
-            stream.Dispose();
-    }
-
-    public override bool CanRead => true;
-    public override bool CanSeek => false;
-    public override bool CanWrite => false;
-    public override long Length => throw new NotSupportedException();
-
-    public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); }
-
-    public override void Flush()
-    {
-    }
-
-    public override int Read(byte[] buffer, int offset, int count)
-    {
-        int n = 0;
-
-        while (n == 0 && !_finished)
-        {
-            n = _iter.Current.Read(buffer, offset, count);
-
-            if (n == 0)
-                _finished = !_iter.MoveNext();
-        }
-
-        return n;
-    }
-
-    public async override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
-    {
-        int n = 0;
-
-        while (n == 0 && !_finished)
-        {
-            n = await _iter.Current.ReadAsync(buffer, offset, count, cancellationToken);
-
-            if (cancellationToken.IsCancellationRequested)
-                break;
-
-            if (n == 0)
-                _finished = !_iter.MoveNext();
-        }
-
-        return n;
-    }
-
-    public override long Seek(long offset, SeekOrigin origin)
-    {
-        throw new NotSupportedException();
-    }
-
-    public override void SetLength(long value)
-    {
-        throw new NotSupportedException();
-    }
-
-    public override void Write(byte[] buffer, int offset, int count)
-    {
-        throw new NotSupportedException();
-    }
-}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestBlockFileListResult.cs

@@ -0,0 +1,50 @@
+using MareSynchronosShared.Metrics;
+using MareSynchronosStaticFilesServer.Services;
+using Microsoft.AspNetCore.Mvc;
+using System.Globalization;
+using System.Text;
+
+namespace MareSynchronosStaticFilesServer.Utils;
+
+public class RequestBlockFileListResult : IActionResult
+{
+    private readonly Guid _requestId;
+    private readonly RequestQueueService _requestQueueService;
+    private readonly MareMetrics _mareMetrics;
+    private readonly IEnumerable<FileInfo> _fileList;
+
+    public RequestBlockFileListResult(Guid requestId, RequestQueueService requestQueueService, MareMetrics mareMetrics, IEnumerable<FileInfo> fileList)
+    {
+        _requestId = requestId;
+        _requestQueueService = requestQueueService;
+        _mareMetrics = mareMetrics;
+        _mareMetrics.IncGauge(MetricsAPI.GaugeCurrentDownloads);
+        _fileList = fileList;
+    }
+
+    public async Task ExecuteResultAsync(ActionContext context)
+    {
+        try
+        {
+            ArgumentNullException.ThrowIfNull(context);
+
+            context.HttpContext.Response.StatusCode = 200;
+            context.HttpContext.Response.ContentType = "application/octet-stream";
+
+            foreach (var file in _fileList)
+            {
+                await context.HttpContext.Response.WriteAsync("#" + file.Name + ":" + file.Length.ToString(CultureInfo.InvariantCulture) + "#", Encoding.ASCII);
+                await context.HttpContext.Response.SendFileAsync(file.FullName);
+            }
+        }
+        catch
+        {
+            throw;
+        }
+        finally
+        {
+            _requestQueueService.FinishRequest(_requestId);
+            _mareMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads);
+        }
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestBlockFileListResultFactory.cs

@@ -1,25 +1,24 @@
-using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Metrics;
 using MareSynchronosShared.Services;
 using MareSynchronosStaticFilesServer.Services;
 
 namespace MareSynchronosStaticFilesServer.Utils;
 
-public class RequestFileStreamResultFactory
+public class RequestBlockFileListResultFactory
 {
     private readonly MareMetrics _metrics;
     private readonly RequestQueueService _requestQueueService;
     private readonly IConfigurationService<StaticFilesServerConfiguration> _configurationService;
 
-    public RequestFileStreamResultFactory(MareMetrics metrics, RequestQueueService requestQueueService, IConfigurationService<StaticFilesServerConfiguration> configurationService)
+    public RequestBlockFileListResultFactory(MareMetrics metrics, RequestQueueService requestQueueService, IConfigurationService<StaticFilesServerConfiguration> configurationService)
     {
         _metrics = metrics;
         _requestQueueService = requestQueueService;
         _configurationService = configurationService;
     }
 
-    public RequestFileStreamResult Create(Guid requestId, Stream stream)
+    public RequestBlockFileListResult Create(Guid requestId, IEnumerable<FileInfo> fileList)
     {
-        return new RequestFileStreamResult(requestId, _requestQueueService,
-            _metrics, stream, "application/octet-stream");
+        return new RequestBlockFileListResult(requestId, _requestQueueService, _metrics, fileList);
     }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs

@@ -1,58 +0,0 @@
-using MareSynchronosShared.Metrics;
-using MareSynchronosStaticFilesServer.Services;
-using Microsoft.AspNetCore.Mvc;
-
-namespace MareSynchronosStaticFilesServer.Utils;
-
-public class RequestFileStreamResult : FileStreamResult
-{
-    private readonly Guid _requestId;
-    private readonly RequestQueueService _requestQueueService;
-    private readonly MareMetrics _mareMetrics;
-
-    public RequestFileStreamResult(Guid requestId, RequestQueueService requestQueueService, MareMetrics mareMetrics,
-        Stream fileStream, string contentType) : base(fileStream, contentType)
-    {
-        _requestId = requestId;
-        _requestQueueService = requestQueueService;
-        _mareMetrics = mareMetrics;
-        _mareMetrics.IncGauge(MetricsAPI.GaugeCurrentDownloads);
-    }
-
-    public override void ExecuteResult(ActionContext context)
-    {
-        try
-        {
-            base.ExecuteResult(context);
-        }
-        catch
-        {
-            throw;
-        }
-        finally
-        {
-            _requestQueueService.FinishRequest(_requestId);
-
-            _mareMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads);
-            FileStream?.Dispose();
-        }
-    }
-
-    public override async Task ExecuteResultAsync(ActionContext context)
-    {
-        try
-        {
-            await base.ExecuteResultAsync(context).ConfigureAwait(false);
-        }
-        catch
-        {
-            throw;
-        }
-        finally
-        {
-            _requestQueueService.FinishRequest(_requestId);
-            _mareMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads);
-            FileStream?.Dispose();
-        }
-    }
-}