From 49695b440354162ff34dfb9501e789df89aab8d0 Mon Sep 17 00:00:00 2001 From: Loporrit <141286461+loporrit@users.noreply.github.com> Date: Fri, 13 Sep 2024 16:11:03 +0000 Subject: [PATCH] Use sendfile for client file get as well --- .../Controllers/CacheController.cs | 22 ++--- .../Startup.cs | 2 +- .../Utils/ConcatenatedStreamReader.cs | 84 ------------------- .../Utils/RequestBlockFileListResult.cs | 50 +++++++++++ ...s => RequestBlockFileListResultFactory.cs} | 11 ++- .../Utils/RequestFileStreamResult.cs | 58 ------------- 6 files changed, 65 insertions(+), 162 deletions(-) delete mode 100644 MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/ConcatenatedStreamReader.cs create mode 100644 MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestBlockFileListResult.cs rename MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/{RequestFileStreamResultFactory.cs => RequestBlockFileListResultFactory.cs} (51%) delete mode 100644 MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs index dc3bb0d..34d5b6e 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs @@ -10,15 +10,15 @@ namespace MareSynchronosStaticFilesServer.Controllers; [Route(MareFiles.Cache)] public class CacheController : ControllerBase { - private readonly RequestFileStreamResultFactory _requestFileStreamResultFactory; + private readonly RequestBlockFileListResultFactory _requestBlockFileListResultFactory; private readonly CachedFileProvider _cachedFileProvider; private readonly RequestQueueService _requestQueue; private readonly FileStatisticsService _fileStatisticsService; - public CacheController(ILogger logger, RequestFileStreamResultFactory requestFileStreamResultFactory, + public CacheController(ILogger logger, RequestBlockFileListResultFactory requestBlockFileListResultFactory, CachedFileProvider cachedFileProvider, RequestQueueService requestQueue, FileStatisticsService fileStatisticsService) : base(logger) { - _requestFileStreamResultFactory = requestFileStreamResultFactory; + _requestBlockFileListResultFactory = requestBlockFileListResultFactory; _cachedFileProvider = cachedFileProvider; _requestQueue = requestQueue; _fileStatisticsService = fileStatisticsService; @@ -33,23 +33,19 @@ public class CacheController : ControllerBase _requestQueue.ActivateRequest(requestId); - Response.ContentType = "application/octet-stream"; - long requestSize = 0; - var streamList = new List(); + var fileList = new List(request.FileIds.Count); foreach (var file in request.FileIds) { - var fs = await _cachedFileProvider.GetAndDownloadFileStream(file); - if (fs == null) continue; - var headerBytes = Encoding.ASCII.GetBytes("#" + file + ":" + fs.Length.ToString(CultureInfo.InvariantCulture) + "#"); - streamList.Add(new MemoryStream(headerBytes)); - streamList.Add(fs); - requestSize += fs.Length; + var fi = await _cachedFileProvider.GetAndDownloadFile(file); + if (fi == null) continue; + requestSize += fi.Length; + fileList.Add(fi); } _fileStatisticsService.LogRequest(requestSize); - return _requestFileStreamResultFactory.Create(requestId, new ConcatenatedStreamReader(streamList)); + return _requestBlockFileListResultFactory.Create(requestId, fileList); } } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs index 2f85504..5520bdf 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs @@ -86,7 +86,7 @@ public class Startup services.AddSingleton(); services.AddHostedService(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddHostedService(p => p.GetService()); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/ConcatenatedStreamReader.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/ConcatenatedStreamReader.cs deleted file mode 100644 index 3402444..0000000 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/ConcatenatedStreamReader.cs +++ /dev/null @@ -1,84 +0,0 @@ -namespace MareSynchronosStaticFilesServer.Utils; - -// Concatenates the content of multiple readable streams -public class ConcatenatedStreamReader : Stream -{ - private IEnumerable _streams; - private IEnumerator _iter; - private bool _finished; - public bool DisposeUnderlying = true; - - public ConcatenatedStreamReader(IEnumerable streams) - { - _streams = streams; - _iter = streams.GetEnumerator(); - _finished = !_iter.MoveNext(); - } - - protected override void Dispose(bool disposing) - { - if (!DisposeUnderlying) - return; - foreach (var stream in _streams) - stream.Dispose(); - } - - public override bool CanRead => true; - public override bool CanSeek => false; - public override bool CanWrite => false; - public override long Length => throw new NotSupportedException(); - - public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } - - public override void Flush() - { - } - - public override int Read(byte[] buffer, int offset, int count) - { - int n = 0; - - while (n == 0 && !_finished) - { - n = _iter.Current.Read(buffer, offset, count); - - if (n == 0) - _finished = !_iter.MoveNext(); - } - - return n; - } - - public async override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - int n = 0; - - while (n == 0 && !_finished) - { - n = await _iter.Current.ReadAsync(buffer, offset, count, cancellationToken); - - if (cancellationToken.IsCancellationRequested) - break; - - if (n == 0) - _finished = !_iter.MoveNext(); - } - - return n; - } - - public override long Seek(long offset, SeekOrigin origin) - { - throw new NotSupportedException(); - } - - public override void SetLength(long value) - { - throw new NotSupportedException(); - } - - public override void Write(byte[] buffer, int offset, int count) - { - throw new NotSupportedException(); - } -} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestBlockFileListResult.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestBlockFileListResult.cs new file mode 100644 index 0000000..61e06dd --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestBlockFileListResult.cs @@ -0,0 +1,50 @@ +using MareSynchronosShared.Metrics; +using MareSynchronosStaticFilesServer.Services; +using Microsoft.AspNetCore.Mvc; +using System.Globalization; +using System.Text; + +namespace MareSynchronosStaticFilesServer.Utils; + +public class RequestBlockFileListResult : IActionResult +{ + private readonly Guid _requestId; + private readonly RequestQueueService _requestQueueService; + private readonly MareMetrics _mareMetrics; + private readonly IEnumerable _fileList; + + public RequestBlockFileListResult(Guid requestId, RequestQueueService requestQueueService, MareMetrics mareMetrics, IEnumerable fileList) + { + _requestId = requestId; + _requestQueueService = requestQueueService; + _mareMetrics = mareMetrics; + _mareMetrics.IncGauge(MetricsAPI.GaugeCurrentDownloads); + _fileList = fileList; + } + + public async Task ExecuteResultAsync(ActionContext context) + { + try + { + ArgumentNullException.ThrowIfNull(context); + + context.HttpContext.Response.StatusCode = 200; + context.HttpContext.Response.ContentType = "application/octet-stream"; + + foreach (var file in _fileList) + { + await context.HttpContext.Response.WriteAsync("#" + file.Name + ":" + file.Length.ToString(CultureInfo.InvariantCulture) + "#", Encoding.ASCII); + await context.HttpContext.Response.SendFileAsync(file.FullName); + } + } + catch + { + throw; + } + finally + { + _requestQueueService.FinishRequest(_requestId); + _mareMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads); + } + } +} \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResultFactory.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestBlockFileListResultFactory.cs similarity index 51% rename from MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResultFactory.cs rename to MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestBlockFileListResultFactory.cs index 43a952d..7779331 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResultFactory.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestBlockFileListResultFactory.cs @@ -1,25 +1,24 @@ -using MareSynchronosShared.Metrics; +using MareSynchronosShared.Metrics; using MareSynchronosShared.Services; using MareSynchronosStaticFilesServer.Services; namespace MareSynchronosStaticFilesServer.Utils; -public class RequestFileStreamResultFactory +public class RequestBlockFileListResultFactory { private readonly MareMetrics _metrics; private readonly RequestQueueService _requestQueueService; private readonly IConfigurationService _configurationService; - public RequestFileStreamResultFactory(MareMetrics metrics, RequestQueueService requestQueueService, IConfigurationService configurationService) + public RequestBlockFileListResultFactory(MareMetrics metrics, RequestQueueService requestQueueService, IConfigurationService configurationService) { _metrics = metrics; _requestQueueService = requestQueueService; _configurationService = configurationService; } - public RequestFileStreamResult Create(Guid requestId, Stream stream) + public RequestBlockFileListResult Create(Guid requestId, IEnumerable fileList) { - return new RequestFileStreamResult(requestId, _requestQueueService, - _metrics, stream, "application/octet-stream"); + return new RequestBlockFileListResult(requestId, _requestQueueService, _metrics, fileList); } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs deleted file mode 100644 index 31e7b0a..0000000 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs +++ /dev/null @@ -1,58 +0,0 @@ -using MareSynchronosShared.Metrics; -using MareSynchronosStaticFilesServer.Services; -using Microsoft.AspNetCore.Mvc; - -namespace MareSynchronosStaticFilesServer.Utils; - -public class RequestFileStreamResult : FileStreamResult -{ - private readonly Guid _requestId; - private readonly RequestQueueService _requestQueueService; - private readonly MareMetrics _mareMetrics; - - public RequestFileStreamResult(Guid requestId, RequestQueueService requestQueueService, MareMetrics mareMetrics, - Stream fileStream, string contentType) : base(fileStream, contentType) - { - _requestId = requestId; - _requestQueueService = requestQueueService; - _mareMetrics = mareMetrics; - _mareMetrics.IncGauge(MetricsAPI.GaugeCurrentDownloads); - } - - public override void ExecuteResult(ActionContext context) - { - try - { - base.ExecuteResult(context); - } - catch - { - throw; - } - finally - { - _requestQueueService.FinishRequest(_requestId); - - _mareMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads); - FileStream?.Dispose(); - } - } - - public override async Task ExecuteResultAsync(ActionContext context) - { - try - { - await base.ExecuteResultAsync(context).ConfigureAwait(false); - } - catch - { - throw; - } - finally - { - _requestQueueService.FinishRequest(_requestId); - _mareMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads); - FileStream?.Dispose(); - } - } -} \ No newline at end of file