From 21c615cdb7b03abe3b4a04b5b78c39c9f577add6 Mon Sep 17 00:00:00 2001 From: Loporrit <141286461+loporrit@users.noreply.github.com> Date: Fri, 13 Sep 2024 15:16:00 +0000 Subject: [PATCH] Optimize dist file get, add nginx x-accel support --- .../Utils/StaticFilesServerConfiguration.cs | 5 +++ .../Controllers/DistributionController.cs | 21 +++++++++--- .../Services/CachedFileProvider.cs | 33 ++++++++++--------- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs b/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs index 169889f..eb053e2 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs @@ -31,6 +31,10 @@ public class StaticFilesServerConfiguration : MareConfigurationBase public Uri CdnFullUrl { get; set; } = null; [RemoteConfiguration] public List CdnShardConfiguration { get; set; } = new(); + + public bool UseXAccelRedirect { get; set; } = false; + public string XAccelRedirectPrefix { get; set; } = "/_internal/mare-files/"; + public override string ToString() { StringBuilder sb = new(); @@ -55,6 +59,7 @@ public class StaticFilesServerConfiguration : MareConfigurationBase sb.AppendLine($"{nameof(DownloadQueueSize)} => {DownloadQueueSize}"); sb.AppendLine($"{nameof(DownloadQueueReleaseSeconds)} => {DownloadQueueReleaseSeconds}"); sb.AppendLine($"{nameof(CdnShardConfiguration)} => {string.Join(", ", CdnShardConfiguration)}"); + sb.AppendLine($"{nameof(UseXAccelRedirect)} => {UseXAccelRedirect}"); return sb.ToString(); } } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/DistributionController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/DistributionController.cs index dbaaa8f..88cef12 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/DistributionController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/DistributionController.cs @@ -1,4 +1,5 @@ using MareSynchronos.API.Routes; +using MareSynchronosShared.Services; using MareSynchronosStaticFilesServer.Services; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; @@ -9,10 +10,13 @@ namespace MareSynchronosStaticFilesServer.Controllers; public class DistributionController : ControllerBase { private readonly CachedFileProvider _cachedFileProvider; + private readonly IConfigurationService _configuration; - public DistributionController(ILogger logger, CachedFileProvider cachedFileProvider) : base(logger) + public DistributionController(ILogger logger, CachedFileProvider cachedFileProvider, + IConfigurationService configuration) : base(logger) { _cachedFileProvider = cachedFileProvider; + _configuration = configuration; } [HttpGet(MareFiles.Distribution_Get)] @@ -21,10 +25,19 @@ public class DistributionController : ControllerBase { _logger.LogInformation($"GetFile:{MareUser}:{file}"); - var fs = await _cachedFileProvider.GetAndDownloadFileStream(file); - if (fs == null) return NotFound(); + var fi = await _cachedFileProvider.GetAndDownloadFile(file); + if (fi == null) return NotFound(); - return File(fs, "application/octet-stream"); + if (_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseXAccelRedirect), false)) + { + var prefix = _configuration.GetValue(nameof(StaticFilesServerConfiguration.XAccelRedirectPrefix)); + Response.Headers.Append("X-Accel-Redirect", Path.Combine(prefix, file)); + return Ok(); + } + else + { + return PhysicalFile(fi.FullName, "application/octet-stream"); + } } [HttpPost("touch")] diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs index 0139a4e..1c8d897 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs @@ -162,6 +162,7 @@ public sealed class CachedFileProvider : IDisposable { _metrics.IncGauge(MetricsAPI.GaugeFilesDownloadingFromCache); await DownloadTask(hash).ConfigureAwait(false); + TryCopyFromColdStorage(hash, FilePathUtil.GetFilePath(_hotStoragePath, hash)); } catch (Exception ex) { @@ -177,20 +178,7 @@ public sealed class CachedFileProvider : IDisposable _downloadSemaphore.Release(); } - public FileStream? GetLocalFileStream(string hash) - { - var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash); - if (fi == null) return null; - fi.LastAccessTimeUtc = DateTime.UtcNow; - - _touchService.TouchColdHash(hash); - - _fileStatisticsService.LogFile(hash, fi.Length); - - return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read); - } - - public async Task GetAndDownloadFileStream(string hash) + public async Task GetAndDownloadFile(string hash) { await DownloadFileWhenRequired(hash).ConfigureAwait(false); @@ -214,7 +202,22 @@ public sealed class CachedFileProvider : IDisposable } } - return GetLocalFileStream(hash); + var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash); + if (fi == null) + return null; + + fi.LastAccessTimeUtc = DateTime.UtcNow; + _touchService.TouchColdHash(hash); + + _fileStatisticsService.LogFile(hash, fi.Length); + + return fi; + } + + public async Task GetAndDownloadFileStream(string hash) + { + var fi = await GetAndDownloadFile(hash).ConfigureAwait(false); + return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read); } public void TouchColdHash(string hash)