t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize dist file get, add nginx x-accel support

Browse Source
This commit is contained in:
Loporrit
2024-09-13 15:16:00 +00:00
parent fa6be0231f
commit 21c615cdb7
3 changed files with 40 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs
View File

@@ -31,6 +31,10 @@ public class StaticFilesServerConfiguration : MareConfigurationBase
public Uri CdnFullUrl { get; set; } = null;
[RemoteConfiguration]
public List<CdnShardConfiguration> CdnShardConfiguration { get; set; } = new();
public bool UseXAccelRedirect { get; set; } = false;
public string XAccelRedirectPrefix { get; set; } = "/_internal/mare-files/";
public override string ToString()
{
StringBuilder sb = new();
@@ -55,6 +59,7 @@ public class StaticFilesServerConfiguration : MareConfigurationBase
sb.AppendLine($"{nameof(DownloadQueueSize)} => {DownloadQueueSize}");
sb.AppendLine($"{nameof(DownloadQueueReleaseSeconds)} => {DownloadQueueReleaseSeconds}");
sb.AppendLine($"{nameof(CdnShardConfiguration)} => {string.Join(", ", CdnShardConfiguration)}");
sb.AppendLine($"{nameof(UseXAccelRedirect)} => {UseXAccelRedirect}");
return sb.ToString();
}
}

21
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/DistributionController.cs
View File

@@ -1,4 +1,5 @@
using MareSynchronos.API.Routes;
using MareSynchronosShared.Services;
using MareSynchronosStaticFilesServer.Services;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
@@ -9,10 +10,13 @@ namespace MareSynchronosStaticFilesServer.Controllers;
public class DistributionController : ControllerBase
{
private readonly CachedFileProvider _cachedFileProvider;
private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
public DistributionController(ILogger<DistributionController> logger, CachedFileProvider cachedFileProvider) : base(logger)
public DistributionController(ILogger<DistributionController> logger, CachedFileProvider cachedFileProvider,
IConfigurationService<StaticFilesServerConfiguration> configuration) : base(logger)
{
_cachedFileProvider = cachedFileProvider;
_configuration = configuration;
}
[HttpGet(MareFiles.Distribution_Get)]
@@ -21,10 +25,19 @@ public class DistributionController : ControllerBase
{
_logger.LogInformation($"GetFile:{MareUser}:{file}");
var fs = await _cachedFileProvider.GetAndDownloadFileStream(file);
if (fs == null) return NotFound();
var fi = await _cachedFileProvider.GetAndDownloadFile(file);
if (fi == null) return NotFound();
return File(fs, "application/octet-stream");
if (_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseXAccelRedirect), false))
{
var prefix = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.XAccelRedirectPrefix));
Response.Headers.Append("X-Accel-Redirect", Path.Combine(prefix, file));
return Ok();
}
else
{
return PhysicalFile(fi.FullName, "application/octet-stream");
}
}
[HttpPost("touch")]

33
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs
View File

@@ -162,6 +162,7 @@ public sealed class CachedFileProvider : IDisposable
{
_metrics.IncGauge(MetricsAPI.GaugeFilesDownloadingFromCache);
await DownloadTask(hash).ConfigureAwait(false);
TryCopyFromColdStorage(hash, FilePathUtil.GetFilePath(_hotStoragePath, hash));
}
catch (Exception ex)
{
@@ -177,20 +178,7 @@ public sealed class CachedFileProvider : IDisposable
_downloadSemaphore.Release();
}
public FileStream? GetLocalFileStream(string hash)
{
var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash);
if (fi == null) return null;
fi.LastAccessTimeUtc = DateTime.UtcNow;
_touchService.TouchColdHash(hash);
_fileStatisticsService.LogFile(hash, fi.Length);
return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read);
}
public async Task<FileStream?> GetAndDownloadFileStream(string hash)
public async Task<FileInfo?> GetAndDownloadFile(string hash)
{
await DownloadFileWhenRequired(hash).ConfigureAwait(false);
@@ -214,7 +202,22 @@ public sealed class CachedFileProvider : IDisposable
}
}
return GetLocalFileStream(hash);
var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash);
if (fi == null)
return null;
fi.LastAccessTimeUtc = DateTime.UtcNow;
_touchService.TouchColdHash(hash);
_fileStatisticsService.LogFile(hash, fi.Length);
return fi;
}
public async Task<FileStream?> GetAndDownloadFileStream(string hash)
{
var fi = await GetAndDownloadFile(hash).ConfigureAwait(false);
return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read);
}
public void TouchColdHash(string hash)