t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/Penumbra-Sync/server

Browse Source
This commit is contained in:
rootdarkarchon
2023-01-18 19:46:36 +01:00
parent fe80faab6b 35f61d5387
commit d7e664477f
8 changed files with 43 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
MareAPI

Submodule MareAPI updated: 5ac8a753dd...9fea4c30da

5
MareSynchronosServer/MareSynchronosServer/Startup.cs
View File

@@ -124,7 +124,7 @@ public class Startup
new RedisHost(){ Host = address, Port = port },
},
AllowAdmin = true,
ConnectTimeout = 3000,
ConnectTimeout = options.ConnectTimeout,
Database = 0,
Ssl = false,
Password = options.Password,
@@ -135,7 +135,8 @@ public class Startup
UnreachableServerAction = ServerEnumerationStrategy.UnreachableServerActionOptions.Throw,
},
MaxValueLength = 1024,
PoolSize = 50,
PoolSize = mareConfig.GetValue(nameof(ServerConfiguration.RedisPool), 50),
SyncTimeout = options.SyncTimeout,
};
services.AddStackExchangeRedisExtensions<SystemTextJsonSerializer>(redisConfiguration);

2
MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs
View File

@@ -5,6 +5,7 @@ namespace MareSynchronosShared.Utils;
public class ServerConfiguration : MareConfigurationAuthBase
{
public string RedisConnectionString { get; set; } = string.Empty;
public int RedisPool { get; set; } = 50;
[RemoteConfiguration]
public Uri CdnFullUrl { get; set; } = null;
@@ -32,6 +33,7 @@ public class ServerConfiguration : MareConfigurationAuthBase
sb.AppendLine($"{nameof(CdnShardConfiguration)} => {string.Join(", ", CdnShardConfiguration.Select(c => c.ToString()))}");
sb.AppendLine($"{nameof(StaticFileServiceAddress)} => {StaticFileServiceAddress}");
sb.AppendLine($"{nameof(RedisConnectionString)} => {RedisConnectionString}");
sb.AppendLine($"{nameof(RedisPool)} => {RedisPool}");
sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");
sb.AppendLine($"{nameof(MaxJoinedGroupsByUser)} => {MaxJoinedGroupsByUser}");
sb.AppendLine($"{nameof(MaxGroupUserCount)} => {MaxGroupUserCount}");

4
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs
View File

@@ -14,7 +14,7 @@ public class CacheController : ControllerBase
private readonly RequestQueueService _requestQueue;
public CacheController(ILogger<CacheController> logger, RequestFileStreamResultFactory requestFileStreamResultFactory,
CachedFileProvider cachedFileProvider, RequestQueueService requestQueue, ServerTokenGenerator generator) : base(logger, generator)
CachedFileProvider cachedFileProvider, RequestQueueService requestQueue) : base(logger)
{
_requestFileStreamResultFactory = requestFileStreamResultFactory;
_cachedFileProvider = cachedFileProvider;
@@ -30,7 +30,7 @@ public class CacheController : ControllerBase
_requestQueue.ActivateRequest(requestId);
var fs = await _cachedFileProvider.GetAndDownloadFileStream(request.FileId, Authorization);
var fs = await _cachedFileProvider.GetAndDownloadFileStream(request.FileId);
if (fs == null)
{
_requestQueue.FinishRequest(requestId);

5
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs
View File

@@ -6,14 +6,11 @@ namespace MareSynchronosStaticFilesServer.Controllers;
public class ControllerBase : Controller
{
protected ILogger _logger;
private readonly ServerTokenGenerator _generator;
public ControllerBase(ILogger logger, ServerTokenGenerator generator)
public ControllerBase(ILogger logger)
{
_logger = logger;
_generator = generator;
}
protected string MareUser => HttpContext.User.Claims.First(f => string.Equals(f.Type, MareClaimTypes.Uid, StringComparison.Ordinal)).Value;
protected string Authorization => _generator.Token;
}

27
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs
View File

@@ -10,10 +10,9 @@ public class RequestController : ControllerBase
{
private readonly CachedFileProvider _cachedFileProvider;
private readonly RequestQueueService _requestQueue;
private static SemaphoreSlim _parallelRequestSemaphore = new(500);
private static readonly SemaphoreSlim _parallelRequestSemaphore = new(500);
public RequestController(ILogger<RequestController> logger, CachedFileProvider cachedFileProvider, RequestQueueService requestQueue,
ServerTokenGenerator generator) : base(logger, generator)
public RequestController(ILogger<RequestController> logger, CachedFileProvider cachedFileProvider, RequestQueueService requestQueue) : base(logger)
{
_cachedFileProvider = cachedFileProvider;
_requestQueue = requestQueue;
@@ -46,7 +45,7 @@ public class RequestController : ControllerBase
foreach (var file in files)
{
_logger.LogDebug("Prerequested file: " + file);
_cachedFileProvider.DownloadFileWhenRequired(file, Authorization);
_cachedFileProvider.DownloadFileWhenRequired(file);
}
return Ok();
@@ -66,7 +65,7 @@ public class RequestController : ControllerBase
{
await _parallelRequestSemaphore.WaitAsync(HttpContext.RequestAborted);
Guid g = Guid.NewGuid();
_cachedFileProvider.DownloadFileWhenRequired(file, Authorization);
_cachedFileProvider.DownloadFileWhenRequired(file);
await _requestQueue.EnqueueUser(new(g, MareUser, file));
return Ok(g);
}
@@ -76,4 +75,22 @@ public class RequestController : ControllerBase
_parallelRequestSemaphore.Release();
}
}
[HttpGet]
[Route(MareFiles.Request_Check)]
public async Task<IActionResult> CheckQueueAsync(Guid requestId, string file)
{
try
{
await _parallelRequestSemaphore.WaitAsync(HttpContext.RequestAborted);
if (!_requestQueue.StillEnqueued(requestId, MareUser))
await _requestQueue.EnqueueUser(new(requestId, MareUser, file));
return Ok();
}
catch (OperationCanceledException) { return BadRequest(); }
finally
{
_parallelRequestSemaphore.Release();
}
}
}

2
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs
View File

@@ -11,7 +11,7 @@ public class ServerFilesController : ControllerBase
{
private readonly CachedFileProvider _cachedFileProvider;
public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider, ServerTokenGenerator generator) : base(logger, generator)
public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider) : base(logger)
{
_cachedFileProvider = cachedFileProvider;
}

17
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs
View File

@@ -4,6 +4,7 @@ using MareSynchronosStaticFilesServer.Utils;
using System.Collections.Concurrent;
using System.Net.Http.Headers;
using MareSynchronos.API;
using MareSynchronosShared.Utils;
namespace MareSynchronosStaticFilesServer.Services;
@@ -12,30 +13,32 @@ public class CachedFileProvider
private readonly ILogger<CachedFileProvider> _logger;
private readonly FileStatisticsService _fileStatisticsService;
private readonly MareMetrics _metrics;
private readonly ServerTokenGenerator _generator;
private readonly Uri _remoteCacheSourceUri;
private readonly string _basePath;
private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
private readonly HttpClient _httpClient;
private bool IsMainServer => _remoteCacheSourceUri == null;
public CachedFileProvider(IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics)
public CachedFileProvider(IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics, ServerTokenGenerator generator)
{
_logger = logger;
_fileStatisticsService = fileStatisticsService;
_metrics = metrics;
_generator = generator;
_remoteCacheSourceUri = configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.RemoteCacheSourceUri), null);
_basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
_httpClient = new HttpClient();
}
private async Task DownloadTask(string hash, string auth)
private async Task DownloadTask(string hash)
{
// download file from remote
var downloadUrl = MareFiles.ServerFilesGetFullPath(_remoteCacheSourceUri, hash);
_logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl);
using var requestMessage = new HttpRequestMessage(HttpMethod.Get, downloadUrl);
requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", auth);
requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _generator.Token);
var response = await _httpClient.SendAsync(requestMessage).ConfigureAwait(false);
try
@@ -63,7 +66,7 @@ public class CachedFileProvider
_metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_basePath, hash).Length);
}
public void DownloadFileWhenRequired(string hash, string auth)
public void DownloadFileWhenRequired(string hash)
{
var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
if (fi == null && IsMainServer) return;
@@ -72,7 +75,7 @@ public class CachedFileProvider
{
_currentTransfers[hash] = Task.Run(async () =>
{
await DownloadTask(hash, auth).ConfigureAwait(false);
await DownloadTask(hash).ConfigureAwait(false);
_currentTransfers.Remove(hash, out _);
});
}
@@ -88,9 +91,9 @@ public class CachedFileProvider
return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read);
}
public async Task<FileStream?> GetAndDownloadFileStream(string hash, string auth)
public async Task<FileStream?> GetAndDownloadFileStream(string hash)
{
DownloadFileWhenRequired(hash, auth);
DownloadFileWhenRequired(hash);
if (_currentTransfers.TryGetValue(hash, out var downloadTask))
{