Merge branch 'main' of https://github.com/Penumbra-Sync/server

MareSynchronosServer/MareSynchronosServer/Startup.cs

@@ -124,7 +124,7 @@ public class Startup
                 new RedisHost(){ Host = address, Port = port },
             },
             AllowAdmin = true,
-            ConnectTimeout = 3000,
+            ConnectTimeout = options.ConnectTimeout,
             Database = 0,
             Ssl = false,
             Password = options.Password,
@@ -135,7 +135,8 @@ public class Startup
                 UnreachableServerAction = ServerEnumerationStrategy.UnreachableServerActionOptions.Throw,
             },
             MaxValueLength = 1024,
-            PoolSize = 50,
+            PoolSize = mareConfig.GetValue(nameof(ServerConfiguration.RedisPool), 50),
+            SyncTimeout = options.SyncTimeout,
         };
 
         services.AddStackExchangeRedisExtensions<SystemTextJsonSerializer>(redisConfiguration);

MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs

@@ -5,6 +5,7 @@ namespace MareSynchronosShared.Utils;
 public class ServerConfiguration : MareConfigurationAuthBase
 {
     public string RedisConnectionString { get; set; } = string.Empty;
+    public int RedisPool { get; set; } = 50;
 
     [RemoteConfiguration]
     public Uri CdnFullUrl { get; set; } = null;
@@ -32,6 +33,7 @@ public class ServerConfiguration : MareConfigurationAuthBase
         sb.AppendLine($"{nameof(CdnShardConfiguration)} => {string.Join(", ", CdnShardConfiguration.Select(c => c.ToString()))}");
         sb.AppendLine($"{nameof(StaticFileServiceAddress)} => {StaticFileServiceAddress}");
         sb.AppendLine($"{nameof(RedisConnectionString)} => {RedisConnectionString}");
+        sb.AppendLine($"{nameof(RedisPool)} => {RedisPool}");
         sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");
         sb.AppendLine($"{nameof(MaxJoinedGroupsByUser)} => {MaxJoinedGroupsByUser}");
         sb.AppendLine($"{nameof(MaxGroupUserCount)} => {MaxGroupUserCount}");

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs

@@ -14,7 +14,7 @@ public class CacheController : ControllerBase
     private readonly RequestQueueService _requestQueue;
 
     public CacheController(ILogger<CacheController> logger, RequestFileStreamResultFactory requestFileStreamResultFactory,
-        CachedFileProvider cachedFileProvider, RequestQueueService requestQueue, ServerTokenGenerator generator) : base(logger, generator)
+        CachedFileProvider cachedFileProvider, RequestQueueService requestQueue) : base(logger)
     {
         _requestFileStreamResultFactory = requestFileStreamResultFactory;
         _cachedFileProvider = cachedFileProvider;
@@ -30,7 +30,7 @@ public class CacheController : ControllerBase
 
         _requestQueue.ActivateRequest(requestId);
 
-        var fs = await _cachedFileProvider.GetAndDownloadFileStream(request.FileId, Authorization);
+        var fs = await _cachedFileProvider.GetAndDownloadFileStream(request.FileId);
         if (fs == null)
         {
             _requestQueue.FinishRequest(requestId);

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs

@@ -6,14 +6,11 @@ namespace MareSynchronosStaticFilesServer.Controllers;
 public class ControllerBase : Controller
 {
     protected ILogger _logger;
-    private readonly ServerTokenGenerator _generator;
 
-    public ControllerBase(ILogger logger, ServerTokenGenerator generator)
+    public ControllerBase(ILogger logger)
     {
         _logger = logger;
-        _generator = generator;
     }
 
     protected string MareUser => HttpContext.User.Claims.First(f => string.Equals(f.Type, MareClaimTypes.Uid, StringComparison.Ordinal)).Value;
-    protected string Authorization => _generator.Token;
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs

@@ -10,10 +10,9 @@ public class RequestController : ControllerBase
 {
     private readonly CachedFileProvider _cachedFileProvider;
     private readonly RequestQueueService _requestQueue;
-    private static SemaphoreSlim _parallelRequestSemaphore = new(500);
+    private static readonly SemaphoreSlim _parallelRequestSemaphore = new(500);
 
-    public RequestController(ILogger<RequestController> logger, CachedFileProvider cachedFileProvider, RequestQueueService requestQueue,
+    public RequestController(ILogger<RequestController> logger, CachedFileProvider cachedFileProvider, RequestQueueService requestQueue) : base(logger)
-        ServerTokenGenerator generator) : base(logger, generator)
     {
         _cachedFileProvider = cachedFileProvider;
         _requestQueue = requestQueue;
@@ -46,7 +45,7 @@ public class RequestController : ControllerBase
             foreach (var file in files)
             {
                 _logger.LogDebug("Prerequested file: " + file);
-                _cachedFileProvider.DownloadFileWhenRequired(file, Authorization);
+                _cachedFileProvider.DownloadFileWhenRequired(file);
             }
 
             return Ok();
@@ -66,7 +65,7 @@ public class RequestController : ControllerBase
         {
             await _parallelRequestSemaphore.WaitAsync(HttpContext.RequestAborted);
             Guid g = Guid.NewGuid();
-            _cachedFileProvider.DownloadFileWhenRequired(file, Authorization);
+            _cachedFileProvider.DownloadFileWhenRequired(file);
             await _requestQueue.EnqueueUser(new(g, MareUser, file));
             return Ok(g);
         }
@@ -76,4 +75,22 @@ public class RequestController : ControllerBase
             _parallelRequestSemaphore.Release();
         }
     }
+
+    [HttpGet]
+    [Route(MareFiles.Request_Check)]
+    public async Task<IActionResult> CheckQueueAsync(Guid requestId, string file)
+    {
+        try
+        {
+            await _parallelRequestSemaphore.WaitAsync(HttpContext.RequestAborted);
+            if (!_requestQueue.StillEnqueued(requestId, MareUser))
+                await _requestQueue.EnqueueUser(new(requestId, MareUser, file));
+            return Ok();
+        }
+        catch (OperationCanceledException) { return BadRequest(); }
+        finally
+        {
+            _parallelRequestSemaphore.Release();
+        }
+    }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs

@@ -11,7 +11,7 @@ public class ServerFilesController : ControllerBase
 {
     private readonly CachedFileProvider _cachedFileProvider;
 
-    public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider, ServerTokenGenerator generator) : base(logger, generator)
+    public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider) : base(logger)
     {
         _cachedFileProvider = cachedFileProvider;
     }

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs

@@ -4,6 +4,7 @@ using MareSynchronosStaticFilesServer.Utils;
 using System.Collections.Concurrent;
 using System.Net.Http.Headers;
 using MareSynchronos.API;
+using MareSynchronosShared.Utils;
 
 namespace MareSynchronosStaticFilesServer.Services;
 
@@ -12,30 +13,32 @@ public class CachedFileProvider
     private readonly ILogger<CachedFileProvider> _logger;
     private readonly FileStatisticsService _fileStatisticsService;
     private readonly MareMetrics _metrics;
+    private readonly ServerTokenGenerator _generator;
     private readonly Uri _remoteCacheSourceUri;
     private readonly string _basePath;
     private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
     private readonly HttpClient _httpClient;
     private bool IsMainServer => _remoteCacheSourceUri == null;
 
-    public CachedFileProvider(IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics)
+    public CachedFileProvider(IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics, ServerTokenGenerator generator)
     {
         _logger = logger;
         _fileStatisticsService = fileStatisticsService;
         _metrics = metrics;
+        _generator = generator;
         _remoteCacheSourceUri = configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.RemoteCacheSourceUri), null);
         _basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
         _httpClient = new HttpClient();
     }
 
-    private async Task DownloadTask(string hash, string auth)
+    private async Task DownloadTask(string hash)
     {
         // download file from remote
         var downloadUrl = MareFiles.ServerFilesGetFullPath(_remoteCacheSourceUri, hash);
         _logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl);
 
         using var requestMessage = new HttpRequestMessage(HttpMethod.Get, downloadUrl);
-        requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", auth);
+        requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _generator.Token);
         var response = await _httpClient.SendAsync(requestMessage).ConfigureAwait(false);
 
         try
@@ -63,7 +66,7 @@ public class CachedFileProvider
         _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_basePath, hash).Length);
     }
 
-    public void DownloadFileWhenRequired(string hash, string auth)
+    public void DownloadFileWhenRequired(string hash)
     {
         var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
         if (fi == null && IsMainServer) return;
@@ -72,7 +75,7 @@ public class CachedFileProvider
         {
             _currentTransfers[hash] = Task.Run(async () =>
             {
-                await DownloadTask(hash, auth).ConfigureAwait(false);
+                await DownloadTask(hash).ConfigureAwait(false);
                 _currentTransfers.Remove(hash, out _);
             });
         }
@@ -88,9 +91,9 @@ public class CachedFileProvider
         return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read);
     }
 
-    public async Task<FileStream?> GetAndDownloadFileStream(string hash, string auth)
+    public async Task<FileStream?> GetAndDownloadFileStream(string hash)
     {
-        DownloadFileWhenRequired(hash, auth);
+        DownloadFileWhenRequired(hash);
 
         if (_currentTransfers.TryGetValue(hash, out var downloadTask))
         {