Add Server-Side Download Queue (#21)

* test add queueing to file service

* further adjustments to download queueing

* add check for whether the request is still in the queue to CheckQueue

* forcefully release slot if download didn't finish in 15s

* actually cancel the delay task

* add metrics and refactor some of the request queue service

* refactor pathing

* reuse httpclient

* add queue request dto to requestfile, enqueue users immediately if a slot is available

* change startup to include all controllers

* update server pathing

* update pathing, again

* several adjustments to auth, banning, jwt server tokens, renaming, authorization

* update api I guess

* adjust automated banning of charaident and reg

* generate jwt on servers for internal authentication

* remove mvcextensions

Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>

Docker/run/config/sharded/files-shard-1.json

@@ -7,7 +7,7 @@
             "Default": "Warning",
             "Microsoft": "Warning",
             "Microsoft.Hosting.Lifetime": "Information",
-            "MareSynchronosStaticFilesServer": "Information",
+            "MareSynchronosStaticFilesServer": "Debug",
             "MareSynchronosShared": "Information",
             "System.IO": "Information"
         },
@@ -33,8 +33,11 @@
         "CacheSizeHardLimitInGiB": 5,
         "UnusedFileRetentionPeriodInDays": 14,
         "CacheDirectory": "/marecache/",
-        "RemoteCacheSourceUri": "http://mare-files:6200/cache/",
+        "RemoteCacheSourceUri": "http://mare-files:6200/",
-        "MainServerGrpcAddress": "http://mare-server:6005"
+        "MainServerGrpcAddress": "http://mare-server:6005",
+        "DownloadTimeoutSeconds": 30,
+        "DownloadQueueSize": 50,
+        "DownloadQueueReleaseSeconds": 15
     },
     "AllowedHosts": "*",
     "Kestrel": {

Docker/run/config/sharded/files-shard-2.json

@@ -7,7 +7,7 @@
             "Default": "Warning",
             "Microsoft": "Warning",
             "Microsoft.Hosting.Lifetime": "Information",
-            "MareSynchronosStaticFilesServer": "Information",
+            "MareSynchronosStaticFilesServer": "Debug",
             "MareSynchronosShared": "Information",
             "System.IO": "Information"
         },
@@ -33,8 +33,11 @@
         "CacheSizeHardLimitInGiB": 5,
         "UnusedFileRetentionPeriodInDays": 14,
         "CacheDirectory": "/marecache/",
-        "RemoteCacheSourceUri": "http://mare-files:6200/cache/",
+        "RemoteCacheSourceUri": "http://mare-files:6200/",
-        "MainServerGrpcAddress": "http://mare-server:6005"
+        "MainServerGrpcAddress": "http://mare-server:6005",
+        "DownloadTimeoutSeconds": 30,
+        "DownloadQueueSize": 50,
+        "DownloadQueueReleaseSeconds": 15
     },
     "AllowedHosts": "*",
     "Kestrel": {

Docker/run/config/sharded/server-shard-main.json

@@ -36,7 +36,7 @@
             ""
         ],
         "RedisConnectionString": "redis,password=secretredispassword",
-        "CdnFullUrl": "http://localhost:6200/cache/",
+        "CdnFullUrl": "http://localhost:6200/",
         "StaticFileServiceAddress": "http://mare-files:6205",
         "MaxExistingGroupsByUser": 3,
         "MaxJoinedGroupsByUser": 6,

Docker/run/config/standalone/server-standalone.json

@@ -36,7 +36,7 @@
             ""
         ],
         "RedisConnectionString": "redis,password=secretredispassword",
-        "CdnFullUrl": "http://localhost:6200/cache/",
+        "CdnFullUrl": "http://localhost:6200/",
         "StaticFileServiceAddress": "http://mare-files:6205",
         "MaxExistingGroupsByUser": 3,
         "MaxJoinedGroupsByUser": 6,

MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthReply.cs

@@ -1,3 +1,3 @@
 namespace MareSynchronosServer.Authentication;
 
-public record SecretKeyAuthReply(bool Success, string Uid, bool TempBan);
+public record SecretKeyAuthReply(bool Success, string Uid, bool TempBan, bool Permaban);

MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthenticatorService.cs

@@ -50,14 +50,14 @@ public class SecretKeyAuthenticatorService
                     _failedAuthorizations.Remove(ip, out _);
                 });
             }
-            return new(Success: false, Uid: null, TempBan: true);
+            return new(Success: false, Uid: null, TempBan: true, Permaban: false);
         }
 
         using var scope = _serviceScopeFactory.CreateScope();
         using var context = scope.ServiceProvider.GetService<MareDbContext>();
         var authReply = await context.Auth.AsNoTracking().SingleOrDefaultAsync(u => u.HashedKey == hashedSecretKey).ConfigureAwait(false);
 
-        SecretKeyAuthReply reply = new(authReply != null, authReply?.UserUID, false);
+        SecretKeyAuthReply reply = new(authReply != null, authReply?.UserUID, false, authReply.IsBanned);
 
         if (reply.Success)
         {
@@ -97,6 +97,6 @@ public class SecretKeyAuthenticatorService
             }
         }
 
-        return new(Success: false, Uid: null, TempBan: false);
+        return new(Success: false, Uid: null, TempBan: false, Permaban: false);
     }
 }

MareSynchronosServer/MareSynchronosServer/Controllers/JwtController.cs

@@ -2,6 +2,7 @@
 using MareSynchronosServer.Authentication;
 using MareSynchronosShared;
 using MareSynchronosShared.Data;
+using MareSynchronosShared.Models;
 using MareSynchronosShared.Services;
 using MareSynchronosShared.Utils;
 using Microsoft.AspNetCore.Authorization;
@@ -38,7 +39,7 @@ public class JwtController : Controller
     }
 
     [AllowAnonymous]
-    [HttpPost(MareAuth.AuthCreateIdent)]
+    [HttpPost(MareAuth.Auth_CreateIdent)]
     public async Task<IActionResult> CreateToken(string auth, string charaIdent)
     {
         if (string.IsNullOrEmpty(auth)) return BadRequest("No Authkey");
@@ -52,10 +53,47 @@ public class JwtController : Controller
         var authResult = await _secretKeyAuthenticatorService.AuthorizeAsync(ip, auth);
 
         if (!authResult.Success && !authResult.TempBan) return Unauthorized("The provided secret key is invalid. Verify your accounts existence and/or recover the secret key.");
-        if (!authResult.Success && authResult.TempBan) return Unauthorized("You are temporarily banned. Try connecting again later.");
+        if (!authResult.Success && authResult.TempBan) return Unauthorized("You are temporarily banned. Try connecting again in 5 minutes.");
+        if (authResult.Permaban)
+        {
+            if (!_mareDbContext.BannedUsers.Any(c => c.CharacterIdentification == charaIdent))
+            {
+                _mareDbContext.BannedUsers.Add(new Banned()
+                {
+                    CharacterIdentification = charaIdent,
+                    Reason = "Autobanned CharacterIdent (" + authResult.Uid + ")"
+                });
+
+                await _mareDbContext.SaveChangesAsync();
+            }
+
+            var lodestone = await _mareDbContext.LodeStoneAuth.Include(a => a.User).FirstOrDefaultAsync(c => c.User.UID == authResult.Uid);
+
+            if (lodestone != null)
+            {
+                if (!_mareDbContext.BannedRegistrations.Any(c => c.DiscordIdOrLodestoneAuth == lodestone.HashedLodestoneId))
+                {
+                    _mareDbContext.BannedRegistrations.Add(new BannedRegistrations()
+                    {
+                        DiscordIdOrLodestoneAuth = lodestone.HashedLodestoneId
+                    });
+                }
+                if (!_mareDbContext.BannedRegistrations.Any(c => c.DiscordIdOrLodestoneAuth == lodestone.DiscordId.ToString()))
+                {
+                    _mareDbContext.BannedRegistrations.Add(new BannedRegistrations()
+                    {
+                        DiscordIdOrLodestoneAuth = lodestone.DiscordId.ToString()
+                    });
+                }
+
+                await _mareDbContext.SaveChangesAsync();
+            }
+
+            return Unauthorized("You are permanently banned.");
+        }
 
         var existingIdent = await _redis.GetAsync<string>("UID:" + authResult.Uid);
-        if (!string.IsNullOrEmpty(existingIdent)) return Unauthorized("Already logged in to this account.");
+        if (!string.IsNullOrEmpty(existingIdent)) return Unauthorized("Already logged in to this account. Reconnect in 60 seconds. If you keep seeing this issue, restart your game.");
 
         var token = CreateToken(new List<Claim>()
         {
@@ -66,26 +104,6 @@ public class JwtController : Controller
         return Content(token.RawData);
     }
 
-    [AllowAnonymous]
-    [HttpPost(MareAuth.AuthCreate)]
-    public async Task<IActionResult> CreateToken(string auth)
-    {
-        if (string.IsNullOrEmpty(auth)) return BadRequest("No Authkey");
-
-        var ip = _accessor.GetIpAddress();
-
-        var authResult = await _secretKeyAuthenticatorService.AuthorizeAsync(ip, auth);
-
-        if (!authResult.Success) return Unauthorized("Invalid Authkey");
-
-        var token = CreateToken(new List<Claim>()
-        {
-            new Claim(MareClaimTypes.Uid, authResult.Uid)
-        });
-
-        return Content(token.RawData);
-    }
-
     private JwtSecurityToken CreateToken(IEnumerable<Claim> authClaims)
     {
         var authSigningKey = new SymmetricSecurityKey(Encoding.ASCII.GetBytes(_configuration.GetValue<string>(nameof(MareConfigurationAuthBase.Jwt))));

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs

@@ -36,7 +36,7 @@ public partial class MareHub
         request.Hash.AddRange(ownFiles.Select(f => f.Hash));
         Metadata headers = new Metadata()
             {
-                { "Authorization", Context.GetHttpContext().Request.Headers["Authorization"].ToString() }
+                { "Authorization", "Bearer " + _generator.Token }
             };
         _ = await _fileServiceClient.DeleteFilesAsync(request, headers).ConfigureAwait(false);
     }
@@ -69,7 +69,7 @@ public partial class MareHub
                 IsForbidden = forbiddenFile != null,
                 Hash = file.Hash,
                 Size = file.Size,
-                Url = new Uri(baseUrl, file.Hash.ToUpperInvariant()).ToString()
+                Url = baseUrl.ToString()
             });
         }
 
@@ -210,7 +210,7 @@ public partial class MareHub
 
             Metadata headers = new Metadata()
             {
-                { "Authorization", Context.GetHttpContext().Request.Headers["Authorization"].ToString() }
+                { "Authorization", "Bearer " + _generator.Token }
             };
             var streamingCall = _fileServiceClient.UploadFile(headers);
             using var tempFileStream = new FileStream(tempFileName, FileMode.Open, FileAccess.Read);

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs

@@ -29,11 +29,12 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
     private readonly int _maxGroupUserCount;
     private readonly IConfigurationService<ServerConfiguration> _configurationService;
     private readonly IRedisDatabase _redis;
+    private readonly ServerTokenGenerator _generator;
 
     public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient,
         MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService,
         IConfigurationService<ServerConfiguration> configuration, IHttpContextAccessor contextAccessor,
-        IRedisDatabase redisDb)
+        IRedisDatabase redisDb, ServerTokenGenerator generator)
     {
         _mareMetrics = mareMetrics;
         _fileServiceClient = fileServiceClient;
@@ -46,6 +47,7 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
         _maxGroupUserCount = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxGroupUserCount), 100);
         _contextAccessor = contextAccessor;
         _redis = redisDb;
+        _generator = generator;
         _logger = new MareHubLogger(this, logger);
         _dbContext = mareDbContext;
     }

MareSynchronosServer/MareSynchronosServer/Startup.cs

@@ -78,6 +78,7 @@ public class Startup
         services.Configure<MareConfigurationBase>(mareConfig);
         services.Configure<MareConfigurationAuthBase>(mareConfig);
 
+        services.AddSingleton<ServerTokenGenerator>();
         services.AddSingleton<SystemInfoService>();
         services.AddSingleton<IUserIdProvider, IdBasedUserIdProvider>();
         services.AddHostedService(provider => provider.GetService<SystemInfoService>());

MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj

@@ -36,8 +36,10 @@
 			<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
 		</PackageReference>
 		<PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="7.0.0" />
+		<PackageReference Include="Microsoft.IdentityModel.Tokens" Version="6.25.1" />
 		<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="7.0.0" />
 		<PackageReference Include="prometheus-net" Version="7.0.0" />
+		<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.25.1" />
 		<PackageReference Include="System.Linq.Async" Version="6.0.1" />
 	</ItemGroup>
 

MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs

@@ -26,4 +26,6 @@ public class MetricsAPI
     public const string GaugeFilesUniquePastHourSize = "mare_files_unique_past_hour_size";
     public const string GaugeFilesUniquePastDay = "mare_files_unique_past_day";
     public const string GaugeFilesUniquePastDaySize = "mare_files_unique_past_day_size";
+    public const string GaugeCurrentDownloads = "mare_current_downloads";
+    public const string GaugeDownloadQueue = "mare_download_queue";
 }

MareSynchronosServer/MareSynchronosShared/Migrations/20230111092127_IsBannedForAuth.Designer.cs

@@ -0,0 +1,510 @@
+// <auto-generated />
+using System;
+using MareSynchronosShared.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+#nullable disable
+
+namespace MareSynchronosServer.Migrations
+{
+    [DbContext(typeof(MareDbContext))]
+    [Migration("20230111092127_IsBannedForAuth")]
+    partial class IsBannedForAuth
+    {
+        /// <inheritdoc />
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("ProductVersion", "7.0.0")
+                .HasAnnotation("Relational:MaxIdentifierLength", 63);
+
+            NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
+
+            modelBuilder.Entity("MareSynchronosShared.Models.Auth", b =>
+                {
+                    b.Property<string>("HashedKey")
+                        .HasMaxLength(64)
+                        .HasColumnType("character varying(64)")
+                        .HasColumnName("hashed_key");
+
+                    b.Property<bool>("IsBanned")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_banned");
+
+                    b.Property<string>("UserUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("user_uid");
+
+                    b.HasKey("HashedKey")
+                        .HasName("pk_auth");
+
+                    b.HasIndex("UserUID")
+                        .HasDatabaseName("ix_auth_user_uid");
+
+                    b.ToTable("auth", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.Banned", b =>
+                {
+                    b.Property<string>("CharacterIdentification")
+                        .HasMaxLength(100)
+                        .HasColumnType("character varying(100)")
+                        .HasColumnName("character_identification");
+
+                    b.Property<string>("Reason")
+                        .HasColumnType("text")
+                        .HasColumnName("reason");
+
+                    b.Property<byte[]>("Timestamp")
+                        .IsConcurrencyToken()
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("bytea")
+                        .HasColumnName("timestamp");
+
+                    b.HasKey("CharacterIdentification")
+                        .HasName("pk_banned_users");
+
+                    b.ToTable("banned_users", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.BannedRegistrations", b =>
+                {
+                    b.Property<string>("DiscordIdOrLodestoneAuth")
+                        .HasMaxLength(100)
+                        .HasColumnType("character varying(100)")
+                        .HasColumnName("discord_id_or_lodestone_auth");
+
+                    b.HasKey("DiscordIdOrLodestoneAuth")
+                        .HasName("pk_banned_registrations");
+
+                    b.ToTable("banned_registrations", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b =>
+                {
+                    b.Property<string>("UserUID")
+                        .HasMaxLength(10)
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("user_uid");
+
+                    b.Property<string>("OtherUserUID")
+                        .HasMaxLength(10)
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("other_user_uid");
+
+                    b.Property<bool>("AllowReceivingMessages")
+                        .HasColumnType("boolean")
+                        .HasColumnName("allow_receiving_messages");
+
+                    b.Property<bool>("IsPaused")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_paused");
+
+                    b.Property<byte[]>("Timestamp")
+                        .IsConcurrencyToken()
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("bytea")
+                        .HasColumnName("timestamp");
+
+                    b.HasKey("UserUID", "OtherUserUID")
+                        .HasName("pk_client_pairs");
+
+                    b.HasIndex("OtherUserUID")
+                        .HasDatabaseName("ix_client_pairs_other_user_uid");
+
+                    b.HasIndex("UserUID")
+                        .HasDatabaseName("ix_client_pairs_user_uid");
+
+                    b.ToTable("client_pairs", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b =>
+                {
+                    b.Property<string>("Hash")
+                        .HasMaxLength(40)
+                        .HasColumnType("character varying(40)")
+                        .HasColumnName("hash");
+
+                    b.Property<long>("Size")
+                        .HasColumnType("bigint")
+                        .HasColumnName("size");
+
+                    b.Property<byte[]>("Timestamp")
+                        .IsConcurrencyToken()
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("bytea")
+                        .HasColumnName("timestamp");
+
+                    b.Property<bool>("Uploaded")
+                        .HasColumnType("boolean")
+                        .HasColumnName("uploaded");
+
+                    b.Property<string>("UploaderUID")
+                        .HasMaxLength(10)
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("uploader_uid");
+
+                    b.HasKey("Hash")
+                        .HasName("pk_file_caches");
+
+                    b.HasIndex("UploaderUID")
+                        .HasDatabaseName("ix_file_caches_uploader_uid");
+
+                    b.ToTable("file_caches", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.ForbiddenUploadEntry", b =>
+                {
+                    b.Property<string>("Hash")
+                        .HasMaxLength(40)
+                        .HasColumnType("character varying(40)")
+                        .HasColumnName("hash");
+
+                    b.Property<string>("ForbiddenBy")
+                        .HasMaxLength(100)
+                        .HasColumnType("character varying(100)")
+                        .HasColumnName("forbidden_by");
+
+                    b.Property<byte[]>("Timestamp")
+                        .IsConcurrencyToken()
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("bytea")
+                        .HasColumnName("timestamp");
+
+                    b.HasKey("Hash")
+                        .HasName("pk_forbidden_upload_entries");
+
+                    b.ToTable("forbidden_upload_entries", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.Group", b =>
+                {
+                    b.Property<string>("GID")
+                        .HasMaxLength(20)
+                        .HasColumnType("character varying(20)")
+                        .HasColumnName("gid");
+
+                    b.Property<string>("Alias")
+                        .HasMaxLength(50)
+                        .HasColumnType("character varying(50)")
+                        .HasColumnName("alias");
+
+                    b.Property<string>("HashedPassword")
+                        .HasColumnType("text")
+                        .HasColumnName("hashed_password");
+
+                    b.Property<bool>("InvitesEnabled")
+                        .HasColumnType("boolean")
+                        .HasColumnName("invites_enabled");
+
+                    b.Property<string>("OwnerUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("owner_uid");
+
+                    b.HasKey("GID")
+                        .HasName("pk_groups");
+
+                    b.HasIndex("OwnerUID")
+                        .HasDatabaseName("ix_groups_owner_uid");
+
+                    b.ToTable("groups", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b =>
+                {
+                    b.Property<string>("GroupGID")
+                        .HasColumnType("character varying(20)")
+                        .HasColumnName("group_gid");
+
+                    b.Property<string>("BannedUserUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("banned_user_uid");
+
+                    b.Property<string>("BannedByUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("banned_by_uid");
+
+                    b.Property<DateTime>("BannedOn")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("banned_on");
+
+                    b.Property<string>("BannedReason")
+                        .HasColumnType("text")
+                        .HasColumnName("banned_reason");
+
+                    b.HasKey("GroupGID", "BannedUserUID")
+                        .HasName("pk_group_bans");
+
+                    b.HasIndex("BannedByUID")
+                        .HasDatabaseName("ix_group_bans_banned_by_uid");
+
+                    b.HasIndex("BannedUserUID")
+                        .HasDatabaseName("ix_group_bans_banned_user_uid");
+
+                    b.HasIndex("GroupGID")
+                        .HasDatabaseName("ix_group_bans_group_gid");
+
+                    b.ToTable("group_bans", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b =>
+                {
+                    b.Property<string>("GroupGID")
+                        .HasColumnType("character varying(20)")
+                        .HasColumnName("group_gid");
+
+                    b.Property<string>("GroupUserUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("group_user_uid");
+
+                    b.Property<bool>("IsModerator")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_moderator");
+
+                    b.Property<bool>("IsPaused")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_paused");
+
+                    b.Property<bool>("IsPinned")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_pinned");
+
+                    b.HasKey("GroupGID", "GroupUserUID")
+                        .HasName("pk_group_pairs");
+
+                    b.HasIndex("GroupGID")
+                        .HasDatabaseName("ix_group_pairs_group_gid");
+
+                    b.HasIndex("GroupUserUID")
+                        .HasDatabaseName("ix_group_pairs_group_user_uid");
+
+                    b.ToTable("group_pairs", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b =>
+                {
+                    b.Property<string>("GroupGID")
+                        .HasColumnType("character varying(20)")
+                        .HasColumnName("group_gid");
+
+                    b.Property<string>("Invite")
+                        .HasMaxLength(64)
+                        .HasColumnType("character varying(64)")
+                        .HasColumnName("invite");
+
+                    b.Property<DateTime>("ExpirationDate")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("expiration_date");
+
+                    b.HasKey("GroupGID", "Invite")
+                        .HasName("pk_group_temp_invites");
+
+                    b.HasIndex("GroupGID")
+                        .HasDatabaseName("ix_group_temp_invites_group_gid");
+
+                    b.HasIndex("Invite")
+                        .HasDatabaseName("ix_group_temp_invites_invite");
+
+                    b.ToTable("group_temp_invites", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b =>
+                {
+                    b.Property<decimal>("DiscordId")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("numeric(20,0)")
+                        .HasColumnName("discord_id");
+
+                    b.Property<string>("HashedLodestoneId")
+                        .HasMaxLength(100)
+                        .HasColumnType("character varying(100)")
+                        .HasColumnName("hashed_lodestone_id");
+
+                    b.Property<string>("LodestoneAuthString")
+                        .HasMaxLength(100)
+                        .HasColumnType("character varying(100)")
+                        .HasColumnName("lodestone_auth_string");
+
+                    b.Property<DateTime?>("StartedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("started_at");
+
+                    b.Property<string>("UserUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("user_uid");
+
+                    b.HasKey("DiscordId")
+                        .HasName("pk_lodestone_auth");
+
+                    b.HasIndex("UserUID")
+                        .HasDatabaseName("ix_lodestone_auth_user_uid");
+
+                    b.ToTable("lodestone_auth", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.User", b =>
+                {
+                    b.Property<string>("UID")
+                        .HasMaxLength(10)
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("uid");
+
+                    b.Property<string>("Alias")
+                        .HasMaxLength(15)
+                        .HasColumnType("character varying(15)")
+                        .HasColumnName("alias");
+
+                    b.Property<bool>("IsAdmin")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_admin");
+
+                    b.Property<bool>("IsModerator")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_moderator");
+
+                    b.Property<DateTime>("LastLoggedIn")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("last_logged_in");
+
+                    b.Property<byte[]>("Timestamp")
+                        .IsConcurrencyToken()
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("bytea")
+                        .HasColumnName("timestamp");
+
+                    b.HasKey("UID")
+                        .HasName("pk_users");
+
+                    b.ToTable("users", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.Auth", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "User")
+                        .WithMany()
+                        .HasForeignKey("UserUID")
+                        .HasConstraintName("fk_auth_users_user_temp_id");
+
+                    b.Navigation("User");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "OtherUser")
+                        .WithMany()
+                        .HasForeignKey("OtherUserUID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_client_pairs_users_other_user_temp_id1");
+
+                    b.HasOne("MareSynchronosShared.Models.User", "User")
+                        .WithMany()
+                        .HasForeignKey("UserUID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_client_pairs_users_user_temp_id2");
+
+                    b.Navigation("OtherUser");
+
+                    b.Navigation("User");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "Uploader")
+                        .WithMany()
+                        .HasForeignKey("UploaderUID")
+                        .HasConstraintName("fk_file_caches_users_uploader_uid");
+
+                    b.Navigation("Uploader");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.Group", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "Owner")
+                        .WithMany()
+                        .HasForeignKey("OwnerUID")
+                        .HasConstraintName("fk_groups_users_owner_temp_id7");
+
+                    b.Navigation("Owner");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "BannedBy")
+                        .WithMany()
+                        .HasForeignKey("BannedByUID")
+                        .HasConstraintName("fk_group_bans_users_banned_by_temp_id4");
+
+                    b.HasOne("MareSynchronosShared.Models.User", "BannedUser")
+                        .WithMany()
+                        .HasForeignKey("BannedUserUID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_group_bans_users_banned_user_temp_id5");
+
+                    b.HasOne("MareSynchronosShared.Models.Group", "Group")
+                        .WithMany()
+                        .HasForeignKey("GroupGID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_group_bans_groups_group_temp_id");
+
+                    b.Navigation("BannedBy");
+
+                    b.Navigation("BannedUser");
+
+                    b.Navigation("Group");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.Group", "Group")
+                        .WithMany()
+                        .HasForeignKey("GroupGID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_group_pairs_groups_group_temp_id1");
+
+                    b.HasOne("MareSynchronosShared.Models.User", "GroupUser")
+                        .WithMany()
+                        .HasForeignKey("GroupUserUID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_group_pairs_users_group_user_temp_id6");
+
+                    b.Navigation("Group");
+
+                    b.Navigation("GroupUser");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.Group", "Group")
+                        .WithMany()
+                        .HasForeignKey("GroupGID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_group_temp_invites_groups_group_gid");
+
+                    b.Navigation("Group");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "User")
+                        .WithMany()
+                        .HasForeignKey("UserUID")
+                        .HasConstraintName("fk_lodestone_auth_users_user_uid");
+
+                    b.Navigation("User");
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}

MareSynchronosServer/MareSynchronosShared/Migrations/20230111092127_IsBannedForAuth.cs

@@ -0,0 +1,29 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace MareSynchronosServer.Migrations
+{
+    /// <inheritdoc />
+    public partial class IsBannedForAuth : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<bool>(
+                name: "is_banned",
+                table: "auth",
+                type: "boolean",
+                nullable: false,
+                defaultValue: false);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                name: "is_banned",
+                table: "auth");
+        }
+    }
+}

MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs

@@ -29,6 +29,10 @@ namespace MareSynchronosServer.Migrations
                         .HasColumnType("character varying(64)")
                         .HasColumnName("hashed_key");
 
+                    b.Property<bool>("IsBanned")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_banned");
+
                     b.Property<string>("UserUID")
                         .HasColumnType("character varying(10)")
                         .HasColumnName("user_uid");

MareSynchronosServer/MareSynchronosShared/Models/Auth.cs

@@ -10,4 +10,5 @@ public class Auth
 
     public string UserUID { get; set; }
     public User User { get; set; }
+    public bool IsBanned { get; set; }
 }

MareSynchronosServer/MareSynchronosShared/Utils/MareClaimTypes.cs

@@ -4,4 +4,5 @@ public static class MareClaimTypes
 {
     public const string Uid = "uid";
     public const string CharaIdent = "character_identification";
+    public const string Internal = "internal";
 }

MareSynchronosServer/MareSynchronosShared/Utils/ServerTokenGenerator.cs

@@ -0,0 +1,54 @@
+using MareSynchronosShared.Services;
+using Microsoft.IdentityModel.Tokens;
+using System.IdentityModel.Tokens.Jwt;
+using System.Security.Claims;
+using System.Text;
+
+namespace MareSynchronosShared.Utils;
+
+public class ServerTokenGenerator
+{
+    private readonly IConfigurationService<MareConfigurationAuthBase> _configuration;
+    private Dictionary<string, string> _tokenDictionary { get; set; } = new(StringComparer.Ordinal);
+    public string Token
+    {
+        get
+        {
+            var currentJwt = _configuration.GetValue<string>(nameof(MareConfigurationAuthBase.Jwt));
+            if (_tokenDictionary.TryGetValue(currentJwt, out var token))
+            {
+                return token;
+            }
+
+            return GenerateToken();
+        }
+    }
+
+    public ServerTokenGenerator(IConfigurationService<MareConfigurationAuthBase> configuration)
+    {
+        _configuration = configuration;
+    }
+
+    private string GenerateToken()
+    {
+        var signingKey = _configuration.GetValue<string>(nameof(MareConfigurationAuthBase.Jwt));
+        var authSigningKey = new SymmetricSecurityKey(Encoding.ASCII.GetBytes(signingKey));
+
+        var token = new SecurityTokenDescriptor()
+        {
+            Subject = new ClaimsIdentity(new List<Claim>()
+            {
+                new Claim(MareClaimTypes.Uid, _configuration.GetValue<string>(nameof(MareConfigurationBase.ShardName))),
+                new Claim(MareClaimTypes.Internal, true.ToString())
+            }),
+            SigningCredentials = new SigningCredentials(authSigningKey, SecurityAlgorithms.HmacSha256Signature)
+        };
+
+        var handler = new JwtSecurityTokenHandler();
+        var rawData = handler.CreateJwtSecurityToken(token).RawData;
+
+        _tokenDictionary[signingKey] = rawData;
+
+        return rawData;
+    }
+}

MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs

@@ -12,6 +12,9 @@ public class StaticFilesServerConfiguration : MareConfigurationBase
     public string CacheDirectory { get; set; }
     public Uri? RemoteCacheSourceUri { get; set; } = null;
     public Uri MainServerGrpcAddress { get; set; } = null;
+    public int DownloadQueueSize { get; set; } = 50;
+    public int DownloadTimeoutSeconds { get; set; } = 5;
+    public int DownloadQueueReleaseSeconds { get; set; } = 15;
     public override string ToString()
     {
         StringBuilder sb = new();
@@ -23,6 +26,8 @@ public class StaticFilesServerConfiguration : MareConfigurationBase
         sb.AppendLine($"{nameof(UnusedFileRetentionPeriodInDays)} => {UnusedFileRetentionPeriodInDays}");
         sb.AppendLine($"{nameof(CacheDirectory)} => {CacheDirectory}");
         sb.AppendLine($"{nameof(RemoteCacheSourceUri)} => {RemoteCacheSourceUri}");
+        sb.AppendLine($"{nameof(DownloadQueueSize)} => {DownloadQueueSize}");
+        sb.AppendLine($"{nameof(DownloadQueueReleaseSeconds)} => {DownloadQueueReleaseSeconds}");
         return sb.ToString();
     }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs

@@ -0,0 +1,42 @@
+using MareSynchronos.API;
+using MareSynchronosShared.Utils;
+using MareSynchronosStaticFilesServer.Services;
+using MareSynchronosStaticFilesServer.Utils;
+using Microsoft.AspNetCore.Mvc;
+
+namespace MareSynchronosStaticFilesServer.Controllers;
+
+[Route(MareFiles.Cache)]
+public class CacheController : ControllerBase
+{
+    private readonly RequestFileStreamResultFactory _requestFileStreamResultFactory;
+    private readonly CachedFileProvider _cachedFileProvider;
+    private readonly RequestQueueService _requestQueue;
+
+    public CacheController(ILogger<CacheController> logger, RequestFileStreamResultFactory requestFileStreamResultFactory,
+        CachedFileProvider cachedFileProvider, RequestQueueService requestQueue, ServerTokenGenerator generator) : base(logger, generator)
+    {
+        _requestFileStreamResultFactory = requestFileStreamResultFactory;
+        _cachedFileProvider = cachedFileProvider;
+        _requestQueue = requestQueue;
+    }
+
+    [HttpGet(MareFiles.Cache_Get)]
+    public async Task<IActionResult> GetFile(Guid requestId)
+    {
+        _logger.LogDebug($"GetFile:{MareUser}:{requestId}");
+
+        if (!_requestQueue.IsActiveProcessing(requestId, MareUser, out var request)) return BadRequest();
+
+        _requestQueue.ActivateRequest(requestId);
+
+        var fs = await _cachedFileProvider.GetAndDownloadFileStream(request.FileId, Authorization);
+        if (fs == null)
+        {
+            _requestQueue.FinishRequest(requestId);
+            return NotFound();
+        }
+
+        return _requestFileStreamResultFactory.Create(requestId, fs);
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs

@@ -0,0 +1,19 @@
+using MareSynchronosShared.Utils;
+using Microsoft.AspNetCore.Mvc;
+
+namespace MareSynchronosStaticFilesServer.Controllers;
+
+public class ControllerBase : Controller
+{
+    protected ILogger _logger;
+    private readonly ServerTokenGenerator _generator;
+
+    public ControllerBase(ILogger logger, ServerTokenGenerator generator)
+    {
+        _logger = logger;
+        _generator = generator;
+    }
+
+    protected string MareUser => HttpContext.User.Claims.First(f => string.Equals(f.Type, MareClaimTypes.Uid, StringComparison.Ordinal)).Value;
+    protected string Authorization => "Bearer " + _generator.Token;
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs

@@ -0,0 +1,54 @@
+using MareSynchronos.API;
+using MareSynchronosShared.Utils;
+using MareSynchronosStaticFilesServer.Services;
+using Microsoft.AspNetCore.Mvc;
+using System.Text.Json;
+
+namespace MareSynchronosStaticFilesServer.Controllers;
+
+[Route(MareFiles.Request)]
+public class RequestController : ControllerBase
+{
+    private readonly CachedFileProvider _cachedFileProvider;
+    private readonly RequestQueueService _requestQueue;
+
+    public RequestController(ILogger<RequestController> logger, CachedFileProvider cachedFileProvider, RequestQueueService requestQueue, 
+        ServerTokenGenerator generator) : base(logger, generator)
+    {
+        _cachedFileProvider = cachedFileProvider;
+        _requestQueue = requestQueue;
+    }
+
+    [HttpPost]
+    [Route(MareFiles.Request_Enqueue)]
+    public IActionResult PreRequestFiles([FromBody] List<string> files)
+    {
+        foreach (var file in files)
+        {
+            _cachedFileProvider.DownloadFileWhenRequired(file, Authorization);
+        }
+
+        return Ok();
+    }
+
+    [HttpGet]
+    [Route(MareFiles.Request_RequestFile)]
+    public async Task<IActionResult> RequestFile(string file)
+    {
+        Guid g = Guid.NewGuid();
+        _cachedFileProvider.DownloadFileWhenRequired(file, Authorization);
+        var queueStatus = await _requestQueue.EnqueueUser(new(g, MareUser, file));
+        return Ok(JsonSerializer.Serialize(new QueueRequestDto(g, queueStatus)));
+    }
+
+    [HttpGet]
+    [Route(MareFiles.Request_CheckQueue)]
+    public IActionResult CheckQueue(Guid requestId)
+    {
+        if (_requestQueue.IsActiveProcessing(requestId, MareUser, out _)) return Ok();
+
+        if (_requestQueue.StillEnqueued(requestId, MareUser, out int position)) return Conflict(position);
+
+        return BadRequest();
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs

@@ -0,0 +1,30 @@
+using MareSynchronos.API;
+using MareSynchronosShared.Utils;
+using MareSynchronosStaticFilesServer.Services;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+
+namespace MareSynchronosStaticFilesServer.Controllers;
+
+[Route(MareFiles.ServerFiles)]
+public class ServerFilesController : ControllerBase
+{
+    private readonly CachedFileProvider _cachedFileProvider;
+
+    public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider, ServerTokenGenerator generator) : base(logger, generator)
+    {
+        _cachedFileProvider = cachedFileProvider;
+    }
+
+    [HttpGet(MareFiles.ServerFiles_Get + "/{fileId}")]
+    [Authorize(Policy = "Internal")]
+    public async Task<IActionResult> GetFile(string fileId)
+    {
+        _logger.LogInformation($"GetFile:{MareUser}:{fileId}");
+
+        var fs = _cachedFileProvider.GetLocalFileStream(fileId);
+        if (fs == null) return NotFound();
+
+        return File(fs, "application/octet-stream");
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs

@@ -1,29 +0,0 @@
-using MareSynchronosShared.Utils;
-using Microsoft.AspNetCore.Mvc;
-
-namespace MareSynchronosStaticFilesServer;
-
-[Route("/cache")]
-public class FilesController : Controller
-{
-    private readonly ILogger<FilesController> _logger;
-    private readonly CachedFileProvider _cachedFileProvider;
-
-    public FilesController(ILogger<FilesController> logger, CachedFileProvider cachedFileProvider)
-    {
-        _logger = logger;
-        _cachedFileProvider = cachedFileProvider;
-    }
-
-    [HttpGet("{fileId}")]
-    public async Task<IActionResult> GetFile(string fileId)
-    {
-        var authedUser = HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, MareClaimTypes.Uid, StringComparison.Ordinal))?.Value ?? "Unknown";
-        _logger.LogInformation($"GetFile:{authedUser}:{fileId}");
-
-        var fs = await _cachedFileProvider.GetFileStream(fileId, Request.Headers["Authorization"]);
-        if (fs == null) return NotFound();
-
-        return File(fs, "application/octet-stream");
-    }
-}

MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj

@@ -30,6 +30,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\MareAPI\MareSynchronosAPI\MareSynchronos.API.csproj" />
     <ProjectReference Include="..\MareSynchronosShared\MareSynchronosShared.csproj" />
   </ItemGroup>
 

MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs

@@ -1,6 +1,5 @@
 using MareSynchronosShared.Services;
 using MareSynchronosShared.Utils;
-using Microsoft.Extensions.Options;
 
 namespace MareSynchronosStaticFilesServer;
 

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs

@@ -1,9 +1,11 @@
 using MareSynchronosShared.Metrics;
 using MareSynchronosShared.Services;
-using Microsoft.Extensions.Options;
+using MareSynchronosStaticFilesServer.Utils;
 using System.Collections.Concurrent;
+using System.Net.Http.Headers;
+using MareSynchronos.API;
 
-namespace MareSynchronosStaticFilesServer;
+namespace MareSynchronosStaticFilesServer.Services;
 
 public class CachedFileProvider
 {
@@ -13,6 +15,7 @@ public class CachedFileProvider
     private readonly Uri _remoteCacheSourceUri;
     private readonly string _basePath;
     private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
+    private readonly HttpClient _httpClient;
     private bool IsMainServer => _remoteCacheSourceUri == null;
 
     public CachedFileProvider(IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics)
@@ -22,16 +25,18 @@ public class CachedFileProvider
         _metrics = metrics;
         _remoteCacheSourceUri = configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.RemoteCacheSourceUri), null);
         _basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
+        _httpClient = new HttpClient();
     }
 
     private async Task DownloadTask(string hash, string auth)
     {
         // download file from remote
-        var downloadUrl = new Uri(_remoteCacheSourceUri, hash);
+        var downloadUrl = MareFiles.ServerFilesGetFullPath(_remoteCacheSourceUri, hash);
         _logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl);
-        using var client = new HttpClient();
+
-        client.DefaultRequestHeaders.Add("Authorization", auth);
+        using var requestMessage = new HttpRequestMessage(HttpMethod.Get, downloadUrl);
-        var response = await client.GetAsync(downloadUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
+        requestMessage.Headers.Authorization = new AuthenticationHeaderValue(auth);
+        var response = await _httpClient.SendAsync(requestMessage).ConfigureAwait(false);
 
         try
         {
@@ -58,26 +63,40 @@ public class CachedFileProvider
         _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_basePath, hash).Length);
     }
 
-    public async Task<FileStream?> GetFileStream(string hash, string auth)
+    public void DownloadFileWhenRequired(string hash, string auth)
     {
         var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
-        if (fi == null && IsMainServer) return null;
+        if (fi == null && IsMainServer) return;
 
         if (fi == null && !_currentTransfers.ContainsKey(hash))
         {
-            _currentTransfers[hash] = DownloadTask(hash, auth).ContinueWith(r => _currentTransfers.Remove(hash, out _));
+            _currentTransfers[hash] = Task.Run(async () =>
+            {
+                await DownloadTask(hash, auth).ConfigureAwait(false);
+                _currentTransfers.Remove(hash, out _);
+            });
         }
+    }
 
-        if (_currentTransfers.TryGetValue(hash, out var downloadTask))
+    public FileStream? GetLocalFileStream(string hash)
-        {
+    {
-            await downloadTask.ConfigureAwait(false);
+        var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
-        }
-
-        fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
         if (fi == null) return null;
 
         _fileStatisticsService.LogFile(hash, fi.Length);
 
         return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read);
     }
+
+    public async Task<FileStream?> GetAndDownloadFileStream(string hash, string auth)
+    {
+        DownloadFileWhenRequired(hash, auth);
+
+        if (_currentTransfers.TryGetValue(hash, out var downloadTask))
+        {
+            await downloadTask.ConfigureAwait(false);
+        }
+
+        return GetLocalFileStream(hash);
+    }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs

@@ -3,9 +3,10 @@ using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
 using MareSynchronosShared.Models;
 using MareSynchronosShared.Services;
+using MareSynchronosStaticFilesServer.Utils;
 using Microsoft.EntityFrameworkCore;
 
-namespace MareSynchronosStaticFilesServer;
+namespace MareSynchronosStaticFilesServer.Services;
 
 public class FileCleanupService : IHostedService
 {
@@ -99,7 +100,7 @@ public class FileCleanupService : IHostedService
                 if (_isMainServer)
                 {
                     FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() };
-                    dbContext.Entry(f).State = Microsoft.EntityFrameworkCore.EntityState.Deleted;
+                    dbContext.Entry(f).State = EntityState.Deleted;
                 }
             }
         }
@@ -113,8 +114,8 @@ public class FileCleanupService : IHostedService
     {
         try
         {
-            var unusedRetention = _configuration.GetValueOrDefault<int>(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
+            var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
-            var forcedDeletionAfterHours = _configuration.GetValueOrDefault<int>(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
+            var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
 
             _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", unusedRetention);
             if (forcedDeletionAfterHours > 0)

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileStatisticsService.cs

@@ -1,7 +1,7 @@
 using MareSynchronosShared.Metrics;
 using System.Collections.Concurrent;
 
-namespace MareSynchronosStaticFilesServer;
+namespace MareSynchronosStaticFilesServer.Services;
 
 public class FileStatisticsService : IHostedService
 {

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/GrpcFileService.cs

@@ -3,10 +3,13 @@ using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
 using MareSynchronosShared.Protos;
 using MareSynchronosShared.Services;
+using MareSynchronosStaticFilesServer.Utils;
+using Microsoft.AspNetCore.Authorization;
 using Microsoft.EntityFrameworkCore;
 
-namespace MareSynchronosStaticFilesServer;
+namespace MareSynchronosStaticFilesServer.Services;
 
+[Authorize(Policy = "Internal")]
 public class GrpcFileService : FileService.FileServiceBase
 {
     private readonly string _basePath;
@@ -22,6 +25,7 @@ public class GrpcFileService : FileService.FileServiceBase
         _metricsClient = metricsClient;
     }
 
+    [Authorize(Policy = "Internal")]
     public override async Task<Empty> UploadFile(IAsyncStreamReader<UploadFileRequest> requestStream, ServerCallContext context)
     {
         _ = await requestStream.MoveNext().ConfigureAwait(false);
@@ -31,7 +35,6 @@ public class GrpcFileService : FileService.FileServiceBase
         var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == uploadMsg.Hash && f.UploaderUID == uploadMsg.Uploader).ConfigureAwait(false);
         try
         {
-
             if (file != null)
             {
                 await fileWriter.WriteAsync(uploadMsg.FileData.ToArray()).ConfigureAwait(false);
@@ -71,6 +74,7 @@ public class GrpcFileService : FileService.FileServiceBase
         return new Empty();
     }
 
+    [Authorize(Policy = "Internal")]
     public override async Task<Empty> DeleteFiles(DeleteFilesRequest request, ServerCallContext context)
     {
         foreach (var hash in request.Hash)

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs

@@ -0,0 +1,137 @@
+using MareSynchronos.API;
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Services;
+using MareSynchronosStaticFilesServer.Utils;
+using System.Collections.Concurrent;
+
+namespace MareSynchronosStaticFilesServer.Services;
+
+public class RequestQueueService : IHostedService
+{
+    private CancellationTokenSource _queueCts = new();
+    private readonly UserQueueEntry[] _userQueueRequests;
+    private readonly ConcurrentQueue<UserRequest> _queue = new();
+    private readonly MareMetrics _metrics;
+    private readonly ILogger<RequestQueueService> _logger;
+    private readonly int _queueExpirationSeconds;
+    private SemaphoreSlim _queueSemaphore = new(1);
+    private SemaphoreSlim _queueProcessingSemaphore = new(1);
+
+    public RequestQueueService(MareMetrics metrics, IConfigurationService<StaticFilesServerConfiguration> configurationService, ILogger<RequestQueueService> logger)
+    {
+        _userQueueRequests = new UserQueueEntry[configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueSize), 50)];
+        _queueExpirationSeconds = configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadTimeoutSeconds), 5);
+        _metrics = metrics;
+        _logger = logger;
+    }
+
+    public async Task<QueueStatus> EnqueueUser(UserRequest request)
+    {
+        _logger.LogDebug("Enqueueing req {guid} from {user} for {file}", request.RequestId, request.User, request.FileId);
+
+        if (_queueProcessingSemaphore.CurrentCount == 0)
+        {
+            _queue.Enqueue(request);
+            return QueueStatus.Waiting;
+        }
+
+        await _queueSemaphore.WaitAsync().ConfigureAwait(false);
+        QueueStatus status = QueueStatus.Waiting;
+        var idx = Array.FindIndex(_userQueueRequests, r => r == null);
+        if (idx == -1)
+        {
+            _queue.Enqueue(request);
+            status = QueueStatus.Waiting;
+        }
+        else
+        {
+            DequeueIntoSlot(request, idx);
+            status = QueueStatus.Ready;
+        }
+        _queueSemaphore.Release();
+
+        return status;
+    }
+
+    public bool StillEnqueued(Guid request, string user, out int queuePosition)
+    {
+        var result = _queue.FirstOrDefault(c => c.RequestId == request && string.Equals(c.User, user, StringComparison.Ordinal));
+        if (result != null)
+        {
+            queuePosition = Array.IndexOf(_queue.ToArray(), result);
+            return true;
+        }
+
+        queuePosition = -1;
+        return false;
+    }
+
+    public bool IsActiveProcessing(Guid request, string user, out UserRequest userRequest)
+    {
+        var userQueueRequest = _userQueueRequests.Where(u => u != null)
+            .FirstOrDefault(f => f.UserRequest.RequestId == request && string.Equals(f.UserRequest.User, user, StringComparison.Ordinal));
+        userRequest = userQueueRequest?.UserRequest ?? null;
+        return userQueueRequest != null && userRequest != null && userQueueRequest.ExpirationDate > DateTime.UtcNow;
+    }
+
+    public void FinishRequest(Guid request)
+    {
+        var req = _userQueueRequests.First(f => f.UserRequest.RequestId == request);
+        var idx = Array.IndexOf(_userQueueRequests, req);
+        _logger.LogDebug("Finishing Request {guid}, clearing slot {idx}", request, idx);
+        _userQueueRequests[idx] = null;
+    }
+
+    public void ActivateRequest(Guid request)
+    {
+        _logger.LogDebug("Activating request {guid}", request);
+        _userQueueRequests.First(f => f.UserRequest.RequestId == request).IsActive = true;
+    }
+
+    private async Task ProcessRequestQueue(CancellationToken ct)
+    {
+        while (!ct.IsCancellationRequested)
+        {
+            await _queueProcessingSemaphore.WaitAsync(ct).ConfigureAwait(false);
+            await _queueSemaphore.WaitAsync(ct).ConfigureAwait(false);
+            for (int i = 0; i < _userQueueRequests.Length; i++)
+            {
+                if (_userQueueRequests[i] != null && !_userQueueRequests[i].IsActive && _userQueueRequests[i].ExpirationDate < DateTime.UtcNow) _userQueueRequests[i] = null;
+
+                if (_userQueueRequests[i] == null)
+                {
+                    if (_queue.TryDequeue(out var request))
+                    {
+                        DequeueIntoSlot(request, i);
+                    }
+                }
+
+                if (!_queue.Any()) break;
+            }
+            _queueProcessingSemaphore.Release();
+            _queueSemaphore.Release();
+
+            _metrics.SetGaugeTo(MetricsAPI.GaugeDownloadQueue, _queue.Count);
+
+            await Task.Delay(250).ConfigureAwait(false);
+        }
+    }
+
+    private void DequeueIntoSlot(UserRequest userRequest, int slot)
+    {
+        _logger.LogDebug("Dequeueing {req} into {i}: {user} with {file}", userRequest.RequestId, slot, userRequest.User, userRequest.FileId);
+        _userQueueRequests[slot] = new(userRequest, DateTime.UtcNow.AddSeconds(_queueExpirationSeconds));
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        _ = Task.Run(() => ProcessRequestQueue(_queueCts.Token));
+        return Task.CompletedTask;
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        _queueCts.Cancel();
+        return Task.CompletedTask;
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs

@@ -5,6 +5,8 @@ using MareSynchronosShared.Metrics;
 using MareSynchronosShared.Protos;
 using MareSynchronosShared.Services;
 using MareSynchronosShared.Utils;
+using MareSynchronosStaticFilesServer.Services;
+using MareSynchronosStaticFilesServer.Utils;
 using Microsoft.AspNetCore.Authentication.JwtBearer;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Server.Kestrel.Core;
@@ -41,8 +43,6 @@ public class Startup
 
         var mareConfig = Configuration.GetRequiredSection("MareSynchronos");
 
-        services.AddControllers();
-
         services.AddSingleton(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string>
         {
         }, new List<string>
@@ -52,10 +52,13 @@ public class Startup
             MetricsAPI.GaugeFilesUniquePastDay,
             MetricsAPI.GaugeFilesUniquePastDaySize,
             MetricsAPI.GaugeFilesUniquePastHour,
-            MetricsAPI.GaugeFilesUniquePastHourSize
+            MetricsAPI.GaugeFilesUniquePastHourSize,
+            MetricsAPI.GaugeCurrentDownloads,
+            MetricsAPI.GaugeDownloadQueue,
         }));
         services.AddSingleton<CachedFileProvider>();
         services.AddSingleton<FileStatisticsService>();
+        services.AddSingleton<RequestFileStreamResultFactory>();
 
         services.AddHostedService(m => m.GetService<FileStatisticsService>());
         services.AddHostedService<FileCleanupService>();
@@ -119,7 +122,11 @@ public class Startup
             o.DefaultScheme = JwtBearerDefaults.AuthenticationScheme;
         }).AddJwtBearer();
 
-        services.AddAuthorization(options => options.FallbackPolicy = new AuthorizationPolicyBuilder().RequireAuthenticatedUser().Build());
+        services.AddAuthorization(options =>
+        {
+            options.FallbackPolicy = new AuthorizationPolicyBuilder().RequireAuthenticatedUser().Build();
+            options.AddPolicy("Internal", new AuthorizationPolicyBuilder().RequireClaim(MareClaimTypes.Internal, "true").Build());
+        });
 
         if (_isMain)
         {
@@ -148,9 +155,15 @@ public class Startup
                 p.GetRequiredService<GrpcClientFactory>(), "MainServer")
         );
 
+        services.AddSingleton<ServerTokenGenerator>();
+        services.AddSingleton<RequestQueueService>();
+        services.AddHostedService(p => p.GetService<RequestQueueService>());
+        services.AddControllers();
+
         services.AddHostedService(p => (MareConfigurationServiceClient<MareConfigurationAuthBase>)p.GetService<IConfigurationService<MareConfigurationAuthBase>>());
 
         services.AddHealthChecks();
+        services.AddControllers();
     }
 
     public void Configure(IApplicationBuilder app, IWebHostEnvironment env)

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/FilePathUtil.cs

@@ -1,8 +1,8 @@
-namespace MareSynchronosStaticFilesServer;
+namespace MareSynchronosStaticFilesServer.Utils;
 
 public static class FilePathUtil
 {
-    public static FileInfo? GetFileInfoForHash(string basePath, string hash)
+    public static FileInfo GetFileInfoForHash(string basePath, string hash)
     {
         FileInfo fi = new(Path.Combine(basePath, hash[0].ToString(), hash));
         if (!fi.Exists)

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs

@@ -0,0 +1,59 @@
+using MareSynchronosShared.Metrics;
+using MareSynchronosStaticFilesServer.Services;
+using Microsoft.AspNetCore.Mvc;
+
+namespace MareSynchronosStaticFilesServer.Utils;
+
+public class RequestFileStreamResult : FileStreamResult
+{
+    private readonly Guid _requestId;
+    private readonly RequestQueueService _requestQueueService;
+    private readonly MareMetrics _mareMetrics;
+    private readonly CancellationTokenSource _releaseCts = new();
+    private bool _releasedSlot = false;
+
+    public RequestFileStreamResult(Guid requestId, int secondsUntilRelease, RequestQueueService requestQueueService,
+        MareMetrics mareMetrics, Stream fileStream, string contentType) : base(fileStream, contentType)
+    {
+        _requestId = requestId;
+        _requestQueueService = requestQueueService;
+        _mareMetrics = mareMetrics;
+        _mareMetrics.IncGauge(MetricsAPI.GaugeCurrentDownloads);
+
+        // forcefully release slot after secondsUntilRelease
+        Task.Run(async () =>
+        {
+            try
+            {
+                await Task.Delay(TimeSpan.FromSeconds(secondsUntilRelease), _releaseCts.Token).ConfigureAwait(false);
+                _requestQueueService.FinishRequest(_requestId);
+                _releasedSlot = true;
+            }
+            catch { }
+        });
+    }
+
+    public override void ExecuteResult(ActionContext context)
+    {
+        base.ExecuteResult(context);
+
+        _releaseCts.Cancel();
+
+        if (!_releasedSlot)
+            _requestQueueService.FinishRequest(_requestId);
+
+        _mareMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads);
+    }
+
+    public override async Task ExecuteResultAsync(ActionContext context)
+    {
+        await base.ExecuteResultAsync(context).ConfigureAwait(false);
+
+        _releaseCts.Cancel();
+
+        if (!_releasedSlot)
+            _requestQueueService.FinishRequest(_requestId);
+
+        _mareMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads);
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResultFactory.cs

@@ -0,0 +1,25 @@
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Services;
+using MareSynchronosStaticFilesServer.Services;
+
+namespace MareSynchronosStaticFilesServer.Utils;
+
+public class RequestFileStreamResultFactory
+{
+    private readonly MareMetrics _metrics;
+    private readonly RequestQueueService _requestQueueService;
+    private readonly IConfigurationService<StaticFilesServerConfiguration> _configurationService;
+
+    public RequestFileStreamResultFactory(MareMetrics metrics, RequestQueueService requestQueueService, IConfigurationService<StaticFilesServerConfiguration> configurationService)
+    {
+        _metrics = metrics;
+        _requestQueueService = requestQueueService;
+        _configurationService = configurationService;
+    }
+
+    public RequestFileStreamResult Create(Guid requestId, FileStream fs)
+    {
+        return new RequestFileStreamResult(requestId, _configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueReleaseSeconds), 15),
+            _requestQueueService, _metrics, fs, "application/octet-stream");
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserQueueEntry.cs

@@ -0,0 +1,6 @@
+namespace MareSynchronosStaticFilesServer.Utils;
+
+public record UserQueueEntry(UserRequest UserRequest, DateTime ExpirationDate)
+{
+    public bool IsActive { get; set; } = false;
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserRequest.cs

@@ -0,0 +1,3 @@
+namespace MareSynchronosStaticFilesServer.Utils;
+
+public record UserRequest(Guid RequestId, string User, string FileId);