t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Server-Side Download Queue (#21)

Browse Source 
* test add queueing to file service

* further adjustments to download queueing

* add check for whether the request is still in the queue to CheckQueue

* forcefully release slot if download didn't finish in 15s

* actually cancel the delay task

* add metrics and refactor some of the request queue service

* refactor pathing

* reuse httpclient

* add queue request dto to requestfile, enqueue users immediately if a slot is available

* change startup to include all controllers

* update server pathing

* update pathing, again

* several adjustments to auth, banning, jwt server tokens, renaming, authorization

* update api I guess

* adjust automated banning of charaident and reg

* generate jwt on servers for internal authentication

* remove mvcextensions

Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>
This commit is contained in:
rootdarkarchon
2023-01-11 12:22:22 +01:00
committed by GitHub
parent db2d0451ca
commit 42b15cb6b7
38 changed files with 1116 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
Docker/run/config/sharded/files-shard-1.json
View File

@@ -7,7 +7,7 @@
"Default": "Warning",
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information",
"MareSynchronosStaticFilesServer": "Information",
"MareSynchronosStaticFilesServer": "Debug",
"MareSynchronosShared": "Information",
"System.IO": "Information"
},
@@ -33,8 +33,11 @@
"CacheSizeHardLimitInGiB": 5,
"UnusedFileRetentionPeriodInDays": 14,
"CacheDirectory": "/marecache/",
"RemoteCacheSourceUri": "http://mare-files:6200/cache/",
"MainServerGrpcAddress": "http://mare-server:6005"
"RemoteCacheSourceUri": "http://mare-files:6200/",
"MainServerGrpcAddress": "http://mare-server:6005",
"DownloadTimeoutSeconds": 30,
"DownloadQueueSize": 50,
"DownloadQueueReleaseSeconds": 15
},
"AllowedHosts": "*",
"Kestrel": {

9
Docker/run/config/sharded/files-shard-2.json
View File

@@ -7,7 +7,7 @@
"Default": "Warning",
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information",
"MareSynchronosStaticFilesServer": "Information",
"MareSynchronosStaticFilesServer": "Debug",
"MareSynchronosShared": "Information",
"System.IO": "Information"
},
@@ -33,8 +33,11 @@
"CacheSizeHardLimitInGiB": 5,
"UnusedFileRetentionPeriodInDays": 14,
"CacheDirectory": "/marecache/",
"RemoteCacheSourceUri": "http://mare-files:6200/cache/",
"MainServerGrpcAddress": "http://mare-server:6005"
"RemoteCacheSourceUri": "http://mare-files:6200/",
"MainServerGrpcAddress": "http://mare-server:6005",
"DownloadTimeoutSeconds": 30,
"DownloadQueueSize": 50,
"DownloadQueueReleaseSeconds": 15
},
"AllowedHosts": "*",
"Kestrel": {

2
Docker/run/config/sharded/server-shard-main.json
View File

@@ -36,7 +36,7 @@
""
],
"RedisConnectionString": "redis,password=secretredispassword",
"CdnFullUrl": "http://localhost:6200/cache/",
"CdnFullUrl": "http://localhost:6200/",
"StaticFileServiceAddress": "http://mare-files:6205",
"MaxExistingGroupsByUser": 3,
"MaxJoinedGroupsByUser": 6,

2
Docker/run/config/standalone/server-standalone.json
View File

@@ -36,7 +36,7 @@
""
],
"RedisConnectionString": "redis,password=secretredispassword",
"CdnFullUrl": "http://localhost:6200/cache/",
"CdnFullUrl": "http://localhost:6200/",
"StaticFileServiceAddress": "http://mare-files:6205",
"MaxExistingGroupsByUser": 3,
"MaxJoinedGroupsByUser": 6,

2
MareAPI

Submodule MareAPI updated: d361cfa3b9...0720e35c78

2
MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthReply.cs
View File

@@ -1,3 +1,3 @@
namespace MareSynchronosServer.Authentication;
public record SecretKeyAuthReply(bool Success, string Uid, bool TempBan);
public record SecretKeyAuthReply(bool Success, string Uid, bool TempBan, bool Permaban);

6
MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthenticatorService.cs
View File

@@ -50,14 +50,14 @@ public class SecretKeyAuthenticatorService
_failedAuthorizations.Remove(ip, out _);
});
}
return new(Success: false, Uid: null, TempBan: true);
return new(Success: false, Uid: null, TempBan: true, Permaban: false);
}
using var scope = _serviceScopeFactory.CreateScope();
using var context = scope.ServiceProvider.GetService<MareDbContext>();
var authReply = await context.Auth.AsNoTracking().SingleOrDefaultAsync(u => u.HashedKey == hashedSecretKey).ConfigureAwait(false);
SecretKeyAuthReply reply = new(authReply != null, authReply?.UserUID, false);
SecretKeyAuthReply reply = new(authReply != null, authReply?.UserUID, false, authReply.IsBanned);
if (reply.Success)
{
@@ -97,6 +97,6 @@ public class SecretKeyAuthenticatorService
}
}
return new(Success: false, Uid: null, TempBan: false);
return new(Success: false, Uid: null, TempBan: false, Permaban: false);
}
}

64
MareSynchronosServer/MareSynchronosServer/Controllers/JwtController.cs
View File

@@ -2,6 +2,7 @@
using MareSynchronosServer.Authentication;
using MareSynchronosShared;
using MareSynchronosShared.Data;
using MareSynchronosShared.Models;
using MareSynchronosShared.Services;
using MareSynchronosShared.Utils;
using Microsoft.AspNetCore.Authorization;
@@ -38,7 +39,7 @@ public class JwtController : Controller
}
[AllowAnonymous]
[HttpPost(MareAuth.AuthCreateIdent)]
[HttpPost(MareAuth.Auth_CreateIdent)]
public async Task<IActionResult> CreateToken(string auth, string charaIdent)
{
if (string.IsNullOrEmpty(auth)) return BadRequest("No Authkey");
@@ -52,10 +53,47 @@ public class JwtController : Controller
var authResult = await _secretKeyAuthenticatorService.AuthorizeAsync(ip, auth);
if (!authResult.Success && !authResult.TempBan) return Unauthorized("The provided secret key is invalid. Verify your accounts existence and/or recover the secret key.");
if (!authResult.Success && authResult.TempBan) return Unauthorized("You are temporarily banned. Try connecting again later.");
if (!authResult.Success && authResult.TempBan) return Unauthorized("You are temporarily banned. Try connecting again in 5 minutes.");
if (authResult.Permaban)
{
if (!_mareDbContext.BannedUsers.Any(c => c.CharacterIdentification == charaIdent))
{
_mareDbContext.BannedUsers.Add(new Banned()
{
CharacterIdentification = charaIdent,
Reason = "Autobanned CharacterIdent (" + authResult.Uid + ")"
});
await _mareDbContext.SaveChangesAsync();
}
var lodestone = await _mareDbContext.LodeStoneAuth.Include(a => a.User).FirstOrDefaultAsync(c => c.User.UID == authResult.Uid);
if (lodestone != null)
{
if (!_mareDbContext.BannedRegistrations.Any(c => c.DiscordIdOrLodestoneAuth == lodestone.HashedLodestoneId))
{
_mareDbContext.BannedRegistrations.Add(new BannedRegistrations()
{
DiscordIdOrLodestoneAuth = lodestone.HashedLodestoneId
});
}
if (!_mareDbContext.BannedRegistrations.Any(c => c.DiscordIdOrLodestoneAuth == lodestone.DiscordId.ToString()))
{
_mareDbContext.BannedRegistrations.Add(new BannedRegistrations()
{
DiscordIdOrLodestoneAuth = lodestone.DiscordId.ToString()
});
}
await _mareDbContext.SaveChangesAsync();
}
return Unauthorized("You are permanently banned.");
}
var existingIdent = await _redis.GetAsync<string>("UID:" + authResult.Uid);
if (!string.IsNullOrEmpty(existingIdent)) return Unauthorized("Already logged in to this account.");
if (!string.IsNullOrEmpty(existingIdent)) return Unauthorized("Already logged in to this account. Reconnect in 60 seconds. If you keep seeing this issue, restart your game.");
var token = CreateToken(new List<Claim>()
{
@@ -66,26 +104,6 @@ public class JwtController : Controller
return Content(token.RawData);
}
[AllowAnonymous]
[HttpPost(MareAuth.AuthCreate)]
public async Task<IActionResult> CreateToken(string auth)
{
if (string.IsNullOrEmpty(auth)) return BadRequest("No Authkey");
var ip = _accessor.GetIpAddress();
var authResult = await _secretKeyAuthenticatorService.AuthorizeAsync(ip, auth);
if (!authResult.Success) return Unauthorized("Invalid Authkey");
var token = CreateToken(new List<Claim>()
{
new Claim(MareClaimTypes.Uid, authResult.Uid)
});
return Content(token.RawData);
}
private JwtSecurityToken CreateToken(IEnumerable<Claim> authClaims)
{
var authSigningKey = new SymmetricSecurityKey(Encoding.ASCII.GetBytes(_configuration.GetValue<string>(nameof(MareConfigurationAuthBase.Jwt))));

6
MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs
View File

@@ -36,7 +36,7 @@ public partial class MareHub
request.Hash.AddRange(ownFiles.Select(f => f.Hash));
Metadata headers = new Metadata()
{
{ "Authorization", Context.GetHttpContext().Request.Headers["Authorization"].ToString() }
{ "Authorization", "Bearer " + _generator.Token }
};
_ = await _fileServiceClient.DeleteFilesAsync(request, headers).ConfigureAwait(false);
}
@@ -69,7 +69,7 @@ public partial class MareHub
IsForbidden = forbiddenFile != null,
Hash = file.Hash,
Size = file.Size,
Url = new Uri(baseUrl, file.Hash.ToUpperInvariant()).ToString()
Url = baseUrl.ToString()
});
}
@@ -210,7 +210,7 @@ public partial class MareHub
Metadata headers = new Metadata()
{
{ "Authorization", Context.GetHttpContext().Request.Headers["Authorization"].ToString() }
{ "Authorization", "Bearer " + _generator.Token }
};
var streamingCall = _fileServiceClient.UploadFile(headers);
using var tempFileStream = new FileStream(tempFileName, FileMode.Open, FileAccess.Read);

4
MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs
View File

@@ -29,11 +29,12 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
private readonly int _maxGroupUserCount;
private readonly IConfigurationService<ServerConfiguration> _configurationService;
private readonly IRedisDatabase _redis;
private readonly ServerTokenGenerator _generator;
public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient,
MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService,
IConfigurationService<ServerConfiguration> configuration, IHttpContextAccessor contextAccessor,
IRedisDatabase redisDb)
IRedisDatabase redisDb, ServerTokenGenerator generator)
{
_mareMetrics = mareMetrics;
_fileServiceClient = fileServiceClient;
@@ -46,6 +47,7 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
_maxGroupUserCount = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxGroupUserCount), 100);
_contextAccessor = contextAccessor;
_redis = redisDb;
_generator = generator;
_logger = new MareHubLogger(this, logger);
_dbContext = mareDbContext;
}

1
MareSynchronosServer/MareSynchronosServer/Startup.cs
View File

@@ -78,6 +78,7 @@ public class Startup
services.Configure<MareConfigurationBase>(mareConfig);
services.Configure<MareConfigurationAuthBase>(mareConfig);
services.AddSingleton<ServerTokenGenerator>();
services.AddSingleton<SystemInfoService>();
services.AddSingleton<IUserIdProvider, IdBasedUserIdProvider>();
services.AddHostedService(provider => provider.GetService<SystemInfoService>());

2
MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj
View File

@@ -36,8 +36,10 @@
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="7.0.0" />
<PackageReference Include="Microsoft.IdentityModel.Tokens" Version="6.25.1" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="7.0.0" />
<PackageReference Include="prometheus-net" Version="7.0.0" />
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.25.1" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>

2
MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs
View File

@@ -26,4 +26,6 @@ public class MetricsAPI
public const string GaugeFilesUniquePastHourSize = "mare_files_unique_past_hour_size";
public const string GaugeFilesUniquePastDay = "mare_files_unique_past_day";
public const string GaugeFilesUniquePastDaySize = "mare_files_unique_past_day_size";
public const string GaugeCurrentDownloads = "mare_current_downloads";
public const string GaugeDownloadQueue = "mare_download_queue";
}

510
MareSynchronosServer/MareSynchronosShared/Migrations/20230111092127_IsBannedForAuth.Designer.cs generated Normal file
View File

@@ -0,0 +1,510 @@
// <auto-generated />
using System;
using MareSynchronosShared.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace MareSynchronosServer.Migrations
{
[DbContext(typeof(MareDbContext))]
[Migration("20230111092127_IsBannedForAuth")]
partial class IsBannedForAuth
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "7.0.0")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("MareSynchronosShared.Models.Auth", b =>
{
b.Property<string>("HashedKey")
.HasMaxLength(64)
.HasColumnType("character varying(64)")
.HasColumnName("hashed_key");
b.Property<bool>("IsBanned")
.HasColumnType("boolean")
.HasColumnName("is_banned");
b.Property<string>("UserUID")
.HasColumnType("character varying(10)")
.HasColumnName("user_uid");
b.HasKey("HashedKey")
.HasName("pk_auth");
b.HasIndex("UserUID")
.HasDatabaseName("ix_auth_user_uid");
b.ToTable("auth", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.Banned", b =>
{
b.Property<string>("CharacterIdentification")
.HasMaxLength(100)
.HasColumnType("character varying(100)")
.HasColumnName("character_identification");
b.Property<string>("Reason")
.HasColumnType("text")
.HasColumnName("reason");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.HasKey("CharacterIdentification")
.HasName("pk_banned_users");
b.ToTable("banned_users", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.BannedRegistrations", b =>
{
b.Property<string>("DiscordIdOrLodestoneAuth")
.HasMaxLength(100)
.HasColumnType("character varying(100)")
.HasColumnName("discord_id_or_lodestone_auth");
b.HasKey("DiscordIdOrLodestoneAuth")
.HasName("pk_banned_registrations");
b.ToTable("banned_registrations", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b =>
{
b.Property<string>("UserUID")
.HasMaxLength(10)
.HasColumnType("character varying(10)")
.HasColumnName("user_uid");
b.Property<string>("OtherUserUID")
.HasMaxLength(10)
.HasColumnType("character varying(10)")
.HasColumnName("other_user_uid");
b.Property<bool>("AllowReceivingMessages")
.HasColumnType("boolean")
.HasColumnName("allow_receiving_messages");
b.Property<bool>("IsPaused")
.HasColumnType("boolean")
.HasColumnName("is_paused");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.HasKey("UserUID", "OtherUserUID")
.HasName("pk_client_pairs");
b.HasIndex("OtherUserUID")
.HasDatabaseName("ix_client_pairs_other_user_uid");
b.HasIndex("UserUID")
.HasDatabaseName("ix_client_pairs_user_uid");
b.ToTable("client_pairs", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b =>
{
b.Property<string>("Hash")
.HasMaxLength(40)
.HasColumnType("character varying(40)")
.HasColumnName("hash");
b.Property<long>("Size")
.HasColumnType("bigint")
.HasColumnName("size");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.Property<bool>("Uploaded")
.HasColumnType("boolean")
.HasColumnName("uploaded");
b.Property<string>("UploaderUID")
.HasMaxLength(10)
.HasColumnType("character varying(10)")
.HasColumnName("uploader_uid");
b.HasKey("Hash")
.HasName("pk_file_caches");
b.HasIndex("UploaderUID")
.HasDatabaseName("ix_file_caches_uploader_uid");
b.ToTable("file_caches", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.ForbiddenUploadEntry", b =>
{
b.Property<string>("Hash")
.HasMaxLength(40)
.HasColumnType("character varying(40)")
.HasColumnName("hash");
b.Property<string>("ForbiddenBy")
.HasMaxLength(100)
.HasColumnType("character varying(100)")
.HasColumnName("forbidden_by");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.HasKey("Hash")
.HasName("pk_forbidden_upload_entries");
b.ToTable("forbidden_upload_entries", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.Group", b =>
{
b.Property<string>("GID")
.HasMaxLength(20)
.HasColumnType("character varying(20)")
.HasColumnName("gid");
b.Property<string>("Alias")
.HasMaxLength(50)
.HasColumnType("character varying(50)")
.HasColumnName("alias");
b.Property<string>("HashedPassword")
.HasColumnType("text")
.HasColumnName("hashed_password");
b.Property<bool>("InvitesEnabled")
.HasColumnType("boolean")
.HasColumnName("invites_enabled");
b.Property<string>("OwnerUID")
.HasColumnType("character varying(10)")
.HasColumnName("owner_uid");
b.HasKey("GID")
.HasName("pk_groups");
b.HasIndex("OwnerUID")
.HasDatabaseName("ix_groups_owner_uid");
b.ToTable("groups", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b =>
{
b.Property<string>("GroupGID")
.HasColumnType("character varying(20)")
.HasColumnName("group_gid");
b.Property<string>("BannedUserUID")
.HasColumnType("character varying(10)")
.HasColumnName("banned_user_uid");
b.Property<string>("BannedByUID")
.HasColumnType("character varying(10)")
.HasColumnName("banned_by_uid");
b.Property<DateTime>("BannedOn")
.HasColumnType("timestamp with time zone")
.HasColumnName("banned_on");
b.Property<string>("BannedReason")
.HasColumnType("text")
.HasColumnName("banned_reason");
b.HasKey("GroupGID", "BannedUserUID")
.HasName("pk_group_bans");
b.HasIndex("BannedByUID")
.HasDatabaseName("ix_group_bans_banned_by_uid");
b.HasIndex("BannedUserUID")
.HasDatabaseName("ix_group_bans_banned_user_uid");
b.HasIndex("GroupGID")
.HasDatabaseName("ix_group_bans_group_gid");
b.ToTable("group_bans", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b =>
{
b.Property<string>("GroupGID")
.HasColumnType("character varying(20)")
.HasColumnName("group_gid");
b.Property<string>("GroupUserUID")
.HasColumnType("character varying(10)")
.HasColumnName("group_user_uid");
b.Property<bool>("IsModerator")
.HasColumnType("boolean")
.HasColumnName("is_moderator");
b.Property<bool>("IsPaused")
.HasColumnType("boolean")
.HasColumnName("is_paused");
b.Property<bool>("IsPinned")
.HasColumnType("boolean")
.HasColumnName("is_pinned");
b.HasKey("GroupGID", "GroupUserUID")
.HasName("pk_group_pairs");
b.HasIndex("GroupGID")
.HasDatabaseName("ix_group_pairs_group_gid");
b.HasIndex("GroupUserUID")
.HasDatabaseName("ix_group_pairs_group_user_uid");
b.ToTable("group_pairs", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b =>
{
b.Property<string>("GroupGID")
.HasColumnType("character varying(20)")
.HasColumnName("group_gid");
b.Property<string>("Invite")
.HasMaxLength(64)
.HasColumnType("character varying(64)")
.HasColumnName("invite");
b.Property<DateTime>("ExpirationDate")
.HasColumnType("timestamp with time zone")
.HasColumnName("expiration_date");
b.HasKey("GroupGID", "Invite")
.HasName("pk_group_temp_invites");
b.HasIndex("GroupGID")
.HasDatabaseName("ix_group_temp_invites_group_gid");
b.HasIndex("Invite")
.HasDatabaseName("ix_group_temp_invites_invite");
b.ToTable("group_temp_invites", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b =>
{
b.Property<decimal>("DiscordId")
.ValueGeneratedOnAdd()
.HasColumnType("numeric(20,0)")
.HasColumnName("discord_id");
b.Property<string>("HashedLodestoneId")
.HasMaxLength(100)
.HasColumnType("character varying(100)")
.HasColumnName("hashed_lodestone_id");
b.Property<string>("LodestoneAuthString")
.HasMaxLength(100)
.HasColumnType("character varying(100)")
.HasColumnName("lodestone_auth_string");
b.Property<DateTime?>("StartedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("started_at");
b.Property<string>("UserUID")
.HasColumnType("character varying(10)")
.HasColumnName("user_uid");
b.HasKey("DiscordId")
.HasName("pk_lodestone_auth");
b.HasIndex("UserUID")
.HasDatabaseName("ix_lodestone_auth_user_uid");
b.ToTable("lodestone_auth", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.User", b =>
{
b.Property<string>("UID")
.HasMaxLength(10)
.HasColumnType("character varying(10)")
.HasColumnName("uid");
b.Property<string>("Alias")
.HasMaxLength(15)
.HasColumnType("character varying(15)")
.HasColumnName("alias");
b.Property<bool>("IsAdmin")
.HasColumnType("boolean")
.HasColumnName("is_admin");
b.Property<bool>("IsModerator")
.HasColumnType("boolean")
.HasColumnName("is_moderator");
b.Property<DateTime>("LastLoggedIn")
.HasColumnType("timestamp with time zone")
.HasColumnName("last_logged_in");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.HasKey("UID")
.HasName("pk_users");
b.ToTable("users", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.Auth", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "User")
.WithMany()
.HasForeignKey("UserUID")
.HasConstraintName("fk_auth_users_user_temp_id");
b.Navigation("User");
});
modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "OtherUser")
.WithMany()
.HasForeignKey("OtherUserUID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_client_pairs_users_other_user_temp_id1");
b.HasOne("MareSynchronosShared.Models.User", "User")
.WithMany()
.HasForeignKey("UserUID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_client_pairs_users_user_temp_id2");
b.Navigation("OtherUser");
b.Navigation("User");
});
modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "Uploader")
.WithMany()
.HasForeignKey("UploaderUID")
.HasConstraintName("fk_file_caches_users_uploader_uid");
b.Navigation("Uploader");
});
modelBuilder.Entity("MareSynchronosShared.Models.Group", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "Owner")
.WithMany()
.HasForeignKey("OwnerUID")
.HasConstraintName("fk_groups_users_owner_temp_id7");
b.Navigation("Owner");
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "BannedBy")
.WithMany()
.HasForeignKey("BannedByUID")
.HasConstraintName("fk_group_bans_users_banned_by_temp_id4");
b.HasOne("MareSynchronosShared.Models.User", "BannedUser")
.WithMany()
.HasForeignKey("BannedUserUID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_group_bans_users_banned_user_temp_id5");
b.HasOne("MareSynchronosShared.Models.Group", "Group")
.WithMany()
.HasForeignKey("GroupGID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_group_bans_groups_group_temp_id");
b.Navigation("BannedBy");
b.Navigation("BannedUser");
b.Navigation("Group");
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b =>
{
b.HasOne("MareSynchronosShared.Models.Group", "Group")
.WithMany()
.HasForeignKey("GroupGID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_group_pairs_groups_group_temp_id1");
b.HasOne("MareSynchronosShared.Models.User", "GroupUser")
.WithMany()
.HasForeignKey("GroupUserUID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_group_pairs_users_group_user_temp_id6");
b.Navigation("Group");
b.Navigation("GroupUser");
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b =>
{
b.HasOne("MareSynchronosShared.Models.Group", "Group")
.WithMany()
.HasForeignKey("GroupGID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_group_temp_invites_groups_group_gid");
b.Navigation("Group");
});
modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "User")
.WithMany()
.HasForeignKey("UserUID")
.HasConstraintName("fk_lodestone_auth_users_user_uid");
b.Navigation("User");
});
#pragma warning restore 612, 618
}
}
}

29
MareSynchronosServer/MareSynchronosShared/Migrations/20230111092127_IsBannedForAuth.cs Normal file
View File

@@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace MareSynchronosServer.Migrations
{
/// <inheritdoc />
public partial class IsBannedForAuth : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<bool>(
name: "is_banned",
table: "auth",
type: "boolean",
nullable: false,
defaultValue: false);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "is_banned",
table: "auth");
}
}
}

4
MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs
View File

@@ -29,6 +29,10 @@ namespace MareSynchronosServer.Migrations
.HasColumnType("character varying(64)")
.HasColumnName("hashed_key");
b.Property<bool>("IsBanned")
.HasColumnType("boolean")
.HasColumnName("is_banned");
b.Property<string>("UserUID")
.HasColumnType("character varying(10)")
.HasColumnName("user_uid");

1
MareSynchronosServer/MareSynchronosShared/Models/Auth.cs
View File

@@ -10,4 +10,5 @@ public class Auth
public string UserUID { get; set; }
public User User { get; set; }
public bool IsBanned { get; set; }
}

1
MareSynchronosServer/MareSynchronosShared/Utils/MareClaimTypes.cs
View File

@@ -4,4 +4,5 @@ public static class MareClaimTypes
{
public const string Uid = "uid";
public const string CharaIdent = "character_identification";
public const string Internal = "internal";
}

54
MareSynchronosServer/MareSynchronosShared/Utils/ServerTokenGenerator.cs Normal file
View File

@@ -0,0 +1,54 @@
using MareSynchronosShared.Services;
using Microsoft.IdentityModel.Tokens;
using System.IdentityModel.Tokens.Jwt;
using System.Security.Claims;
using System.Text;
namespace MareSynchronosShared.Utils;
public class ServerTokenGenerator
{
private readonly IConfigurationService<MareConfigurationAuthBase> _configuration;
private Dictionary<string, string> _tokenDictionary { get; set; } = new(StringComparer.Ordinal);
public string Token
{
get
{
var currentJwt = _configuration.GetValue<string>(nameof(MareConfigurationAuthBase.Jwt));
if (_tokenDictionary.TryGetValue(currentJwt, out var token))
{
return token;
}
return GenerateToken();
}
}
public ServerTokenGenerator(IConfigurationService<MareConfigurationAuthBase> configuration)
{
_configuration = configuration;
}
private string GenerateToken()
{
var signingKey = _configuration.GetValue<string>(nameof(MareConfigurationAuthBase.Jwt));
var authSigningKey = new SymmetricSecurityKey(Encoding.ASCII.GetBytes(signingKey));
var token = new SecurityTokenDescriptor()
{
Subject = new ClaimsIdentity(new List<Claim>()
{
new Claim(MareClaimTypes.Uid, _configuration.GetValue<string>(nameof(MareConfigurationBase.ShardName))),
new Claim(MareClaimTypes.Internal, true.ToString())
}),
SigningCredentials = new SigningCredentials(authSigningKey, SecurityAlgorithms.HmacSha256Signature)
};
var handler = new JwtSecurityTokenHandler();
var rawData = handler.CreateJwtSecurityToken(token).RawData;
_tokenDictionary[signingKey] = rawData;
return rawData;
}
}

5
MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs
View File

@@ -12,6 +12,9 @@ public class StaticFilesServerConfiguration : MareConfigurationBase
public string CacheDirectory { get; set; }
public Uri? RemoteCacheSourceUri { get; set; } = null;
public Uri MainServerGrpcAddress { get; set; } = null;
public int DownloadQueueSize { get; set; } = 50;
public int DownloadTimeoutSeconds { get; set; } = 5;
public int DownloadQueueReleaseSeconds { get; set; } = 15;
public override string ToString()
{
StringBuilder sb = new();
@@ -23,6 +26,8 @@ public class StaticFilesServerConfiguration : MareConfigurationBase
sb.AppendLine($"{nameof(UnusedFileRetentionPeriodInDays)} => {UnusedFileRetentionPeriodInDays}");
sb.AppendLine($"{nameof(CacheDirectory)} => {CacheDirectory}");
sb.AppendLine($"{nameof(RemoteCacheSourceUri)} => {RemoteCacheSourceUri}");
sb.AppendLine($"{nameof(DownloadQueueSize)} => {DownloadQueueSize}");
sb.AppendLine($"{nameof(DownloadQueueReleaseSeconds)} => {DownloadQueueReleaseSeconds}");
return sb.ToString();
}
}

42
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/CacheController.cs Normal file
View File

@@ -0,0 +1,42 @@
using MareSynchronos.API;
using MareSynchronosShared.Utils;
using MareSynchronosStaticFilesServer.Services;
using MareSynchronosStaticFilesServer.Utils;
using Microsoft.AspNetCore.Mvc;
namespace MareSynchronosStaticFilesServer.Controllers;
[Route(MareFiles.Cache)]
public class CacheController : ControllerBase
{
private readonly RequestFileStreamResultFactory _requestFileStreamResultFactory;
private readonly CachedFileProvider _cachedFileProvider;
private readonly RequestQueueService _requestQueue;
public CacheController(ILogger<CacheController> logger, RequestFileStreamResultFactory requestFileStreamResultFactory,
CachedFileProvider cachedFileProvider, RequestQueueService requestQueue, ServerTokenGenerator generator) : base(logger, generator)
{
_requestFileStreamResultFactory = requestFileStreamResultFactory;
_cachedFileProvider = cachedFileProvider;
_requestQueue = requestQueue;
}
[HttpGet(MareFiles.Cache_Get)]
public async Task<IActionResult> GetFile(Guid requestId)
{
_logger.LogDebug($"GetFile:{MareUser}:{requestId}");
if (!_requestQueue.IsActiveProcessing(requestId, MareUser, out var request)) return BadRequest();
_requestQueue.ActivateRequest(requestId);
var fs = await _cachedFileProvider.GetAndDownloadFileStream(request.FileId, Authorization);
if (fs == null)
{
_requestQueue.FinishRequest(requestId);
return NotFound();
}
return _requestFileStreamResultFactory.Create(requestId, fs);
}
}

19
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs Normal file
View File

@@ -0,0 +1,19 @@
using MareSynchronosShared.Utils;
using Microsoft.AspNetCore.Mvc;
namespace MareSynchronosStaticFilesServer.Controllers;
public class ControllerBase : Controller
{
protected ILogger _logger;
private readonly ServerTokenGenerator _generator;
public ControllerBase(ILogger logger, ServerTokenGenerator generator)
{
_logger = logger;
_generator = generator;
}
protected string MareUser => HttpContext.User.Claims.First(f => string.Equals(f.Type, MareClaimTypes.Uid, StringComparison.Ordinal)).Value;
protected string Authorization => "Bearer " + _generator.Token;
}

54
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs Normal file
View File

@@ -0,0 +1,54 @@
using MareSynchronos.API;
using MareSynchronosShared.Utils;
using MareSynchronosStaticFilesServer.Services;
using Microsoft.AspNetCore.Mvc;
using System.Text.Json;
namespace MareSynchronosStaticFilesServer.Controllers;
[Route(MareFiles.Request)]
public class RequestController : ControllerBase
{
private readonly CachedFileProvider _cachedFileProvider;
private readonly RequestQueueService _requestQueue;
public RequestController(ILogger<RequestController> logger, CachedFileProvider cachedFileProvider, RequestQueueService requestQueue,
ServerTokenGenerator generator) : base(logger, generator)
{
_cachedFileProvider = cachedFileProvider;
_requestQueue = requestQueue;
}
[HttpPost]
[Route(MareFiles.Request_Enqueue)]
public IActionResult PreRequestFiles([FromBody] List<string> files)
{
foreach (var file in files)
{
_cachedFileProvider.DownloadFileWhenRequired(file, Authorization);
}
return Ok();
}
[HttpGet]
[Route(MareFiles.Request_RequestFile)]
public async Task<IActionResult> RequestFile(string file)
{
Guid g = Guid.NewGuid();
_cachedFileProvider.DownloadFileWhenRequired(file, Authorization);
var queueStatus = await _requestQueue.EnqueueUser(new(g, MareUser, file));
return Ok(JsonSerializer.Serialize(new QueueRequestDto(g, queueStatus)));
}
[HttpGet]
[Route(MareFiles.Request_CheckQueue)]
public IActionResult CheckQueue(Guid requestId)
{
if (_requestQueue.IsActiveProcessing(requestId, MareUser, out _)) return Ok();
if (_requestQueue.StillEnqueued(requestId, MareUser, out int position)) return Conflict(position);
return BadRequest();
}
}

30
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs Normal file
View File

@@ -0,0 +1,30 @@
using MareSynchronos.API;
using MareSynchronosShared.Utils;
using MareSynchronosStaticFilesServer.Services;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
namespace MareSynchronosStaticFilesServer.Controllers;
[Route(MareFiles.ServerFiles)]
public class ServerFilesController : ControllerBase
{
private readonly CachedFileProvider _cachedFileProvider;
public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider, ServerTokenGenerator generator) : base(logger, generator)
{
_cachedFileProvider = cachedFileProvider;
}
[HttpGet(MareFiles.ServerFiles_Get + "/{fileId}")]
[Authorize(Policy = "Internal")]
public async Task<IActionResult> GetFile(string fileId)
{
_logger.LogInformation($"GetFile:{MareUser}:{fileId}");
var fs = _cachedFileProvider.GetLocalFileStream(fileId);
if (fs == null) return NotFound();
return File(fs, "application/octet-stream");
}
}

29
MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs
View File

@@ -1,29 +0,0 @@
using MareSynchronosShared.Utils;
using Microsoft.AspNetCore.Mvc;
namespace MareSynchronosStaticFilesServer;
[Route("/cache")]
public class FilesController : Controller
{
private readonly ILogger<FilesController> _logger;
private readonly CachedFileProvider _cachedFileProvider;
public FilesController(ILogger<FilesController> logger, CachedFileProvider cachedFileProvider)
{
_logger = logger;
_cachedFileProvider = cachedFileProvider;
}
[HttpGet("{fileId}")]
public async Task<IActionResult> GetFile(string fileId)
{
var authedUser = HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, MareClaimTypes.Uid, StringComparison.Ordinal))?.Value ?? "Unknown";
_logger.LogInformation($"GetFile:{authedUser}:{fileId}");
var fs = await _cachedFileProvider.GetFileStream(fileId, Request.Headers["Authorization"]);
if (fs == null) return NotFound();
return File(fs, "application/octet-stream");
}
}

1
MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj
View File

@@ -30,6 +30,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\MareAPI\MareSynchronosAPI\MareSynchronos.API.csproj" />
<ProjectReference Include="..\MareSynchronosShared\MareSynchronosShared.csproj" />
</ItemGroup>

1
MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs
View File

@@ -1,6 +1,5 @@
using MareSynchronosShared.Services;
using MareSynchronosShared.Utils;
using Microsoft.Extensions.Options;
namespace MareSynchronosStaticFilesServer;

47
MareSynchronosServer/MareSynchronosStaticFilesServer/CachedFileProvider.cs → MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs
View File

@@ -1,9 +1,11 @@
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Services;
using Microsoft.Extensions.Options;
using MareSynchronosStaticFilesServer.Utils;
using System.Collections.Concurrent;
using System.Net.Http.Headers;
using MareSynchronos.API;
namespace MareSynchronosStaticFilesServer;
namespace MareSynchronosStaticFilesServer.Services;
public class CachedFileProvider
{
@@ -13,6 +15,7 @@ public class CachedFileProvider
private readonly Uri _remoteCacheSourceUri;
private readonly string _basePath;
private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
private readonly HttpClient _httpClient;
private bool IsMainServer => _remoteCacheSourceUri == null;
public CachedFileProvider(IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics)
@@ -22,16 +25,18 @@ public class CachedFileProvider
_metrics = metrics;
_remoteCacheSourceUri = configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.RemoteCacheSourceUri), null);
_basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
_httpClient = new HttpClient();
}
private async Task DownloadTask(string hash, string auth)
{
// download file from remote
var downloadUrl = new Uri(_remoteCacheSourceUri, hash);
var downloadUrl = MareFiles.ServerFilesGetFullPath(_remoteCacheSourceUri, hash);
_logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl);
using var client = new HttpClient();
client.DefaultRequestHeaders.Add("Authorization", auth);
var response = await client.GetAsync(downloadUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
using var requestMessage = new HttpRequestMessage(HttpMethod.Get, downloadUrl);
requestMessage.Headers.Authorization = new AuthenticationHeaderValue(auth);
var response = await _httpClient.SendAsync(requestMessage).ConfigureAwait(false);
try
{
@@ -58,26 +63,40 @@ public class CachedFileProvider
_metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_basePath, hash).Length);
}
public async Task<FileStream?> GetFileStream(string hash, string auth)
public void DownloadFileWhenRequired(string hash, string auth)
{
var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
if (fi == null && IsMainServer) return null;
if (fi == null && IsMainServer) return;
if (fi == null && !_currentTransfers.ContainsKey(hash))
{
_currentTransfers[hash] = DownloadTask(hash, auth).ContinueWith(r => _currentTransfers.Remove(hash, out _));
}
if (_currentTransfers.TryGetValue(hash, out var downloadTask))
_currentTransfers[hash] = Task.Run(async () =>
{
await downloadTask.ConfigureAwait(false);
await DownloadTask(hash, auth).ConfigureAwait(false);
_currentTransfers.Remove(hash, out _);
});
}
}
fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
public FileStream? GetLocalFileStream(string hash)
{
var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
if (fi == null) return null;
_fileStatisticsService.LogFile(hash, fi.Length);
return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read);
}
public async Task<FileStream?> GetAndDownloadFileStream(string hash, string auth)
{
DownloadFileWhenRequired(hash, auth);
if (_currentTransfers.TryGetValue(hash, out var downloadTask))
{
await downloadTask.ConfigureAwait(false);
}
return GetLocalFileStream(hash);
}
}

9
MareSynchronosServer/MareSynchronosStaticFilesServer/FileCleanupService.cs → MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs
View File

@@ -3,9 +3,10 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Models;
using MareSynchronosShared.Services;
using MareSynchronosStaticFilesServer.Utils;
using Microsoft.EntityFrameworkCore;
namespace MareSynchronosStaticFilesServer;
namespace MareSynchronosStaticFilesServer.Services;
public class FileCleanupService : IHostedService
{
@@ -99,7 +100,7 @@ public class FileCleanupService : IHostedService
if (_isMainServer)
{
FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() };
dbContext.Entry(f).State = Microsoft.EntityFrameworkCore.EntityState.Deleted;
dbContext.Entry(f).State = EntityState.Deleted;
}
}
}
@@ -113,8 +114,8 @@ public class FileCleanupService : IHostedService
{
try
{
var unusedRetention = _configuration.GetValueOrDefault<int>(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
var forcedDeletionAfterHours = _configuration.GetValueOrDefault<int>(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
_logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", unusedRetention);
if (forcedDeletionAfterHours > 0)

2
MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs → MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileStatisticsService.cs
View File

@@ -1,7 +1,7 @@
using MareSynchronosShared.Metrics;
using System.Collections.Concurrent;
namespace MareSynchronosStaticFilesServer;
namespace MareSynchronosStaticFilesServer.Services;
public class FileStatisticsService : IHostedService
{

8
MareSynchronosServer/MareSynchronosStaticFilesServer/GrpcFileService.cs → MareSynchronosServer/MareSynchronosStaticFilesServer/Services/GrpcFileService.cs
View File

@@ -3,10 +3,13 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Protos;
using MareSynchronosShared.Services;
using MareSynchronosStaticFilesServer.Utils;
using Microsoft.AspNetCore.Authorization;
using Microsoft.EntityFrameworkCore;
namespace MareSynchronosStaticFilesServer;
namespace MareSynchronosStaticFilesServer.Services;
[Authorize(Policy = "Internal")]
public class GrpcFileService : FileService.FileServiceBase
{
private readonly string _basePath;
@@ -22,6 +25,7 @@ public class GrpcFileService : FileService.FileServiceBase
_metricsClient = metricsClient;
}
[Authorize(Policy = "Internal")]
public override async Task<Empty> UploadFile(IAsyncStreamReader<UploadFileRequest> requestStream, ServerCallContext context)
{
_ = await requestStream.MoveNext().ConfigureAwait(false);
@@ -31,7 +35,6 @@ public class GrpcFileService : FileService.FileServiceBase
var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == uploadMsg.Hash && f.UploaderUID == uploadMsg.Uploader).ConfigureAwait(false);
try
{
if (file != null)
{
await fileWriter.WriteAsync(uploadMsg.FileData.ToArray()).ConfigureAwait(false);
@@ -71,6 +74,7 @@ public class GrpcFileService : FileService.FileServiceBase
return new Empty();
}
[Authorize(Policy = "Internal")]
public override async Task<Empty> DeleteFiles(DeleteFilesRequest request, ServerCallContext context)
{
foreach (var hash in request.Hash)

137
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs Normal file
View File

@@ -0,0 +1,137 @@
using MareSynchronos.API;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Services;
using MareSynchronosStaticFilesServer.Utils;
using System.Collections.Concurrent;
namespace MareSynchronosStaticFilesServer.Services;
public class RequestQueueService : IHostedService
{
private CancellationTokenSource _queueCts = new();
private readonly UserQueueEntry[] _userQueueRequests;
private readonly ConcurrentQueue<UserRequest> _queue = new();
private readonly MareMetrics _metrics;
private readonly ILogger<RequestQueueService> _logger;
private readonly int _queueExpirationSeconds;
private SemaphoreSlim _queueSemaphore = new(1);
private SemaphoreSlim _queueProcessingSemaphore = new(1);
public RequestQueueService(MareMetrics metrics, IConfigurationService<StaticFilesServerConfiguration> configurationService, ILogger<RequestQueueService> logger)
{
_userQueueRequests = new UserQueueEntry[configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueSize), 50)];
_queueExpirationSeconds = configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadTimeoutSeconds), 5);
_metrics = metrics;
_logger = logger;
}
public async Task<QueueStatus> EnqueueUser(UserRequest request)
{
_logger.LogDebug("Enqueueing req {guid} from {user} for {file}", request.RequestId, request.User, request.FileId);
if (_queueProcessingSemaphore.CurrentCount == 0)
{
_queue.Enqueue(request);
return QueueStatus.Waiting;
}
await _queueSemaphore.WaitAsync().ConfigureAwait(false);
QueueStatus status = QueueStatus.Waiting;
var idx = Array.FindIndex(_userQueueRequests, r => r == null);
if (idx == -1)
{
_queue.Enqueue(request);
status = QueueStatus.Waiting;
}
else
{
DequeueIntoSlot(request, idx);
status = QueueStatus.Ready;
}
_queueSemaphore.Release();
return status;
}
public bool StillEnqueued(Guid request, string user, out int queuePosition)
{
var result = _queue.FirstOrDefault(c => c.RequestId == request && string.Equals(c.User, user, StringComparison.Ordinal));
if (result != null)
{
queuePosition = Array.IndexOf(_queue.ToArray(), result);
return true;
}
queuePosition = -1;
return false;
}
public bool IsActiveProcessing(Guid request, string user, out UserRequest userRequest)
{
var userQueueRequest = _userQueueRequests.Where(u => u != null)
.FirstOrDefault(f => f.UserRequest.RequestId == request && string.Equals(f.UserRequest.User, user, StringComparison.Ordinal));
userRequest = userQueueRequest?.UserRequest ?? null;
return userQueueRequest != null && userRequest != null && userQueueRequest.ExpirationDate > DateTime.UtcNow;
}
public void FinishRequest(Guid request)
{
var req = _userQueueRequests.First(f => f.UserRequest.RequestId == request);
var idx = Array.IndexOf(_userQueueRequests, req);
_logger.LogDebug("Finishing Request {guid}, clearing slot {idx}", request, idx);
_userQueueRequests[idx] = null;
}
public void ActivateRequest(Guid request)
{
_logger.LogDebug("Activating request {guid}", request);
_userQueueRequests.First(f => f.UserRequest.RequestId == request).IsActive = true;
}
private async Task ProcessRequestQueue(CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
await _queueProcessingSemaphore.WaitAsync(ct).ConfigureAwait(false);
await _queueSemaphore.WaitAsync(ct).ConfigureAwait(false);
for (int i = 0; i < _userQueueRequests.Length; i++)
{
if (_userQueueRequests[i] != null && !_userQueueRequests[i].IsActive && _userQueueRequests[i].ExpirationDate < DateTime.UtcNow) _userQueueRequests[i] = null;
if (_userQueueRequests[i] == null)
{
if (_queue.TryDequeue(out var request))
{
DequeueIntoSlot(request, i);
}
}
if (!_queue.Any()) break;
}
_queueProcessingSemaphore.Release();
_queueSemaphore.Release();
_metrics.SetGaugeTo(MetricsAPI.GaugeDownloadQueue, _queue.Count);
await Task.Delay(250).ConfigureAwait(false);
}
}
private void DequeueIntoSlot(UserRequest userRequest, int slot)
{
_logger.LogDebug("Dequeueing {req} into {i}: {user} with {file}", userRequest.RequestId, slot, userRequest.User, userRequest.FileId);
_userQueueRequests[slot] = new(userRequest, DateTime.UtcNow.AddSeconds(_queueExpirationSeconds));
}
public Task StartAsync(CancellationToken cancellationToken)
{
_ = Task.Run(() => ProcessRequestQueue(_queueCts.Token));
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken)
{
_queueCts.Cancel();
return Task.CompletedTask;
}
}

21
MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs
View File

@@ -5,6 +5,8 @@ using MareSynchronosShared.Metrics;
using MareSynchronosShared.Protos;
using MareSynchronosShared.Services;
using MareSynchronosShared.Utils;
using MareSynchronosStaticFilesServer.Services;
using MareSynchronosStaticFilesServer.Utils;
using Microsoft.AspNetCore.Authentication.JwtBearer;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Server.Kestrel.Core;
@@ -41,8 +43,6 @@ public class Startup
var mareConfig = Configuration.GetRequiredSection("MareSynchronos");
services.AddControllers();
services.AddSingleton(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string>
{
}, new List<string>
@@ -52,10 +52,13 @@ public class Startup
MetricsAPI.GaugeFilesUniquePastDay,
MetricsAPI.GaugeFilesUniquePastDaySize,
MetricsAPI.GaugeFilesUniquePastHour,
MetricsAPI.GaugeFilesUniquePastHourSize
MetricsAPI.GaugeFilesUniquePastHourSize,
MetricsAPI.GaugeCurrentDownloads,
MetricsAPI.GaugeDownloadQueue,
}));
services.AddSingleton<CachedFileProvider>();
services.AddSingleton<FileStatisticsService>();
services.AddSingleton<RequestFileStreamResultFactory>();
services.AddHostedService(m => m.GetService<FileStatisticsService>());
services.AddHostedService<FileCleanupService>();
@@ -119,7 +122,11 @@ public class Startup
o.DefaultScheme = JwtBearerDefaults.AuthenticationScheme;
}).AddJwtBearer();
services.AddAuthorization(options => options.FallbackPolicy = new AuthorizationPolicyBuilder().RequireAuthenticatedUser().Build());
services.AddAuthorization(options =>
{
options.FallbackPolicy = new AuthorizationPolicyBuilder().RequireAuthenticatedUser().Build();
options.AddPolicy("Internal", new AuthorizationPolicyBuilder().RequireClaim(MareClaimTypes.Internal, "true").Build());
});
if (_isMain)
{
@@ -148,9 +155,15 @@ public class Startup
p.GetRequiredService<GrpcClientFactory>(), "MainServer")
);
services.AddSingleton<ServerTokenGenerator>();
services.AddSingleton<RequestQueueService>();
services.AddHostedService(p => p.GetService<RequestQueueService>());
services.AddControllers();
services.AddHostedService(p => (MareConfigurationServiceClient<MareConfigurationAuthBase>)p.GetService<IConfigurationService<MareConfigurationAuthBase>>());
services.AddHealthChecks();
services.AddControllers();
}
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)

4
MareSynchronosServer/MareSynchronosStaticFilesServer/FilePathUtil.cs → MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/FilePathUtil.cs
View File

@@ -1,8 +1,8 @@
namespace MareSynchronosStaticFilesServer;
namespace MareSynchronosStaticFilesServer.Utils;
public static class FilePathUtil
{
public static FileInfo? GetFileInfoForHash(string basePath, string hash)
public static FileInfo GetFileInfoForHash(string basePath, string hash)
{
FileInfo fi = new(Path.Combine(basePath, hash[0].ToString(), hash));
if (!fi.Exists)

59
MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs Normal file
View File

@@ -0,0 +1,59 @@
using MareSynchronosShared.Metrics;
using MareSynchronosStaticFilesServer.Services;
using Microsoft.AspNetCore.Mvc;
namespace MareSynchronosStaticFilesServer.Utils;
public class RequestFileStreamResult : FileStreamResult
{
private readonly Guid _requestId;
private readonly RequestQueueService _requestQueueService;
private readonly MareMetrics _mareMetrics;
private readonly CancellationTokenSource _releaseCts = new();
private bool _releasedSlot = false;
public RequestFileStreamResult(Guid requestId, int secondsUntilRelease, RequestQueueService requestQueueService,
MareMetrics mareMetrics, Stream fileStream, string contentType) : base(fileStream, contentType)
{
_requestId = requestId;
_requestQueueService = requestQueueService;
_mareMetrics = mareMetrics;
_mareMetrics.IncGauge(MetricsAPI.GaugeCurrentDownloads);
// forcefully release slot after secondsUntilRelease
Task.Run(async () =>
{
try
{
await Task.Delay(TimeSpan.FromSeconds(secondsUntilRelease), _releaseCts.Token).ConfigureAwait(false);
_requestQueueService.FinishRequest(_requestId);
_releasedSlot = true;
}
catch { }
});
}
public override void ExecuteResult(ActionContext context)
{
base.ExecuteResult(context);
_releaseCts.Cancel();
if (!_releasedSlot)
_requestQueueService.FinishRequest(_requestId);
_mareMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads);
}
public override async Task ExecuteResultAsync(ActionContext context)
{
await base.ExecuteResultAsync(context).ConfigureAwait(false);
_releaseCts.Cancel();
if (!_releasedSlot)
_requestQueueService.FinishRequest(_requestId);
_mareMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads);
}
}

25
MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResultFactory.cs Normal file
View File

@@ -0,0 +1,25 @@
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Services;
using MareSynchronosStaticFilesServer.Services;
namespace MareSynchronosStaticFilesServer.Utils;
public class RequestFileStreamResultFactory
{
private readonly MareMetrics _metrics;
private readonly RequestQueueService _requestQueueService;
private readonly IConfigurationService<StaticFilesServerConfiguration> _configurationService;
public RequestFileStreamResultFactory(MareMetrics metrics, RequestQueueService requestQueueService, IConfigurationService<StaticFilesServerConfiguration> configurationService)
{
_metrics = metrics;
_requestQueueService = requestQueueService;
_configurationService = configurationService;
}
public RequestFileStreamResult Create(Guid requestId, FileStream fs)
{
return new RequestFileStreamResult(requestId, _configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueReleaseSeconds), 15),
_requestQueueService, _metrics, fs, "application/octet-stream");
}
}

6
MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserQueueEntry.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace MareSynchronosStaticFilesServer.Utils;
public record UserQueueEntry(UserRequest UserRequest, DateTime ExpirationDate)
{
public bool IsActive { get; set; } = false;
}

3
MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserRequest.cs Normal file
View File

@@ -0,0 +1,3 @@
namespace MareSynchronosStaticFilesServer.Utils;
public record UserRequest(Guid RequestId, string User, string FileId);