t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'root/main'

Browse Source
This commit is contained in:
Stefan Berg
2022-08-15 02:04:25 +02:00
parent c95d0e1d09 a33cf2b2d1
commit d72ed1ba3b
9 changed files with 78 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthenticationHandler.cs
View File

@@ -18,12 +18,23 @@ using Microsoft.Extensions.Options;
namespace MareSynchronosServer.Authentication
{
public class FailedAuthorization
public class FailedAuthorization : IDisposable
{
private int failedAttempts = 1;
public int FailedAttempts => failedAttempts;
public Task ResetTask { get; set; }
public CancellationTokenSource ResetCts { get; set; } = new();
public void Dispose()
{
try
{
ResetCts?.Cancel();
ResetCts?.Dispose();
}
catch { }
}
public void IncreaseFailedAttempts()
{
Interlocked.Increment(ref failedAttempts);
@@ -82,13 +93,15 @@ namespace MareSynchronosServer.Authentication
if (failedAuth.FailedAttempts > failedAttemptsForTempBan)
{
failedAuth.ResetCts.Cancel();
failedAuth.ResetCts.Dispose();
failedAuth.ResetCts = new CancellationTokenSource();
var token = failedAuth.ResetCts.Token;
failedAuth.ResetTask = Task.Run(async () =>
{
await Task.Delay(TimeSpan.FromMinutes(tempBanMinutes), token);
if (token.IsCancellationRequested) return;
FailedAuthorizations.Remove(ip, out _);
FailedAuthorizations.Remove(ip, out var fauth);
fauth.Dispose();
}, token);
Logger.LogWarning("TempBan " + ip + " for authorization spam");
return AuthenticateResult.Fail("Failed Authorization");

2
MareSynchronosServer/MareSynchronosServer/CleanupService.cs
View File

@@ -51,7 +51,7 @@ namespace MareSynchronosServer
try
{
using var scope = _services.CreateScope();
var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays));

28
MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs
View File

@@ -30,8 +30,8 @@ namespace MareSynchronosServer.Discord
private string authToken = string.Empty;
DiscordSocketClient discordClient;
ConcurrentDictionary<ulong, string> DiscordLodestoneMapping = new();
private Timer _timer;
private CancellationTokenSource verificationTaskCts;
private CancellationTokenSource updateStatusCts;
private readonly string[] LodestoneServers = new[] { "eu", "na", "jp", "fr", "de" };
private readonly ConcurrentQueue<SocketSlashCommand> verificationQueue = new();
@@ -248,7 +248,7 @@ namespace MareSynchronosServer.Discord
var hashedLodestoneId = BitConverter.ToString(sha256.ComputeHash(Encoding.UTF8.GetBytes(lodestoneId.ToString()))).Replace("-", "");
var db = scope.ServiceProvider.GetService<MareDbContext>();
using var db = scope.ServiceProvider.GetService<MareDbContext>();
// check if discord id or lodestone id is banned
if (db.BannedRegistrations.Any(a => a.DiscordIdOrLodestoneAuth == arg.User.Id.ToString() || a.DiscordIdOrLodestoneAuth == hashedLodestoneId))
@@ -374,9 +374,8 @@ namespace MareSynchronosServer.Discord
discordClient.SlashCommandExecuted += DiscordClient_SlashCommandExecuted;
discordClient.ModalSubmitted += DiscordClient_ModalSubmitted;
_timer = new Timer(UpdateStatus, null, TimeSpan.Zero, TimeSpan.FromSeconds(15));
ProcessQueueWork();
_ = ProcessQueueWork();
_ = UpdateStatusAsync();
}
}
@@ -385,7 +384,6 @@ namespace MareSynchronosServer.Discord
verificationTaskCts = new CancellationTokenSource();
while (!verificationTaskCts.IsCancellationRequested)
{
if (verificationQueue.TryDequeue(out var queueitem))
{
try
@@ -405,20 +403,26 @@ namespace MareSynchronosServer.Discord
}
}
private void UpdateStatus(object state)
private async Task UpdateStatusAsync()
{
using var scope = services.CreateScope();
var db = scope.ServiceProvider.GetService<MareDbContext>();
updateStatusCts = new();
while (!updateStatusCts.IsCancellationRequested)
{
using var scope = services.CreateScope();
using var db = scope.ServiceProvider.GetService<MareDbContext>();
var users = db.Users.Count(c => c.CharacterIdentification != null);
var users = db.Users.Count(c => c.CharacterIdentification != null);
discordClient.SetActivityAsync(new Game("Mare for " + users + " Users"));
await discordClient.SetActivityAsync(new Game("Mare for " + users + " Users"));
await Task.Delay(TimeSpan.FromSeconds(15));
}
}
public async Task StopAsync(CancellationToken cancellationToken)
{
_timer?.Change(Timeout.Infinite, 0);
verificationTaskCts?.Cancel();
updateStatusCts?.Cancel();
await discordClient.LogoutAsync();
await discordClient.StopAsync();

26
MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs
View File

@@ -158,18 +158,27 @@ namespace MareSynchronosServer.Hubs
if (relatedFile == null) return;
var forbiddenFile = _dbContext.ForbiddenUploadEntries.SingleOrDefault(f => f.Hash == hash);
if (forbiddenFile != null) return;
var uploadedFile = new List<byte>();
var finalFileName = Path.Combine(BasePath, hash);
var tempFileName = finalFileName + ".tmp";
using var fileStream = new FileStream(tempFileName, FileMode.OpenOrCreate);
long length = 0;
try
{
await foreach (var chunk in fileContent)
{
uploadedFile.AddRange(chunk);
length += chunk.Length;
await fileStream.WriteAsync(chunk);
}
await fileStream.FlushAsync();
await fileStream.DisposeAsync();
}
catch
{
try
{
await fileStream.FlushAsync();
await fileStream.DisposeAsync();
_dbContext.Files.Remove(relatedFile);
await _dbContext.SaveChangesAsync();
}
@@ -177,15 +186,19 @@ namespace MareSynchronosServer.Hubs
{
// already removed
}
finally
{
File.Delete(tempFileName);
}
return;
}
_logger.LogInformation("User " + AuthenticatedUserId + " upload finished: " + hash + ", size: " + uploadedFile.Count);
_logger.LogInformation("User " + AuthenticatedUserId + " upload finished: " + hash + ", size: " + length);
try
{
var decodedFile = LZ4.LZ4Codec.Unwrap(uploadedFile.ToArray());
var decodedFile = LZ4.LZ4Codec.Unwrap(await File.ReadAllBytesAsync(tempFileName));
using var sha1 = SHA1.Create();
using var ms = new MemoryStream(decodedFile);
var computedHash = await sha1.ComputeHashAsync(ms);
@@ -199,12 +212,12 @@ namespace MareSynchronosServer.Hubs
return;
}
await File.WriteAllBytesAsync(Path.Combine(BasePath, hash), uploadedFile.ToArray());
File.Move(tempFileName, finalFileName, true);
relatedFile = _dbContext.Files.Single(f => f.Hash == hash);
relatedFile.Uploaded = true;
MareMetrics.FilesTotal.Inc();
MareMetrics.FilesTotalSize.Inc(uploadedFile.Count);
MareMetrics.FilesTotalSize.Inc(length);
await _dbContext.SaveChangesAsync();
_logger.LogInformation("File " + hash + " added to DB");
@@ -215,7 +228,6 @@ namespace MareSynchronosServer.Hubs
_dbContext.Remove(relatedFile);
await _dbContext.SaveChangesAsync();
}
}
}
}

12
MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj
View File

@@ -11,18 +11,18 @@
<PackageReference Include="Bazinga.AspNetCore.Authentication.Basic" Version="2.0.1" />
<PackageReference Include="Discord.Net" Version="3.7.2" />
<PackageReference Include="EFCore.NamingConventions" Version="6.0.0" />
<PackageReference Include="Karambolo.Extensions.Logging.File" Version="3.3.0" />
<PackageReference Include="Karambolo.Extensions.Logging.File" Version="3.3.1" />
<PackageReference Include="lz4net" Version="1.0.15.93" />
<PackageReference Include="Microsoft.AspNetCore.Diagnostics.EntityFrameworkCore" Version="6.0.6" />
<PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="6.0.6" />
<PackageReference Include="Microsoft.AspNetCore.Identity.UI" Version="6.0.6" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.6">
<PackageReference Include="Microsoft.AspNetCore.Diagnostics.EntityFrameworkCore" Version="6.0.8" />
<PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="6.0.8" />
<PackageReference Include="Microsoft.AspNetCore.Identity.UI" Version="6.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.8">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="6.0.0" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="6.0.5" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="6.0.6" />
<PackageReference Include="prometheus-net" Version="6.0.0" />
<PackageReference Include="prometheus-net.AspNetCore" Version="6.0.0" />
</ItemGroup>

2
MareSynchronosServer/MareSynchronosServer/Metrics/MareMetrics.cs
View File

@@ -14,6 +14,8 @@ namespace MareSynchronosServer.Metrics
new(Prometheus.Metrics.CreateGauge("mare_unauthorized_connections", "Unauthorized Connections"));
public static readonly LockedProxyGauge AuthorizedConnections =
new(Prometheus.Metrics.CreateGauge("mare_authorized_connections", "Authorized Connections"));
public static readonly LockedProxyGauge AvailableWorkerThreads = new(Prometheus.Metrics.CreateGauge("mare_available_threadpool", "Available Threadpool Workers"));
public static readonly LockedProxyGauge AvailableIOWorkerThreads = new(Prometheus.Metrics.CreateGauge("mare_available_threadpool_io", "Available Threadpool IO Workers"));
public static readonly LockedProxyGauge UsersRegistered = new(Prometheus.Metrics.CreateGauge("mare_users_registered", "Total Registrations"));

7
MareSynchronosServer/MareSynchronosServer/Program.cs
View File

@@ -20,9 +20,6 @@ namespace MareSynchronosServer
var hostBuilder = CreateHostBuilder(args);
var host = hostBuilder.Build();
System.Threading.ThreadPool.GetMaxThreads(out int worker, out int io);
Console.WriteLine($"Before: Worker threads {worker}, IO threads {io}");
using (var scope = host.Services.CreateScope())
{
var services = scope.ServiceProvider;
@@ -42,10 +39,6 @@ namespace MareSynchronosServer
context.RemoveRange(looseFiles);
context.SaveChanges();
System.Threading.ThreadPool.SetMaxThreads(worker, context.Users.Count() * 5);
System.Threading.ThreadPool.GetMaxThreads(out int workerNew, out int ioNew);
Console.WriteLine($"After: Worker threads {workerNew}, IO threads {ioNew}");
MareMetrics.InitializeMetrics(context, services.GetRequiredService<IConfiguration>());
}

6
MareSynchronosServer/MareSynchronosServer/Startup.cs
View File

@@ -38,18 +38,17 @@ namespace MareSynchronosServer
{
services.AddHttpContextAccessor();
services.AddMemoryCache();
services.Configure<IpRateLimitOptions>(Configuration.GetSection("IpRateLimiting"));
services.Configure<IpRateLimitPolicies>(Configuration.GetSection("IpRateLimitPolicies"));
services.AddMemoryCache();
services.AddInMemoryRateLimiting();
services.AddSingleton<SystemInfoService, SystemInfoService>();
services.AddSingleton<IUserIdProvider, IdBasedUserIdProvider>();
services.AddTransient(_ => Configuration);
services.AddDbContextPool<MareDbContext>(options =>
services.AddDbContext<MareDbContext>(options =>
{
options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder =>
{
@@ -61,7 +60,6 @@ namespace MareSynchronosServer
services.AddHostedService(provider => provider.GetService<SystemInfoService>());
services.AddHostedService<DiscordBot>();
services.AddDatabaseDeveloperPageExceptionFilter();
services.AddAuthentication(options =>
{
options.DefaultScheme = SecretKeyAuthenticationHandler.AuthScheme;

19
MareSynchronosServer/MareSynchronosServer/SystemInfoService.cs
View File

@@ -1,10 +1,13 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MareSynchronos.API;
using MareSynchronosServer.Data;
using MareSynchronosServer.Hubs;
using MareSynchronosServer.Metrics;
using Microsoft.AspNetCore.SignalR;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
@@ -12,13 +15,15 @@ namespace MareSynchronosServer;
public class SystemInfoService : IHostedService, IDisposable
{
private readonly IServiceProvider _services;
private readonly ILogger<SystemInfoService> _logger;
private readonly IHubContext<MareHub> _hubContext;
private Timer _timer;
public SystemInfoDto SystemInfoDto { get; private set; } = new();
public SystemInfoService(ILogger<SystemInfoService> logger, IHubContext<MareHub> hubContext)
public SystemInfoService(IServiceProvider services, ILogger<SystemInfoService> logger, IHubContext<MareHub> hubContext)
{
_services = services;
_logger = logger;
_hubContext = hubContext;
}
@@ -34,6 +39,16 @@ public class SystemInfoService : IHostedService, IDisposable
private void PushSystemInfo(object state)
{
ThreadPool.GetAvailableThreads(out int workerThreads, out int ioThreads);
_logger.LogInformation($"ThreadPool: {workerThreads} workers available, {ioThreads} IO workers available");
MareMetrics.AvailableWorkerThreads.Set(workerThreads);
MareMetrics.AvailableIOWorkerThreads.Set(ioThreads);
using var scope = _services.CreateScope();
using var db = scope.ServiceProvider.GetService<MareDbContext>();
var users = db.Users.Count(c => c.CharacterIdentification != null);
SystemInfoDto = new SystemInfoDto()
{
CacheUsage = 0,
@@ -41,7 +56,7 @@ public class SystemInfoService : IHostedService, IDisposable
RAMUsage = 0,
NetworkIn = 0,
NetworkOut = 0,
OnlineUsers = (int)MareMetrics.AuthorizedConnections.Value,
OnlineUsers = users,
UploadedFiles = 0
};