From ef70dd58828c61f43ae7f79c3673539ae026f6a2 Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Mon, 8 Aug 2022 11:27:57 +0200 Subject: [PATCH 01/10] add threadpool info to statistics, change systeminfo to grab online users from db instead of metrics --- .../Metrics/MareMetrics.cs | 2 ++ .../MareSynchronosServer/SystemInfoService.cs | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/MareSynchronosServer/MareSynchronosServer/Metrics/MareMetrics.cs b/MareSynchronosServer/MareSynchronosServer/Metrics/MareMetrics.cs index 4150c9f..89c801b 100644 --- a/MareSynchronosServer/MareSynchronosServer/Metrics/MareMetrics.cs +++ b/MareSynchronosServer/MareSynchronosServer/Metrics/MareMetrics.cs @@ -14,6 +14,8 @@ namespace MareSynchronosServer.Metrics new(Prometheus.Metrics.CreateGauge("mare_unauthorized_connections", "Unauthorized Connections")); public static readonly LockedProxyGauge AuthorizedConnections = new(Prometheus.Metrics.CreateGauge("mare_authorized_connections", "Authorized Connections")); + public static readonly LockedProxyGauge AvailableWorkerThreads = new(Prometheus.Metrics.CreateGauge("mare_available_threadpool", "Available Threadpool Workers")); + public static readonly LockedProxyGauge AvailableIOWorkerThreads = new(Prometheus.Metrics.CreateGauge("mare_available_threadpool_io", "Available Threadpool IO Workers")); public static readonly LockedProxyGauge UsersRegistered = new(Prometheus.Metrics.CreateGauge("mare_users_registered", "Total Registrations")); diff --git a/MareSynchronosServer/MareSynchronosServer/SystemInfoService.cs b/MareSynchronosServer/MareSynchronosServer/SystemInfoService.cs index bce65d7..4e8f7e2 100644 --- a/MareSynchronosServer/MareSynchronosServer/SystemInfoService.cs +++ b/MareSynchronosServer/MareSynchronosServer/SystemInfoService.cs @@ -1,10 +1,13 @@ using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; using MareSynchronos.API; +using MareSynchronosServer.Data; using MareSynchronosServer.Hubs; using MareSynchronosServer.Metrics; using Microsoft.AspNetCore.SignalR; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -12,13 +15,15 @@ namespace MareSynchronosServer; public class SystemInfoService : IHostedService, IDisposable { + private readonly IServiceProvider _services; private readonly ILogger _logger; private readonly IHubContext _hubContext; private Timer _timer; public SystemInfoDto SystemInfoDto { get; private set; } = new(); - public SystemInfoService(ILogger logger, IHubContext hubContext) + public SystemInfoService(IServiceProvider services, ILogger logger, IHubContext hubContext) { + _services = services; _logger = logger; _hubContext = hubContext; } @@ -34,6 +39,16 @@ public class SystemInfoService : IHostedService, IDisposable private void PushSystemInfo(object state) { + ThreadPool.GetAvailableThreads(out int workerThreads, out int ioThreads); + _logger.LogInformation($"ThreadPool: {workerThreads} workers available, {ioThreads} IO workers available"); + MareMetrics.AvailableWorkerThreads.Set(workerThreads); + MareMetrics.AvailableIOWorkerThreads.Set(ioThreads); + + using var scope = _services.CreateScope(); + var db = scope.ServiceProvider.GetService(); + + var users = db.Users.Count(c => c.CharacterIdentification != null); + SystemInfoDto = new SystemInfoDto() { CacheUsage = 0, @@ -41,7 +56,7 @@ public class SystemInfoService : IHostedService, IDisposable RAMUsage = 0, NetworkIn = 0, NetworkOut = 0, - OnlineUsers = (int)MareMetrics.AuthorizedConnections.Value, + OnlineUsers = users, UploadedFiles = 0 }; From ffe24b0e716ec0282d8fc60c8f150de5269de5ad Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Wed, 10 Aug 2022 23:24:10 +0200 Subject: [PATCH 02/10] revert to defaults worker count --- MareSynchronosServer/MareSynchronosServer/Program.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/MareSynchronosServer/MareSynchronosServer/Program.cs b/MareSynchronosServer/MareSynchronosServer/Program.cs index 904dc78..3cf2d65 100644 --- a/MareSynchronosServer/MareSynchronosServer/Program.cs +++ b/MareSynchronosServer/MareSynchronosServer/Program.cs @@ -20,9 +20,6 @@ namespace MareSynchronosServer var hostBuilder = CreateHostBuilder(args); var host = hostBuilder.Build(); - System.Threading.ThreadPool.GetMaxThreads(out int worker, out int io); - Console.WriteLine($"Before: Worker threads {worker}, IO threads {io}"); - using (var scope = host.Services.CreateScope()) { var services = scope.ServiceProvider; @@ -42,10 +39,6 @@ namespace MareSynchronosServer context.RemoveRange(looseFiles); context.SaveChanges(); - System.Threading.ThreadPool.SetMaxThreads(worker, context.Users.Count() * 5); - System.Threading.ThreadPool.GetMaxThreads(out int workerNew, out int ioNew); - Console.WriteLine($"After: Worker threads {workerNew}, IO threads {ioNew}"); - MareMetrics.InitializeMetrics(context, services.GetRequiredService()); } From f29edbd1c34a1a4e3fb58a98c451916cd17056c8 Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Wed, 10 Aug 2022 23:39:55 +0200 Subject: [PATCH 03/10] dispose db context from services --- MareSynchronosServer/MareSynchronosServer/CleanupService.cs | 2 +- .../MareSynchronosServer/Discord/DiscordBot.cs | 4 ++-- .../MareSynchronosServer/SystemInfoService.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/MareSynchronosServer/MareSynchronosServer/CleanupService.cs b/MareSynchronosServer/MareSynchronosServer/CleanupService.cs index 6b71283..90358e3 100644 --- a/MareSynchronosServer/MareSynchronosServer/CleanupService.cs +++ b/MareSynchronosServer/MareSynchronosServer/CleanupService.cs @@ -51,7 +51,7 @@ namespace MareSynchronosServer try { using var scope = _services.CreateScope(); - var dbContext = scope.ServiceProvider.GetService()!; + using var dbContext = scope.ServiceProvider.GetService()!; var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays)); diff --git a/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs b/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs index f0bf51e..4a8b517 100644 --- a/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs +++ b/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs @@ -248,7 +248,7 @@ namespace MareSynchronosServer.Discord var hashedLodestoneId = BitConverter.ToString(sha256.ComputeHash(Encoding.UTF8.GetBytes(lodestoneId.ToString()))).Replace("-", ""); - var db = scope.ServiceProvider.GetService(); + using var db = scope.ServiceProvider.GetService(); // check if discord id or lodestone id is banned if (db.BannedRegistrations.Any(a => a.DiscordIdOrLodestoneAuth == arg.User.Id.ToString() || a.DiscordIdOrLodestoneAuth == hashedLodestoneId)) @@ -408,7 +408,7 @@ namespace MareSynchronosServer.Discord private void UpdateStatus(object state) { using var scope = services.CreateScope(); - var db = scope.ServiceProvider.GetService(); + using var db = scope.ServiceProvider.GetService(); var users = db.Users.Count(c => c.CharacterIdentification != null); diff --git a/MareSynchronosServer/MareSynchronosServer/SystemInfoService.cs b/MareSynchronosServer/MareSynchronosServer/SystemInfoService.cs index 4e8f7e2..f8a06cf 100644 --- a/MareSynchronosServer/MareSynchronosServer/SystemInfoService.cs +++ b/MareSynchronosServer/MareSynchronosServer/SystemInfoService.cs @@ -45,7 +45,7 @@ public class SystemInfoService : IHostedService, IDisposable MareMetrics.AvailableIOWorkerThreads.Set(ioThreads); using var scope = _services.CreateScope(); - var db = scope.ServiceProvider.GetService(); + using var db = scope.ServiceProvider.GetService(); var users = db.Users.Count(c => c.CharacterIdentification != null); From a5fe927cd5e0fc4879cbbc21168964ba625ce776 Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Thu, 11 Aug 2022 10:50:23 +0200 Subject: [PATCH 04/10] disable context pooling --- .../MareSynchronosServer/Discord/DiscordBot.cs | 3 +-- MareSynchronosServer/MareSynchronosServer/Startup.cs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs b/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs index 4a8b517..2672b12 100644 --- a/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs +++ b/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs @@ -376,7 +376,7 @@ namespace MareSynchronosServer.Discord _timer = new Timer(UpdateStatus, null, TimeSpan.Zero, TimeSpan.FromSeconds(15)); - ProcessQueueWork(); + _ = ProcessQueueWork(); } } @@ -385,7 +385,6 @@ namespace MareSynchronosServer.Discord verificationTaskCts = new CancellationTokenSource(); while (!verificationTaskCts.IsCancellationRequested) { - if (verificationQueue.TryDequeue(out var queueitem)) { try diff --git a/MareSynchronosServer/MareSynchronosServer/Startup.cs b/MareSynchronosServer/MareSynchronosServer/Startup.cs index 7d8e199..1447493 100644 --- a/MareSynchronosServer/MareSynchronosServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosServer/Startup.cs @@ -49,7 +49,7 @@ namespace MareSynchronosServer services.AddSingleton(); services.AddTransient(_ => Configuration); - services.AddDbContextPool(options => + services.AddDbContext(options => { options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder => { From d4c8a15f502f13ae1f2597e8ca4a2f65342caec7 Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Fri, 12 Aug 2022 00:26:27 +0200 Subject: [PATCH 05/10] update status via task, update nuget packages --- .../Discord/DiscordBot.cs | 21 ++++++++++++------- .../MareSynchronosServer.csproj | 12 +++++------ .../MareSynchronosServer/Startup.cs | 3 --- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs b/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs index 2672b12..a7db726 100644 --- a/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs +++ b/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs @@ -30,8 +30,8 @@ namespace MareSynchronosServer.Discord private string authToken = string.Empty; DiscordSocketClient discordClient; ConcurrentDictionary DiscordLodestoneMapping = new(); - private Timer _timer; private CancellationTokenSource verificationTaskCts; + private CancellationTokenSource updateStatusCts; private readonly string[] LodestoneServers = new[] { "eu", "na", "jp", "fr", "de" }; private readonly ConcurrentQueue verificationQueue = new(); @@ -374,9 +374,8 @@ namespace MareSynchronosServer.Discord discordClient.SlashCommandExecuted += DiscordClient_SlashCommandExecuted; discordClient.ModalSubmitted += DiscordClient_ModalSubmitted; - _timer = new Timer(UpdateStatus, null, TimeSpan.Zero, TimeSpan.FromSeconds(15)); - _ = ProcessQueueWork(); + _ = UpdateStatusAsync(); } } @@ -404,14 +403,20 @@ namespace MareSynchronosServer.Discord } } - private void UpdateStatus(object state) + private async Task UpdateStatusAsync() { - using var scope = services.CreateScope(); - using var db = scope.ServiceProvider.GetService(); + updateStatusCts = new(); + while (!updateStatusCts.IsCancellationRequested) + { + using var scope = services.CreateScope(); + using var db = scope.ServiceProvider.GetService(); - var users = db.Users.Count(c => c.CharacterIdentification != null); + var users = db.Users.Count(c => c.CharacterIdentification != null); - discordClient.SetActivityAsync(new Game("Mare for " + users + " Users")); + await discordClient.SetActivityAsync(new Game("Mare for " + users + " Users")); + + await Task.Delay(TimeSpan.FromSeconds(15)); + } } public async Task StopAsync(CancellationToken cancellationToken) diff --git a/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj b/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj index 9e45326..1b774cf 100644 --- a/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj +++ b/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj @@ -11,18 +11,18 @@ - + - - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/MareSynchronosServer/MareSynchronosServer/Startup.cs b/MareSynchronosServer/MareSynchronosServer/Startup.cs index 1447493..fcf2765 100644 --- a/MareSynchronosServer/MareSynchronosServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosServer/Startup.cs @@ -38,8 +38,6 @@ namespace MareSynchronosServer { services.AddHttpContextAccessor(); - services.AddMemoryCache(); - services.Configure(Configuration.GetSection("IpRateLimiting")); services.Configure(Configuration.GetSection("IpRateLimitPolicies")); @@ -61,7 +59,6 @@ namespace MareSynchronosServer services.AddHostedService(provider => provider.GetService()); services.AddHostedService(); - services.AddDatabaseDeveloperPageExceptionFilter(); services.AddAuthentication(options => { options.DefaultScheme = SecretKeyAuthenticationHandler.AuthScheme; From cc7d6382492b597bc7cfe0b92db5d9891a9fa699 Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Fri, 12 Aug 2022 00:35:09 +0200 Subject: [PATCH 06/10] forgot to remove timer --- MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs b/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs index a7db726..67dd217 100644 --- a/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs +++ b/MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs @@ -421,8 +421,8 @@ namespace MareSynchronosServer.Discord public async Task StopAsync(CancellationToken cancellationToken) { - _timer?.Change(Timeout.Infinite, 0); verificationTaskCts?.Cancel(); + updateStatusCts?.Cancel(); await discordClient.LogoutAsync(); await discordClient.StopAsync(); From c3fc83e8196cf8d59712e5d1d43b6bff591ae8bd Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Fri, 12 Aug 2022 00:38:09 +0200 Subject: [PATCH 07/10] re-add memorycache --- MareSynchronosServer/MareSynchronosServer/Startup.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/MareSynchronosServer/MareSynchronosServer/Startup.cs b/MareSynchronosServer/MareSynchronosServer/Startup.cs index fcf2765..dc4d820 100644 --- a/MareSynchronosServer/MareSynchronosServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosServer/Startup.cs @@ -41,6 +41,7 @@ namespace MareSynchronosServer services.Configure(Configuration.GetSection("IpRateLimiting")); services.Configure(Configuration.GetSection("IpRateLimitPolicies")); + services.AddMemoryCache(); services.AddInMemoryRateLimiting(); services.AddSingleton(); From 50ff75c7ab78e14aa1eeced5afb9bb061225fdd3 Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Fri, 12 Aug 2022 10:57:03 +0200 Subject: [PATCH 08/10] cancel and dispose cts in ban auth --- .../SecretKeyAuthenticationHandler.cs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthenticationHandler.cs b/MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthenticationHandler.cs index 7a9ec8e..d5a046d 100644 --- a/MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthenticationHandler.cs +++ b/MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthenticationHandler.cs @@ -18,12 +18,23 @@ using Microsoft.Extensions.Options; namespace MareSynchronosServer.Authentication { - public class FailedAuthorization + public class FailedAuthorization : IDisposable { private int failedAttempts = 1; public int FailedAttempts => failedAttempts; public Task ResetTask { get; set; } public CancellationTokenSource ResetCts { get; set; } = new(); + + public void Dispose() + { + try + { + ResetCts?.Cancel(); + ResetCts?.Dispose(); + } + catch { } + } + public void IncreaseFailedAttempts() { Interlocked.Increment(ref failedAttempts); @@ -82,13 +93,15 @@ namespace MareSynchronosServer.Authentication if (failedAuth.FailedAttempts > failedAttemptsForTempBan) { failedAuth.ResetCts.Cancel(); + failedAuth.ResetCts.Dispose(); failedAuth.ResetCts = new CancellationTokenSource(); var token = failedAuth.ResetCts.Token; failedAuth.ResetTask = Task.Run(async () => { await Task.Delay(TimeSpan.FromMinutes(tempBanMinutes), token); if (token.IsCancellationRequested) return; - FailedAuthorizations.Remove(ip, out _); + FailedAuthorizations.Remove(ip, out var fauth); + fauth.Dispose(); }, token); Logger.LogWarning("TempBan " + ip + " for authorization spam"); return AuthenticateResult.Fail("Failed Authorization"); From ea74dde4c3f85c51ccb460e07b0b400e8bad0aa2 Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Fri, 12 Aug 2022 12:47:06 +0200 Subject: [PATCH 09/10] write uploads to file instead of memory --- .../Hubs/MareHub.Files.cs | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs index d129a86..7d98a32 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs @@ -158,18 +158,27 @@ namespace MareSynchronosServer.Hubs if (relatedFile == null) return; var forbiddenFile = _dbContext.ForbiddenUploadEntries.SingleOrDefault(f => f.Hash == hash); if (forbiddenFile != null) return; - var uploadedFile = new List(); + var finalFileName = Path.Combine(BasePath, hash); + var tempFileName = finalFileName + ".tmp"; + using var fileStream = new FileStream(tempFileName, FileMode.OpenOrCreate); + long length = 0; try { await foreach (var chunk in fileContent) { - uploadedFile.AddRange(chunk); + length += chunk.Length; + await fileStream.WriteAsync(chunk); } + + await fileStream.FlushAsync(); + await fileStream.DisposeAsync(); } catch { try { + await fileStream.FlushAsync(); + await fileStream.DisposeAsync(); _dbContext.Files.Remove(relatedFile); await _dbContext.SaveChangesAsync(); } @@ -177,15 +186,19 @@ namespace MareSynchronosServer.Hubs { // already removed } + finally + { + File.Delete(tempFileName); + } return; } - _logger.LogInformation("User " + AuthenticatedUserId + " upload finished: " + hash + ", size: " + uploadedFile.Count); + _logger.LogInformation("User " + AuthenticatedUserId + " upload finished: " + hash + ", size: " + length); try { - var decodedFile = LZ4.LZ4Codec.Unwrap(uploadedFile.ToArray()); + var decodedFile = LZ4.LZ4Codec.Unwrap(await File.ReadAllBytesAsync(tempFileName)); using var sha1 = SHA1.Create(); using var ms = new MemoryStream(decodedFile); var computedHash = await sha1.ComputeHashAsync(ms); @@ -199,12 +212,12 @@ namespace MareSynchronosServer.Hubs return; } - await File.WriteAllBytesAsync(Path.Combine(BasePath, hash), uploadedFile.ToArray()); + File.Move(tempFileName, finalFileName); relatedFile = _dbContext.Files.Single(f => f.Hash == hash); relatedFile.Uploaded = true; MareMetrics.FilesTotal.Inc(); - MareMetrics.FilesTotalSize.Inc(uploadedFile.Count); + MareMetrics.FilesTotalSize.Inc(length); await _dbContext.SaveChangesAsync(); _logger.LogInformation("File " + hash + " added to DB"); @@ -215,7 +228,6 @@ namespace MareSynchronosServer.Hubs _dbContext.Remove(relatedFile); await _dbContext.SaveChangesAsync(); } - } } } From a33cf2b2d1abb3c7cdbd94544628f8c68d1d590e Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Fri, 12 Aug 2022 17:41:09 +0200 Subject: [PATCH 10/10] overwrite on move file --- MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs index 7d98a32..a9de7f9 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs @@ -212,7 +212,7 @@ namespace MareSynchronosServer.Hubs return; } - File.Move(tempFileName, finalFileName); + File.Move(tempFileName, finalFileName, true); relatedFile = _dbContext.Files.Single(f => f.Hash == hash); relatedFile.Uploaded = true;