Merge remote-tracking branch 'root/main'

MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthenticationHandler.cs

@@ -18,12 +18,23 @@ using Microsoft.Extensions.Options;
 
 namespace MareSynchronosServer.Authentication
 {
-    public class FailedAuthorization
+    public class FailedAuthorization : IDisposable
     {
         private int failedAttempts = 1;
         public int FailedAttempts => failedAttempts;
         public Task ResetTask { get; set; }
         public CancellationTokenSource ResetCts { get; set; } = new();
+
+        public void Dispose()
+        {
+            try
+            {
+                ResetCts?.Cancel();
+                ResetCts?.Dispose();
+            }
+            catch { }
+        }
+
         public void IncreaseFailedAttempts()
         {
             Interlocked.Increment(ref failedAttempts);
@@ -82,13 +93,15 @@ namespace MareSynchronosServer.Authentication
                 if (failedAuth.FailedAttempts > failedAttemptsForTempBan)
                 {
                     failedAuth.ResetCts.Cancel();
+                    failedAuth.ResetCts.Dispose();
                     failedAuth.ResetCts = new CancellationTokenSource();
                     var token = failedAuth.ResetCts.Token;
                     failedAuth.ResetTask = Task.Run(async () =>
                     {
                         await Task.Delay(TimeSpan.FromMinutes(tempBanMinutes), token);
                         if (token.IsCancellationRequested) return;
-                        FailedAuthorizations.Remove(ip, out _);
+                        FailedAuthorizations.Remove(ip, out var fauth);
+                        fauth.Dispose();
                     }, token);
                     Logger.LogWarning("TempBan " + ip + " for authorization spam");
                     return AuthenticateResult.Fail("Failed Authorization");

MareSynchronosServer/MareSynchronosServer/CleanupService.cs

@@ -51,7 +51,7 @@ namespace MareSynchronosServer
             try
             {
                 using var scope = _services.CreateScope();
-                var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
+                using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
 
                 var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays));
 

MareSynchronosServer/MareSynchronosServer/Discord/DiscordBot.cs

@@ -30,8 +30,8 @@ namespace MareSynchronosServer.Discord
         private string authToken = string.Empty;
         DiscordSocketClient discordClient;
         ConcurrentDictionary<ulong, string> DiscordLodestoneMapping = new();
-        private Timer _timer;
         private CancellationTokenSource verificationTaskCts;
+        private CancellationTokenSource updateStatusCts;
         private readonly string[] LodestoneServers = new[] { "eu", "na", "jp", "fr", "de" };
         private readonly ConcurrentQueue<SocketSlashCommand> verificationQueue = new();
 
@@ -248,7 +248,7 @@ namespace MareSynchronosServer.Discord
 
                 var hashedLodestoneId = BitConverter.ToString(sha256.ComputeHash(Encoding.UTF8.GetBytes(lodestoneId.ToString()))).Replace("-", "");
 
-                var db = scope.ServiceProvider.GetService<MareDbContext>();
+                using var db = scope.ServiceProvider.GetService<MareDbContext>();
 
                 // check if discord id or lodestone id is banned
                 if (db.BannedRegistrations.Any(a => a.DiscordIdOrLodestoneAuth == arg.User.Id.ToString() || a.DiscordIdOrLodestoneAuth == hashedLodestoneId))
@@ -374,9 +374,8 @@ namespace MareSynchronosServer.Discord
                 discordClient.SlashCommandExecuted += DiscordClient_SlashCommandExecuted;
                 discordClient.ModalSubmitted += DiscordClient_ModalSubmitted;
 
-                _timer = new Timer(UpdateStatus, null, TimeSpan.Zero, TimeSpan.FromSeconds(15));
+                _ = ProcessQueueWork();
-
+                _ = UpdateStatusAsync();
-                ProcessQueueWork();
             }
         }
 
@@ -385,7 +384,6 @@ namespace MareSynchronosServer.Discord
             verificationTaskCts = new CancellationTokenSource();
             while (!verificationTaskCts.IsCancellationRequested)
             {
-
                 if (verificationQueue.TryDequeue(out var queueitem))
                 {
                     try
@@ -405,20 +403,26 @@ namespace MareSynchronosServer.Discord
             }
         }
 
-        private void UpdateStatus(object state)
+        private async Task UpdateStatusAsync()
+        {
+            updateStatusCts = new();
+            while (!updateStatusCts.IsCancellationRequested)
             {
                 using var scope = services.CreateScope();
-            var db = scope.ServiceProvider.GetService<MareDbContext>();
+                using var db = scope.ServiceProvider.GetService<MareDbContext>();
 
                 var users = db.Users.Count(c => c.CharacterIdentification != null);
 
-            discordClient.SetActivityAsync(new Game("Mare for " + users + " Users"));
+                await discordClient.SetActivityAsync(new Game("Mare for " + users + " Users"));
+
+                await Task.Delay(TimeSpan.FromSeconds(15));
+            }
         }
 
         public async Task StopAsync(CancellationToken cancellationToken)
         {
-            _timer?.Change(Timeout.Infinite, 0);
             verificationTaskCts?.Cancel();
+            updateStatusCts?.Cancel();
 
             await discordClient.LogoutAsync();
             await discordClient.StopAsync();

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs

@@ -158,18 +158,27 @@ namespace MareSynchronosServer.Hubs
             if (relatedFile == null) return;
             var forbiddenFile = _dbContext.ForbiddenUploadEntries.SingleOrDefault(f => f.Hash == hash);
             if (forbiddenFile != null) return;
-            var uploadedFile = new List<byte>();
+            var finalFileName = Path.Combine(BasePath, hash);
+            var tempFileName = finalFileName + ".tmp";
+            using var fileStream = new FileStream(tempFileName, FileMode.OpenOrCreate);
+            long length = 0;
             try
             {
                 await foreach (var chunk in fileContent)
                 {
-                    uploadedFile.AddRange(chunk);
+                    length += chunk.Length;
+                    await fileStream.WriteAsync(chunk);
                 }
+
+                await fileStream.FlushAsync();
+                await fileStream.DisposeAsync();
             }
             catch
             {
                 try
                 {
+                    await fileStream.FlushAsync();
+                    await fileStream.DisposeAsync();
                     _dbContext.Files.Remove(relatedFile);
                     await _dbContext.SaveChangesAsync();
                 }
@@ -177,15 +186,19 @@ namespace MareSynchronosServer.Hubs
                 {
                     // already removed
                 }
+                finally
+                {
+                    File.Delete(tempFileName);
+                }
 
                 return;
             }
 
-            _logger.LogInformation("User " + AuthenticatedUserId + " upload finished: " + hash + ", size: " + uploadedFile.Count);
+            _logger.LogInformation("User " + AuthenticatedUserId + " upload finished: " + hash + ", size: " + length);
 
             try
             {
-                var decodedFile = LZ4.LZ4Codec.Unwrap(uploadedFile.ToArray());
+                var decodedFile = LZ4.LZ4Codec.Unwrap(await File.ReadAllBytesAsync(tempFileName));
                 using var sha1 = SHA1.Create();
                 using var ms = new MemoryStream(decodedFile);
                 var computedHash = await sha1.ComputeHashAsync(ms);
@@ -199,12 +212,12 @@ namespace MareSynchronosServer.Hubs
                     return;
                 }
 
-                await File.WriteAllBytesAsync(Path.Combine(BasePath, hash), uploadedFile.ToArray());
+                File.Move(tempFileName, finalFileName, true);
                 relatedFile = _dbContext.Files.Single(f => f.Hash == hash);
                 relatedFile.Uploaded = true;
 
                 MareMetrics.FilesTotal.Inc();
-                MareMetrics.FilesTotalSize.Inc(uploadedFile.Count);
+                MareMetrics.FilesTotalSize.Inc(length);
 
                 await _dbContext.SaveChangesAsync();
                 _logger.LogInformation("File " + hash + " added to DB");
@@ -215,7 +228,6 @@ namespace MareSynchronosServer.Hubs
                 _dbContext.Remove(relatedFile);
                 await _dbContext.SaveChangesAsync();
             }
-
         }
     }
 }

MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj

@@ -11,18 +11,18 @@
     <PackageReference Include="Bazinga.AspNetCore.Authentication.Basic" Version="2.0.1" />
     <PackageReference Include="Discord.Net" Version="3.7.2" />
     <PackageReference Include="EFCore.NamingConventions" Version="6.0.0" />
-    <PackageReference Include="Karambolo.Extensions.Logging.File" Version="3.3.0" />
+    <PackageReference Include="Karambolo.Extensions.Logging.File" Version="3.3.1" />
     <PackageReference Include="lz4net" Version="1.0.15.93" />
-    <PackageReference Include="Microsoft.AspNetCore.Diagnostics.EntityFrameworkCore" Version="6.0.6" />
+    <PackageReference Include="Microsoft.AspNetCore.Diagnostics.EntityFrameworkCore" Version="6.0.8" />
-    <PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="6.0.6" />
+    <PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="6.0.8" />
-    <PackageReference Include="Microsoft.AspNetCore.Identity.UI" Version="6.0.6" />
+    <PackageReference Include="Microsoft.AspNetCore.Identity.UI" Version="6.0.8" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.6">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.8">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="6.0.0" />
-    <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="6.0.5" />
+    <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="6.0.6" />
     <PackageReference Include="prometheus-net" Version="6.0.0" />
     <PackageReference Include="prometheus-net.AspNetCore" Version="6.0.0" />
   </ItemGroup>

MareSynchronosServer/MareSynchronosServer/Metrics/MareMetrics.cs

@@ -14,6 +14,8 @@ namespace MareSynchronosServer.Metrics
             new(Prometheus.Metrics.CreateGauge("mare_unauthorized_connections", "Unauthorized Connections"));
         public static readonly LockedProxyGauge AuthorizedConnections =
             new(Prometheus.Metrics.CreateGauge("mare_authorized_connections", "Authorized Connections"));
+        public static readonly LockedProxyGauge AvailableWorkerThreads = new(Prometheus.Metrics.CreateGauge("mare_available_threadpool", "Available Threadpool Workers"));
+        public static readonly LockedProxyGauge AvailableIOWorkerThreads = new(Prometheus.Metrics.CreateGauge("mare_available_threadpool_io", "Available Threadpool IO Workers"));
 
         public static readonly LockedProxyGauge UsersRegistered = new(Prometheus.Metrics.CreateGauge("mare_users_registered", "Total Registrations"));
 

MareSynchronosServer/MareSynchronosServer/Program.cs

@@ -20,9 +20,6 @@ namespace MareSynchronosServer
             var hostBuilder = CreateHostBuilder(args);
             var host = hostBuilder.Build();
 
-            System.Threading.ThreadPool.GetMaxThreads(out int worker, out int io);
-            Console.WriteLine($"Before: Worker threads {worker}, IO threads {io}");
-
             using (var scope = host.Services.CreateScope())
             {
                 var services = scope.ServiceProvider;
@@ -42,10 +39,6 @@ namespace MareSynchronosServer
                 context.RemoveRange(looseFiles);
                 context.SaveChanges();
 
-                System.Threading.ThreadPool.SetMaxThreads(worker, context.Users.Count() * 5);
-                System.Threading.ThreadPool.GetMaxThreads(out int workerNew, out int ioNew);
-                Console.WriteLine($"After: Worker threads {workerNew}, IO threads {ioNew}");
-
                 MareMetrics.InitializeMetrics(context, services.GetRequiredService<IConfiguration>());
             }
 

MareSynchronosServer/MareSynchronosServer/Startup.cs

@@ -38,18 +38,17 @@ namespace MareSynchronosServer
         {
             services.AddHttpContextAccessor();
 
-            services.AddMemoryCache();
-
             services.Configure<IpRateLimitOptions>(Configuration.GetSection("IpRateLimiting"));
             services.Configure<IpRateLimitPolicies>(Configuration.GetSection("IpRateLimitPolicies"));
 
+            services.AddMemoryCache();
             services.AddInMemoryRateLimiting();
 
             services.AddSingleton<SystemInfoService, SystemInfoService>();
             services.AddSingleton<IUserIdProvider, IdBasedUserIdProvider>();
             services.AddTransient(_ => Configuration);
 
-            services.AddDbContextPool<MareDbContext>(options =>
+            services.AddDbContext<MareDbContext>(options =>
             {
                 options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder =>
                 {
@@ -61,7 +60,6 @@ namespace MareSynchronosServer
             services.AddHostedService(provider => provider.GetService<SystemInfoService>());
             services.AddHostedService<DiscordBot>();
 
-            services.AddDatabaseDeveloperPageExceptionFilter();
             services.AddAuthentication(options =>
                 {
                     options.DefaultScheme = SecretKeyAuthenticationHandler.AuthScheme;

MareSynchronosServer/MareSynchronosServer/SystemInfoService.cs

@@ -1,10 +1,13 @@
 using System;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using MareSynchronos.API;
+using MareSynchronosServer.Data;
 using MareSynchronosServer.Hubs;
 using MareSynchronosServer.Metrics;
 using Microsoft.AspNetCore.SignalR;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 
@@ -12,13 +15,15 @@ namespace MareSynchronosServer;
 
 public class SystemInfoService : IHostedService, IDisposable
 {
+    private readonly IServiceProvider _services;
     private readonly ILogger<SystemInfoService> _logger;
     private readonly IHubContext<MareHub> _hubContext;
     private Timer _timer;
     public SystemInfoDto SystemInfoDto { get; private set; } = new();
 
-    public SystemInfoService(ILogger<SystemInfoService> logger, IHubContext<MareHub> hubContext)
+    public SystemInfoService(IServiceProvider services, ILogger<SystemInfoService> logger, IHubContext<MareHub> hubContext)
     {
+        _services = services;
         _logger = logger;
         _hubContext = hubContext;
     }
@@ -34,6 +39,16 @@ public class SystemInfoService : IHostedService, IDisposable
 
     private void PushSystemInfo(object state)
     {
+        ThreadPool.GetAvailableThreads(out int workerThreads, out int ioThreads);
+        _logger.LogInformation($"ThreadPool: {workerThreads} workers available, {ioThreads} IO workers available");
+        MareMetrics.AvailableWorkerThreads.Set(workerThreads);
+        MareMetrics.AvailableIOWorkerThreads.Set(ioThreads);
+
+        using var scope = _services.CreateScope();
+        using var db = scope.ServiceProvider.GetService<MareDbContext>();
+
+        var users = db.Users.Count(c => c.CharacterIdentification != null);
+
         SystemInfoDto = new SystemInfoDto()
         {
             CacheUsage = 0,
@@ -41,7 +56,7 @@ public class SystemInfoService : IHostedService, IDisposable
             RAMUsage = 0,
             NetworkIn = 0,
             NetworkOut = 0,
-            OnlineUsers = (int)MareMetrics.AuthorizedConnections.Value,
+            OnlineUsers = users,
             UploadedFiles = 0
         };