From 2510cce7114a6472bb8b8f549de7c6cb2ee3c9d8 Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Tue, 23 Aug 2022 11:21:21 +0200 Subject: [PATCH] adjustments to config and cleanup nuget, remove implicit usings --- .../Hubs/MareHub.Files.cs | 2 +- .../MareSynchronosServer/Hubs/MareHub.cs | 4 +-- .../MareSynchronosServer.csproj | 12 ------- .../MareSynchronosServer/Startup.cs | 13 ++++---- .../MareSynchronosServer/appsettings.json | 10 +++--- .../Authentication/FailedAuthorization.cs | 6 +++- .../SecretKeyAuthenticationHandler.cs | 22 ++++++++----- .../MareSynchronosServices/CleanupService.cs | 31 +++++++++---------- .../Discord/DiscordBot.cs | 31 ++++++++++++------- .../MareSynchronosServices.csproj | 8 ----- .../Metrics/MareMetrics.cs | 7 +++-- .../MareSynchronosServices/Program.cs | 3 ++ .../Services/AuthenticationService.cs | 2 ++ .../Services/MetricsService.cs | 1 + .../MareSynchronosServices/Startup.cs | 4 +++ .../MareSynchronosServices/appsettings.json | 15 ++++----- .../MareSynchronosShared.csproj | 6 +++- .../CleanupService.cs | 30 ++++++++++++++++-- .../FileService.cs | 8 ++++- .../MareSynchronosStaticFilesServer.csproj | 2 -- .../Program.cs | 5 +-- .../Startup.cs | 15 ++++++--- .../appsettings.json | 15 ++++++--- 23 files changed, 156 insertions(+), 96 deletions(-) diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs index 5471cf8..ca0a16c 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs @@ -66,7 +66,7 @@ namespace MareSynchronosServer.Hubs IsForbidden = forbiddenFile != null, Hash = hash.Key, Size = hash.Value, - Url = _configuration["CdnFullUrl"] + hash.Key.ToUpperInvariant() + Url = new Uri(cdnFullUri, hash.Key.ToUpperInvariant()).ToString() }); } diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs index 14fda49..9c714fc 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs @@ -23,10 +23,10 @@ namespace MareSynchronosServer.Hubs private readonly AuthService.AuthServiceClient _authServiceClient; private readonly FileService.FileServiceClient _fileServiceClient; private readonly SystemInfoService _systemInfoService; - private readonly IConfiguration _configuration; private readonly IHttpContextAccessor contextAccessor; private readonly ILogger _logger; private readonly MareDbContext _dbContext; + private readonly Uri cdnFullUri; public MareHub(MetricsService.MetricsServiceClient metricsClient, AuthService.AuthServiceClient authServiceClient, FileService.FileServiceClient fileServiceClient, MareDbContext mareDbContext, ILogger logger, SystemInfoService systemInfoService, IConfiguration configuration, IHttpContextAccessor contextAccessor) { @@ -34,7 +34,7 @@ namespace MareSynchronosServer.Hubs _authServiceClient = authServiceClient; _fileServiceClient = fileServiceClient; _systemInfoService = systemInfoService; - _configuration = configuration; + cdnFullUri = new Uri(configuration.GetRequiredSection("MareSynchronos").GetValue("CdnFullUrl")); this.contextAccessor = contextAccessor; _logger = logger; _dbContext = mareDbContext; diff --git a/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj b/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj index 38e2081..0fd7b11 100644 --- a/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj +++ b/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj @@ -9,26 +9,14 @@ - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - diff --git a/MareSynchronosServer/MareSynchronosServer/Startup.cs b/MareSynchronosServer/MareSynchronosServer/Startup.cs index 374f236..3149e8a 100644 --- a/MareSynchronosServer/MareSynchronosServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosServer/Startup.cs @@ -12,7 +12,6 @@ using Microsoft.AspNetCore.Http.Connections; using Microsoft.AspNetCore.SignalR; using Microsoft.AspNetCore.Authorization; using AspNetCoreRateLimit; -using Ben.Diagnostics; using MareSynchronosShared.Authentication; using MareSynchronosShared.Data; using MareSynchronosShared.Protos; @@ -45,17 +44,19 @@ namespace MareSynchronosServer services.AddSingleton(); services.AddTransient(_ => Configuration); + var mareConfig = Configuration.GetRequiredSection("MareSynchronos"); + services.AddGrpcClient(c => { - c.Address = new Uri(Configuration.GetValue("ServiceAddress")); + c.Address = new Uri(mareConfig.GetValue("ServiceAddress")); }); services.AddGrpcClient(c => { - c.Address = new Uri(Configuration.GetValue("ServiceAddress")); + c.Address = new Uri(mareConfig.GetValue("ServiceAddress")); }); services.AddGrpcClient(c => { - c.Address = new Uri(Configuration.GetValue("StaticFileServiceAddress")); + c.Address = new Uri(mareConfig.GetValue("StaticFileServiceAddress")); }); services.AddDbContextPool(options => @@ -66,7 +67,7 @@ namespace MareSynchronosServer builder.MigrationsAssembly("MareSynchronosShared"); }).UseSnakeCaseNamingConvention(); options.EnableThreadSafetyChecks(false); - }, Configuration.GetValue("DbContextPoolSize", 1024)); + }, mareConfig.GetValue("DbContextPoolSize", 1024)); services.AddHostedService(provider => provider.GetService()); @@ -92,8 +93,6 @@ namespace MareSynchronosServer // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { - app.UseBlockingDetection(); - if (env.IsDevelopment()) { app.UseDeveloperExceptionPage(); diff --git a/MareSynchronosServer/MareSynchronosServer/appsettings.json b/MareSynchronosServer/MareSynchronosServer/appsettings.json index 1883cef..053a19c 100644 --- a/MareSynchronosServer/MareSynchronosServer/appsettings.json +++ b/MareSynchronosServer/MareSynchronosServer/appsettings.json @@ -23,10 +23,12 @@ ] } }, - "DbContextPoolSize": 2000, - "CdnFullUrl": "https:///cache/", - "ServiceAddress": "http://localhost:5002", - "StaticFileServiceAddress": "http://localhost:5001", + "MareSynchronos": { + "DbContextPoolSize": 2000, + "CdnFullUrl": "https:///cache/", + "ServiceAddress": "http://localhost:5002", + "StaticFileServiceAddress": "http://localhost:5001" + }, "AllowedHosts": "*", "Kestrel": { "Endpoints": { diff --git a/MareSynchronosServer/MareSynchronosServices/Authentication/FailedAuthorization.cs b/MareSynchronosServer/MareSynchronosServices/Authentication/FailedAuthorization.cs index 3521102..404546d 100644 --- a/MareSynchronosServer/MareSynchronosServices/Authentication/FailedAuthorization.cs +++ b/MareSynchronosServer/MareSynchronosServices/Authentication/FailedAuthorization.cs @@ -1,4 +1,8 @@ -namespace MareSynchronosServices.Authentication; +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace MareSynchronosServices.Authentication; public class FailedAuthorization : IDisposable { diff --git a/MareSynchronosServer/MareSynchronosServices/Authentication/SecretKeyAuthenticationHandler.cs b/MareSynchronosServer/MareSynchronosServices/Authentication/SecretKeyAuthenticationHandler.cs index 960eb60..9977016 100644 --- a/MareSynchronosServer/MareSynchronosServices/Authentication/SecretKeyAuthenticationHandler.cs +++ b/MareSynchronosServer/MareSynchronosServices/Authentication/SecretKeyAuthenticationHandler.cs @@ -1,10 +1,17 @@ -using System.Security.Cryptography; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; using System.Text; +using System.Threading; +using System.Threading.Tasks; using MareSynchronosServices.Metrics; using MareSynchronosShared.Data; using MareSynchronosShared.Metrics; using MareSynchronosShared.Protos; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; namespace MareSynchronosServices.Authentication; @@ -17,8 +24,8 @@ public class SecretKeyAuthenticationHandler private readonly Dictionary failedAuthorizations = new(); private readonly object authDictLock = new(); private readonly object failedAuthLock = new(); - private readonly int failedAttemptsForTempBan; - private readonly int tempBanMinutes; + private readonly int _failedAttemptsForTempBan; + private readonly int _tempBanMinutes; public void ClearUnauthorizedUsers() { @@ -58,7 +65,7 @@ public class SecretKeyAuthenticationHandler lock (failedAuthLock) { - if (failedAuthorizations.TryGetValue(ip, out var existingFailedAuthorization) && existingFailedAuthorization.FailedAttempts > failedAttemptsForTempBan) + if (failedAuthorizations.TryGetValue(ip, out var existingFailedAuthorization) && existingFailedAuthorization.FailedAttempts > _failedAttemptsForTempBan) { metrics.IncCounter(MetricsAPI.CounterAuthenticationFailures); @@ -68,7 +75,7 @@ public class SecretKeyAuthenticationHandler var token = existingFailedAuthorization.ResetCts.Token; existingFailedAuthorization.ResetTask = Task.Run(async () => { - await Task.Delay(TimeSpan.FromMinutes(tempBanMinutes), token).ConfigureAwait(false); + await Task.Delay(TimeSpan.FromMinutes(_tempBanMinutes), token).ConfigureAwait(false); if (token.IsCancellationRequested) return; FailedAuthorization? failedAuthorization; lock (failedAuthLock) @@ -159,7 +166,8 @@ public class SecretKeyAuthenticationHandler { this.logger = logger; this.metrics = metrics; - failedAttemptsForTempBan = configuration.GetValue("FailedAuthForTempBan", 5); - tempBanMinutes = configuration.GetValue("TempBanDurationInMinutes", 30); + var config = configuration.GetRequiredSection("MareSynchronos"); + _failedAttemptsForTempBan = config.GetValue("FailedAuthForTempBan", 5); + _tempBanMinutes = config.GetValue("TempBanDurationInMinutes", 30); } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosServices/CleanupService.cs b/MareSynchronosServer/MareSynchronosServices/CleanupService.cs index efc8a03..c21ea73 100644 --- a/MareSynchronosServer/MareSynchronosServices/CleanupService.cs +++ b/MareSynchronosServer/MareSynchronosServices/CleanupService.cs @@ -3,9 +3,17 @@ using MareSynchronosServices.Metrics; using MareSynchronosShared.Data; using MareSynchronosShared.Metrics; using MareSynchronosShared.Models; -using MareSynchronosShared.Protos; using Microsoft.EntityFrameworkCore; -using MetricsService = MareSynchronosShared.Protos.MetricsService; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; namespace MareSynchronosServices { @@ -16,7 +24,7 @@ namespace MareSynchronosServices private readonly ILogger _logger; private readonly IServiceProvider _services; private readonly IConfiguration _configuration; - private Timer _timer; + private Timer? _timer; public CleanupService(MareMetrics metrics, SecretKeyAuthenticationHandler authService, ILogger logger, IServiceProvider services, IConfiguration configuration) { @@ -24,7 +32,7 @@ namespace MareSynchronosServices _authService = authService; _logger = logger; _services = services; - _configuration = configuration; + _configuration = configuration.GetRequiredSection("MareSynchronos"); } public Task StartAsync(CancellationToken cancellationToken) @@ -158,7 +166,7 @@ namespace MareSynchronosServices foreach (var user in usersToRemove) { - PurgeUser(user, dbContext, _configuration); + PurgeUser(user, dbContext); } } @@ -176,7 +184,7 @@ namespace MareSynchronosServices dbContext.SaveChanges(); } - public void PurgeUser(User user, MareDbContext dbContext, IConfiguration _configuration) + public void PurgeUser(User user, MareDbContext dbContext) { var lodestone = dbContext.LodeStoneAuth.SingleOrDefault(a => a.User.UID == user.UID); @@ -190,17 +198,6 @@ namespace MareSynchronosServices var auth = dbContext.Auth.Single(a => a.UserUID == user.UID); var userFiles = dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == user.UID).ToList(); - foreach (var file in userFiles) - { - var fi = new FileInfo(Path.Combine(_configuration["CacheDirectory"], file.Hash)); - if (fi.Exists) - { - metrics.DecGaugeBy(MetricsAPI.GaugeFilesTotalSize, fi.Length); - metrics.DecGaugeBy(MetricsAPI.GaugeFilesTotal, 1); - fi.Delete(); - } - } - dbContext.Files.RemoveRange(userFiles); var ownPairData = dbContext.ClientPairs.Where(u => u.User.UID == user.UID).ToList(); diff --git a/MareSynchronosServer/MareSynchronosServices/Discord/DiscordBot.cs b/MareSynchronosServer/MareSynchronosServices/Discord/DiscordBot.cs index 3fdc054..4de2b25 100644 --- a/MareSynchronosServer/MareSynchronosServices/Discord/DiscordBot.cs +++ b/MareSynchronosServer/MareSynchronosServices/Discord/DiscordBot.cs @@ -1,7 +1,12 @@ -using System.Collections.Concurrent; +using System; +using System.Collections.Concurrent; +using System.Linq; +using System.Net.Http; using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; using Discord; using Discord.WebSocket; using MareSynchronosServices.Metrics; @@ -9,6 +14,10 @@ using MareSynchronosShared.Data; using MareSynchronosShared.Metrics; using MareSynchronosShared.Models; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; namespace MareSynchronosServices.Discord; @@ -17,14 +26,14 @@ public class DiscordBot : IHostedService private readonly CleanupService cleanupService; private readonly MareMetrics metrics; private readonly IServiceProvider services; - private readonly IConfiguration configuration; + private readonly IConfiguration _configuration; private readonly ILogger logger; private readonly Random random; private string authToken = string.Empty; DiscordSocketClient discordClient; ConcurrentDictionary DiscordLodestoneMapping = new(); - private CancellationTokenSource verificationTaskCts; - private CancellationTokenSource updateStatusCts; + private CancellationTokenSource? verificationTaskCts; + private CancellationTokenSource? updateStatusCts; private readonly string[] LodestoneServers = new[] { "eu", "na", "jp", "fr", "de" }; private readonly ConcurrentQueue verificationQueue = new(); @@ -35,13 +44,13 @@ public class DiscordBot : IHostedService this.cleanupService = cleanupService; this.metrics = metrics; this.services = services; - this.configuration = configuration; + _configuration = configuration.GetRequiredSection("MareSynchronos"); this.logger = logger; this.verificationQueue = new ConcurrentQueue(); this.semaphore = new SemaphoreSlim(1); random = new(); - authToken = configuration.GetValue("DiscordBotToken"); + authToken = _configuration.GetValue("DiscordBotToken"); discordClient = new(new DiscordSocketConfig() { @@ -110,7 +119,7 @@ public class DiscordBot : IHostedService { if (discordAuthedUser.User != null) { - cleanupService.PurgeUser(discordAuthedUser.User, db, configuration); + cleanupService.PurgeUser(discordAuthedUser.User, db); } else { @@ -168,9 +177,9 @@ public class DiscordBot : IHostedService user.IsAdmin = true; } - if (configuration.GetValue("PurgeUnusedAccounts")) + if (_configuration.GetValue("PurgeUnusedAccounts")) { - var purgedDays = configuration.GetValue("PurgeUnusedAccountsPeriodInDays"); + var purgedDays = _configuration.GetValue("PurgeUnusedAccountsPeriodInDays"); user.LastLoggedIn = DateTime.UtcNow - TimeSpan.FromDays(purgedDays) + TimeSpan.FromDays(1); } @@ -360,7 +369,7 @@ public class DiscordBot : IHostedService { if (!string.IsNullOrEmpty(authToken)) { - authToken = configuration.GetValue("DiscordBotToken"); + authToken = _configuration.GetValue("DiscordBotToken"); await discordClient.LoginAsync(TokenType.Bot, authToken).ConfigureAwait(false); await discordClient.StartAsync().ConfigureAwait(false); @@ -423,7 +432,7 @@ public class DiscordBot : IHostedService await discordClient.StopAsync().ConfigureAwait(false); } - public static string GenerateRandomString(int length, string allowableChars = null) + public static string GenerateRandomString(int length, string? allowableChars = null) { if (string.IsNullOrEmpty(allowableChars)) allowableChars = @"ABCDEFGHJKLMNPQRSTUVWXYZ0123456789"; diff --git a/MareSynchronosServer/MareSynchronosServices/MareSynchronosServices.csproj b/MareSynchronosServer/MareSynchronosServices/MareSynchronosServices.csproj index 0ed1bf3..6dcbfc0 100644 --- a/MareSynchronosServer/MareSynchronosServices/MareSynchronosServices.csproj +++ b/MareSynchronosServer/MareSynchronosServices/MareSynchronosServices.csproj @@ -2,21 +2,13 @@ net6.0 - enable - enable - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - diff --git a/MareSynchronosServer/MareSynchronosServices/Metrics/MareMetrics.cs b/MareSynchronosServer/MareSynchronosServices/Metrics/MareMetrics.cs index 85d0cce..88467f5 100644 --- a/MareSynchronosServer/MareSynchronosServices/Metrics/MareMetrics.cs +++ b/MareSynchronosServer/MareSynchronosServices/Metrics/MareMetrics.cs @@ -1,7 +1,11 @@ using MareSynchronosShared.Data; using MareSynchronosShared.Metrics; - +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Prometheus; +using System; +using System.Collections.Generic; +using System.Linq; namespace MareSynchronosServices.Metrics; @@ -16,7 +20,6 @@ public class MareMetrics gauges[MetricsAPI.GaugePairs].IncTo(dbContext.ClientPairs.Count()); gauges[MetricsAPI.GaugePairsPaused].IncTo(dbContext.ClientPairs.Count(p => p.IsPaused)); gauges[MetricsAPI.GaugeFilesTotal].IncTo(dbContext.Files.Count()); - gauges[MetricsAPI.GaugeFilesTotalSize].IncTo(Directory.EnumerateFiles(configuration["CacheDirectory"]).Sum(f => new FileInfo(f).Length)); } private readonly Dictionary counters = new() diff --git a/MareSynchronosServer/MareSynchronosServices/Program.cs b/MareSynchronosServer/MareSynchronosServices/Program.cs index 8da5729..0e78505 100644 --- a/MareSynchronosServer/MareSynchronosServices/Program.cs +++ b/MareSynchronosServer/MareSynchronosServices/Program.cs @@ -1,4 +1,7 @@ using MareSynchronosServices; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Hosting; +using System; public class Program { diff --git a/MareSynchronosServer/MareSynchronosServices/Services/AuthenticationService.cs b/MareSynchronosServer/MareSynchronosServices/Services/AuthenticationService.cs index a720c61..d8b62e2 100644 --- a/MareSynchronosServer/MareSynchronosServices/Services/AuthenticationService.cs +++ b/MareSynchronosServer/MareSynchronosServices/Services/AuthenticationService.cs @@ -2,6 +2,8 @@ using Grpc.Core; using MareSynchronosServices.Authentication; using MareSynchronosShared.Data; using MareSynchronosShared.Protos; +using Microsoft.Extensions.Logging; +using System.Threading.Tasks; namespace MareSynchronosServices.Services { diff --git a/MareSynchronosServer/MareSynchronosServices/Services/MetricsService.cs b/MareSynchronosServer/MareSynchronosServices/Services/MetricsService.cs index 26bae42..44e8fb1 100644 --- a/MareSynchronosServer/MareSynchronosServices/Services/MetricsService.cs +++ b/MareSynchronosServer/MareSynchronosServices/Services/MetricsService.cs @@ -1,6 +1,7 @@ using Grpc.Core; using MareSynchronosServices.Metrics; using MareSynchronosShared.Protos; +using System.Threading.Tasks; namespace MareSynchronosServices.Services; diff --git a/MareSynchronosServer/MareSynchronosServices/Startup.cs b/MareSynchronosServer/MareSynchronosServices/Startup.cs index 0095df9..791bfac 100644 --- a/MareSynchronosServer/MareSynchronosServices/Startup.cs +++ b/MareSynchronosServer/MareSynchronosServices/Startup.cs @@ -5,7 +5,11 @@ using MareSynchronosServices.Metrics; using MareSynchronosServices.Services; using MareSynchronosShared.Authentication; using MareSynchronosShared.Data; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Prometheus; namespace MareSynchronosServices; diff --git a/MareSynchronosServer/MareSynchronosServices/appsettings.json b/MareSynchronosServer/MareSynchronosServices/appsettings.json index 51429f3..88eee64 100644 --- a/MareSynchronosServer/MareSynchronosServices/appsettings.json +++ b/MareSynchronosServer/MareSynchronosServices/appsettings.json @@ -16,12 +16,13 @@ } } }, - "DbContextPoolSize": 1024, - "DiscordBotToken": "", - "PurgeUnusedAccounts": true, - "PurgeUnusedAccountsPeriodInDays": 14, - "CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored - "FailedAuthForTempBan": 5, - "TempBanDurationInMinutes": 30, + "MareSynchronos": { + "DbContextPoolSize": 1024, + "DiscordBotToken": "", + "PurgeUnusedAccounts": true, + "PurgeUnusedAccountsPeriodInDays": 14, + "FailedAuthForTempBan": 5, + "TempBanDurationInMinutes": 30 + }, "AllowedHosts": "*" } diff --git a/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj b/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj index dd32764..2964410 100644 --- a/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj +++ b/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj @@ -1,4 +1,4 @@ - + net6.0 @@ -16,6 +16,10 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs index 72c6cca..e2125b4 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs @@ -1,5 +1,15 @@ using MareSynchronosShared.Data; using MareSynchronosShared.Metrics; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; using static MareSynchronosShared.Protos.MetricsService; namespace MareSynchronosStaticFilesServer; @@ -17,7 +27,12 @@ public class CleanupService : IHostedService, IDisposable _metrics = metrics; _logger = logger; _services = services; - _configuration = configuration; + _configuration = configuration.GetRequiredSection("MareSynchronos"); + metrics.SetGauge(new MareSynchronosShared.Protos.SetGaugeRequest() + { + GaugeName = MetricsAPI.GaugeFilesTotalSize, + Value = Directory.EnumerateFiles(_configuration["CacheDirectory"]).Sum(f => new FileInfo(f).Length) + }); } public Task StartAsync(CancellationToken cancellationToken) @@ -65,6 +80,16 @@ public class CleanupService : IHostedService, IDisposable fi.Delete(); } } + + foreach (var file in Directory.EnumerateFiles(cachedir).ToList()) + { + FileInfo fi = new(file); + if (!allFiles.Any(f => f.Hash == fi.Name.ToUpperInvariant())) + { + fi.Delete(); + _logger.LogInformation("File not in DB, deleting: {fileName}", fi.FullName); + } + } } catch (Exception ex) { @@ -78,7 +103,8 @@ public class CleanupService : IHostedService, IDisposable if (cacheSizeLimitInGiB > 0) { _logger.LogInformation("Cleaning up files beyond the cache size limit"); - var allLocalFiles = Directory.EnumerateFiles(_configuration["CacheDirectory"]).Select(f => new FileInfo(f)).ToList().OrderBy(f => f.LastAccessTimeUtc).ToList(); + var allLocalFiles = Directory.EnumerateFiles(_configuration["CacheDirectory"]) + .Select(f => new FileInfo(f)).ToList().OrderBy(f => f.LastAccessTimeUtc).ToList(); var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length); long cacheSizeLimitInBytes = (long)(cacheSizeLimitInGiB * 1024 * 1024 * 1024); HashSet removedHashes = new(); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs index a7a02f6..27b2fc6 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs @@ -3,7 +3,13 @@ using MareSynchronosShared.Data; using MareSynchronosShared.Metrics; using MareSynchronosShared.Protos; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using System; +using System.IO; +using System.Linq; using System.Security.Policy; +using System.Threading.Tasks; namespace MareSynchronosStaticFilesServer; @@ -16,7 +22,7 @@ public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBa public FileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger logger, MetricsService.MetricsServiceClient metricsClient) { - _basePath = configuration["CacheDirectory"]; + _basePath = configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"]; _mareDbContext = mareDbContext; _logger = logger; _metricsClient = metricsClient; diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj b/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj index 23a6cae..7a4c93b 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj @@ -2,8 +2,6 @@ net6.0 - enable - enable diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs index 663da32..8c8d8f9 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs @@ -1,5 +1,6 @@ -using Microsoft.AspNetCore.Http.Connections; -using Microsoft.AspNetCore.SignalR; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Hosting; +using System; namespace MareSynchronosStaticFilesServer; diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs index cb3760c..5ca1ad8 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs @@ -3,8 +3,13 @@ using MareSynchronosShared.Data; using MareSynchronosShared.Protos; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.FileProviders; +using System; namespace MareSynchronosStaticFilesServer; @@ -23,13 +28,15 @@ public class Startup services.AddTransient(_ => Configuration); + var mareSettings = Configuration.GetRequiredSection("MareSynchronos"); + services.AddGrpcClient(c => { - c.Address = new Uri(Configuration.GetValue("ServiceAddress")); + c.Address = new Uri(mareSettings.GetValue("ServiceAddress")); }); services.AddGrpcClient(c => { - c.Address = new Uri(Configuration.GetValue("ServiceAddress")); + c.Address = new Uri(mareSettings.GetValue("ServiceAddress")); }); services.AddDbContextPool(options => @@ -39,7 +46,7 @@ public class Startup builder.MigrationsHistoryTable("_efmigrationshistory", "public"); }).UseSnakeCaseNamingConvention(); options.EnableThreadSafetyChecks(false); - }, Configuration.GetValue("DbContextPoolSize", 1024)); + }, mareSettings.GetValue("DbContextPoolSize", 1024)); services.AddAuthentication(options => { @@ -66,7 +73,7 @@ public class Startup app.UseStaticFiles(new StaticFileOptions() { - FileProvider = new PhysicalFileProvider(Configuration["CacheDirectory"]), + FileProvider = new PhysicalFileProvider(Configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"]), RequestPath = "/cache", ServeUnknownFileTypes = true }); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json b/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json index 2e9aa47..abe9aed 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json @@ -1,4 +1,7 @@ { + "ConnectionStrings": { + "DefaultConnection": "Host=localhost;Port=5432;Database=mare;Username=postgres" + }, "Logging": { "LogLevel": { "Default": "Information", @@ -12,9 +15,11 @@ } } }, - "CacheSizeHardLimitInGiB": -1, - "UnusedFileRetentionPeriodInDays": 7, - "AllowedHosts": "*", - "CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored - "ServiceAddress": "http://localhost:5002" + "MareSynchronos": { + "CacheSizeHardLimitInGiB": -1, + "UnusedFileRetentionPeriodInDays": 7, + "CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored + "ServiceAddress": "http://localhost:5002" + }, + "AllowedHosts": "*" }