t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adjustments to config and cleanup nuget, remove implicit usings

Browse Source
This commit is contained in:
Stanley Dimant
2022-08-23 11:21:21 +02:00
parent afcb268ccc
commit 2510cce711
23 changed files with 156 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs
View File

@@ -66,7 +66,7 @@ namespace MareSynchronosServer.Hubs
IsForbidden = forbiddenFile != null, IsForbidden = forbiddenFile != null,
Hash = hash.Key, Hash = hash.Key,
Size = hash.Value, Size = hash.Value,
Url = _configuration["CdnFullUrl"] + hash.Key.ToUpperInvariant() Url = new Uri(cdnFullUri, hash.Key.ToUpperInvariant()).ToString()
}); });
} }

4
MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs
View File

@@ -23,10 +23,10 @@ namespace MareSynchronosServer.Hubs
private readonly AuthService.AuthServiceClient _authServiceClient; private readonly AuthService.AuthServiceClient _authServiceClient;
private readonly FileService.FileServiceClient _fileServiceClient; private readonly FileService.FileServiceClient _fileServiceClient;
private readonly SystemInfoService _systemInfoService; private readonly SystemInfoService _systemInfoService;
private readonly IConfiguration _configuration;
private readonly IHttpContextAccessor contextAccessor; private readonly IHttpContextAccessor contextAccessor;
private readonly ILogger<MareHub> _logger; private readonly ILogger<MareHub> _logger;
private readonly MareDbContext _dbContext; private readonly MareDbContext _dbContext;
private readonly Uri cdnFullUri;
public MareHub(MetricsService.MetricsServiceClient metricsClient, AuthService.AuthServiceClient authServiceClient, FileService.FileServiceClient fileServiceClient, public MareHub(MetricsService.MetricsServiceClient metricsClient, AuthService.AuthServiceClient authServiceClient, FileService.FileServiceClient fileServiceClient,
MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService, IConfiguration configuration, IHttpContextAccessor contextAccessor) MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService, IConfiguration configuration, IHttpContextAccessor contextAccessor)
{ {
@@ -34,7 +34,7 @@ namespace MareSynchronosServer.Hubs
_authServiceClient = authServiceClient; _authServiceClient = authServiceClient;
_fileServiceClient = fileServiceClient; _fileServiceClient = fileServiceClient;
_systemInfoService = systemInfoService; _systemInfoService = systemInfoService;
_configuration = configuration; cdnFullUri = new Uri(configuration.GetRequiredSection("MareSynchronos").GetValue<string>("CdnFullUrl"));
this.contextAccessor = contextAccessor; this.contextAccessor = contextAccessor;
_logger = logger; _logger = logger;
_dbContext = mareDbContext; _dbContext = mareDbContext;

12
MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj
View File

@@ -9,26 +9,14 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="AspNetCoreRateLimit" Version="4.0.2" /> <PackageReference Include="AspNetCoreRateLimit" Version="4.0.2" />
<PackageReference Include="Bazinga.AspNetCore.Authentication.Basic" Version="2.0.1" /> <PackageReference Include="Bazinga.AspNetCore.Authentication.Basic" Version="2.0.1" />
<PackageReference Include="Ben.BlockingDetector" Version="0.0.4" />
<PackageReference Include="EFCore.NamingConventions" Version="6.0.0" /> <PackageReference Include="EFCore.NamingConventions" Version="6.0.0" />
<PackageReference Include="Grpc.AspNetCore" Version="2.47.0" /> <PackageReference Include="Grpc.AspNetCore" Version="2.47.0" />
<PackageReference Include="Grpc.Net.Client" Version="2.47.0" /> <PackageReference Include="Grpc.Net.Client" Version="2.47.0" />
<PackageReference Include="Karambolo.Extensions.Logging.File" Version="3.3.1" /> <PackageReference Include="Karambolo.Extensions.Logging.File" Version="3.3.1" />
<PackageReference Include="lz4net" Version="1.0.15.93" /> <PackageReference Include="lz4net" Version="1.0.15.93" />
<PackageReference Include="Meziantou.Analyzer" Version="1.0.715">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.AspNetCore.Diagnostics.EntityFrameworkCore" Version="6.0.8" /> <PackageReference Include="Microsoft.AspNetCore.Diagnostics.EntityFrameworkCore" Version="6.0.8" />
<PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="6.0.8" /> <PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="6.0.8" />
<PackageReference Include="Microsoft.AspNetCore.Identity.UI" Version="6.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="6.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.8">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="6.0.0" /> <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="6.0.0" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="6.0.6" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>

13
MareSynchronosServer/MareSynchronosServer/Startup.cs
View File

@@ -12,7 +12,6 @@ using Microsoft.AspNetCore.Http.Connections;
using Microsoft.AspNetCore.SignalR; using Microsoft.AspNetCore.SignalR;
using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Authorization;
using AspNetCoreRateLimit; using AspNetCoreRateLimit;
using Ben.Diagnostics;
using MareSynchronosShared.Authentication; using MareSynchronosShared.Authentication;
using MareSynchronosShared.Data; using MareSynchronosShared.Data;
using MareSynchronosShared.Protos; using MareSynchronosShared.Protos;
@@ -45,17 +44,19 @@ namespace MareSynchronosServer
services.AddSingleton<IUserIdProvider, IdBasedUserIdProvider>(); services.AddSingleton<IUserIdProvider, IdBasedUserIdProvider>();
services.AddTransient(_ => Configuration); services.AddTransient(_ => Configuration);
var mareConfig = Configuration.GetRequiredSection("MareSynchronos");
services.AddGrpcClient<AuthService.AuthServiceClient>(c => services.AddGrpcClient<AuthService.AuthServiceClient>(c =>
{ {
c.Address = new Uri(Configuration.GetValue<string>("ServiceAddress")); c.Address = new Uri(mareConfig.GetValue<string>("ServiceAddress"));
}); });
services.AddGrpcClient<MetricsService.MetricsServiceClient>(c => services.AddGrpcClient<MetricsService.MetricsServiceClient>(c =>
{ {
c.Address = new Uri(Configuration.GetValue<string>("ServiceAddress")); c.Address = new Uri(mareConfig.GetValue<string>("ServiceAddress"));
}); });
services.AddGrpcClient<FileService.FileServiceClient>(c => services.AddGrpcClient<FileService.FileServiceClient>(c =>
{ {
c.Address = new Uri(Configuration.GetValue<string>("StaticFileServiceAddress")); c.Address = new Uri(mareConfig.GetValue<string>("StaticFileServiceAddress"));
}); });
services.AddDbContextPool<MareDbContext>(options => services.AddDbContextPool<MareDbContext>(options =>
@@ -66,7 +67,7 @@ namespace MareSynchronosServer
builder.MigrationsAssembly("MareSynchronosShared"); builder.MigrationsAssembly("MareSynchronosShared");
}).UseSnakeCaseNamingConvention(); }).UseSnakeCaseNamingConvention();
options.EnableThreadSafetyChecks(false); options.EnableThreadSafetyChecks(false);
}, Configuration.GetValue("DbContextPoolSize", 1024)); }, mareConfig.GetValue("DbContextPoolSize", 1024));
services.AddHostedService(provider => provider.GetService<SystemInfoService>()); services.AddHostedService(provider => provider.GetService<SystemInfoService>());
@@ -92,8 +93,6 @@ namespace MareSynchronosServer
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline. // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IWebHostEnvironment env) public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{ {
app.UseBlockingDetection();
if (env.IsDevelopment()) if (env.IsDevelopment())
{ {
app.UseDeveloperExceptionPage(); app.UseDeveloperExceptionPage();

10
MareSynchronosServer/MareSynchronosServer/appsettings.json
View File

@@ -23,10 +23,12 @@
] ]
} }
}, },
"DbContextPoolSize": 2000, "MareSynchronos": {
"CdnFullUrl": "https://<url or ip to your server>/cache/", "DbContextPoolSize": 2000,
"ServiceAddress": "http://localhost:5002", "CdnFullUrl": "https://<url or ip to your server>/cache/",
"StaticFileServiceAddress": "http://localhost:5001", "ServiceAddress": "http://localhost:5002",
"StaticFileServiceAddress": "http://localhost:5001"
},
"AllowedHosts": "*", "AllowedHosts": "*",
"Kestrel": { "Kestrel": {
"Endpoints": { "Endpoints": {

6
MareSynchronosServer/MareSynchronosServices/Authentication/FailedAuthorization.cs
View File

@@ -1,4 +1,8 @@
namespace MareSynchronosServices.Authentication; using System;
using System.Threading;
using System.Threading.Tasks;
namespace MareSynchronosServices.Authentication;
public class FailedAuthorization : IDisposable public class FailedAuthorization : IDisposable
{ {

22
MareSynchronosServer/MareSynchronosServices/Authentication/SecretKeyAuthenticationHandler.cs
View File

@@ -1,10 +1,17 @@
using System.Security.Cryptography; using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Text; using System.Text;
using System.Threading;
using System.Threading.Tasks;
using MareSynchronosServices.Metrics; using MareSynchronosServices.Metrics;
using MareSynchronosShared.Data; using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using MareSynchronosShared.Protos; using MareSynchronosShared.Protos;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
namespace MareSynchronosServices.Authentication; namespace MareSynchronosServices.Authentication;
@@ -17,8 +24,8 @@ public class SecretKeyAuthenticationHandler
private readonly Dictionary<string, FailedAuthorization?> failedAuthorizations = new(); private readonly Dictionary<string, FailedAuthorization?> failedAuthorizations = new();
private readonly object authDictLock = new(); private readonly object authDictLock = new();
private readonly object failedAuthLock = new(); private readonly object failedAuthLock = new();
private readonly int failedAttemptsForTempBan; private readonly int _failedAttemptsForTempBan;
private readonly int tempBanMinutes; private readonly int _tempBanMinutes;
public void ClearUnauthorizedUsers() public void ClearUnauthorizedUsers()
{ {
@@ -58,7 +65,7 @@ public class SecretKeyAuthenticationHandler
lock (failedAuthLock) lock (failedAuthLock)
{ {
if (failedAuthorizations.TryGetValue(ip, out var existingFailedAuthorization) && existingFailedAuthorization.FailedAttempts > failedAttemptsForTempBan) if (failedAuthorizations.TryGetValue(ip, out var existingFailedAuthorization) && existingFailedAuthorization.FailedAttempts > _failedAttemptsForTempBan)
{ {
metrics.IncCounter(MetricsAPI.CounterAuthenticationFailures); metrics.IncCounter(MetricsAPI.CounterAuthenticationFailures);
@@ -68,7 +75,7 @@ public class SecretKeyAuthenticationHandler
var token = existingFailedAuthorization.ResetCts.Token; var token = existingFailedAuthorization.ResetCts.Token;
existingFailedAuthorization.ResetTask = Task.Run(async () => existingFailedAuthorization.ResetTask = Task.Run(async () =>
{ {
await Task.Delay(TimeSpan.FromMinutes(tempBanMinutes), token).ConfigureAwait(false); await Task.Delay(TimeSpan.FromMinutes(_tempBanMinutes), token).ConfigureAwait(false);
if (token.IsCancellationRequested) return; if (token.IsCancellationRequested) return;
FailedAuthorization? failedAuthorization; FailedAuthorization? failedAuthorization;
lock (failedAuthLock) lock (failedAuthLock)
@@ -159,7 +166,8 @@ public class SecretKeyAuthenticationHandler
{ {
this.logger = logger; this.logger = logger;
this.metrics = metrics; this.metrics = metrics;
failedAttemptsForTempBan = configuration.GetValue<int>("FailedAuthForTempBan", 5); var config = configuration.GetRequiredSection("MareSynchronos");
tempBanMinutes = configuration.GetValue<int>("TempBanDurationInMinutes", 30); _failedAttemptsForTempBan = config.GetValue<int>("FailedAuthForTempBan", 5);
_tempBanMinutes = config.GetValue<int>("TempBanDurationInMinutes", 30);
} }
} }

31
MareSynchronosServer/MareSynchronosServices/CleanupService.cs
View File

@@ -3,9 +3,17 @@ using MareSynchronosServices.Metrics;
using MareSynchronosShared.Data; using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using MareSynchronosShared.Models; using MareSynchronosShared.Models;
using MareSynchronosShared.Protos;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using MetricsService = MareSynchronosShared.Protos.MetricsService; using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace MareSynchronosServices namespace MareSynchronosServices
{ {
@@ -16,7 +24,7 @@ namespace MareSynchronosServices
private readonly ILogger<CleanupService> _logger; private readonly ILogger<CleanupService> _logger;
private readonly IServiceProvider _services; private readonly IServiceProvider _services;
private readonly IConfiguration _configuration; private readonly IConfiguration _configuration;
private Timer _timer; private Timer? _timer;
public CleanupService(MareMetrics metrics, SecretKeyAuthenticationHandler authService, ILogger<CleanupService> logger, IServiceProvider services, IConfiguration configuration) public CleanupService(MareMetrics metrics, SecretKeyAuthenticationHandler authService, ILogger<CleanupService> logger, IServiceProvider services, IConfiguration configuration)
{ {
@@ -24,7 +32,7 @@ namespace MareSynchronosServices
_authService = authService; _authService = authService;
_logger = logger; _logger = logger;
_services = services; _services = services;
_configuration = configuration; _configuration = configuration.GetRequiredSection("MareSynchronos");
} }
public Task StartAsync(CancellationToken cancellationToken) public Task StartAsync(CancellationToken cancellationToken)
@@ -158,7 +166,7 @@ namespace MareSynchronosServices
foreach (var user in usersToRemove) foreach (var user in usersToRemove)
{ {
PurgeUser(user, dbContext, _configuration); PurgeUser(user, dbContext);
} }
} }
@@ -176,7 +184,7 @@ namespace MareSynchronosServices
dbContext.SaveChanges(); dbContext.SaveChanges();
} }
public void PurgeUser(User user, MareDbContext dbContext, IConfiguration _configuration) public void PurgeUser(User user, MareDbContext dbContext)
{ {
var lodestone = dbContext.LodeStoneAuth.SingleOrDefault(a => a.User.UID == user.UID); var lodestone = dbContext.LodeStoneAuth.SingleOrDefault(a => a.User.UID == user.UID);
@@ -190,17 +198,6 @@ namespace MareSynchronosServices
var auth = dbContext.Auth.Single(a => a.UserUID == user.UID); var auth = dbContext.Auth.Single(a => a.UserUID == user.UID);
var userFiles = dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == user.UID).ToList(); var userFiles = dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == user.UID).ToList();
foreach (var file in userFiles)
{
var fi = new FileInfo(Path.Combine(_configuration["CacheDirectory"], file.Hash));
if (fi.Exists)
{
metrics.DecGaugeBy(MetricsAPI.GaugeFilesTotalSize, fi.Length);
metrics.DecGaugeBy(MetricsAPI.GaugeFilesTotal, 1);
fi.Delete();
}
}
dbContext.Files.RemoveRange(userFiles); dbContext.Files.RemoveRange(userFiles);
var ownPairData = dbContext.ClientPairs.Where(u => u.User.UID == user.UID).ToList(); var ownPairData = dbContext.ClientPairs.Where(u => u.User.UID == user.UID).ToList();

31
MareSynchronosServer/MareSynchronosServices/Discord/DiscordBot.cs
View File

@@ -1,7 +1,12 @@
using System.Collections.Concurrent; using System;
using System.Collections.Concurrent;
using System.Linq;
using System.Net.Http;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Text; using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Discord; using Discord;
using Discord.WebSocket; using Discord.WebSocket;
using MareSynchronosServices.Metrics; using MareSynchronosServices.Metrics;
@@ -9,6 +14,10 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using MareSynchronosShared.Models; using MareSynchronosShared.Models;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace MareSynchronosServices.Discord; namespace MareSynchronosServices.Discord;
@@ -17,14 +26,14 @@ public class DiscordBot : IHostedService
private readonly CleanupService cleanupService; private readonly CleanupService cleanupService;
private readonly MareMetrics metrics; private readonly MareMetrics metrics;
private readonly IServiceProvider services; private readonly IServiceProvider services;
private readonly IConfiguration configuration; private readonly IConfiguration _configuration;
private readonly ILogger<DiscordBot> logger; private readonly ILogger<DiscordBot> logger;
private readonly Random random; private readonly Random random;
private string authToken = string.Empty; private string authToken = string.Empty;
DiscordSocketClient discordClient; DiscordSocketClient discordClient;
ConcurrentDictionary<ulong, string> DiscordLodestoneMapping = new(); ConcurrentDictionary<ulong, string> DiscordLodestoneMapping = new();
private CancellationTokenSource verificationTaskCts; private CancellationTokenSource? verificationTaskCts;
private CancellationTokenSource updateStatusCts; private CancellationTokenSource? updateStatusCts;
private readonly string[] LodestoneServers = new[] { "eu", "na", "jp", "fr", "de" }; private readonly string[] LodestoneServers = new[] { "eu", "na", "jp", "fr", "de" };
private readonly ConcurrentQueue<SocketSlashCommand> verificationQueue = new(); private readonly ConcurrentQueue<SocketSlashCommand> verificationQueue = new();
@@ -35,13 +44,13 @@ public class DiscordBot : IHostedService
this.cleanupService = cleanupService; this.cleanupService = cleanupService;
this.metrics = metrics; this.metrics = metrics;
this.services = services; this.services = services;
this.configuration = configuration; _configuration = configuration.GetRequiredSection("MareSynchronos");
this.logger = logger; this.logger = logger;
this.verificationQueue = new ConcurrentQueue<SocketSlashCommand>(); this.verificationQueue = new ConcurrentQueue<SocketSlashCommand>();
this.semaphore = new SemaphoreSlim(1); this.semaphore = new SemaphoreSlim(1);
random = new(); random = new();
authToken = configuration.GetValue<string>("DiscordBotToken"); authToken = _configuration.GetValue<string>("DiscordBotToken");
discordClient = new(new DiscordSocketConfig() discordClient = new(new DiscordSocketConfig()
{ {
@@ -110,7 +119,7 @@ public class DiscordBot : IHostedService
{ {
if (discordAuthedUser.User != null) if (discordAuthedUser.User != null)
{ {
cleanupService.PurgeUser(discordAuthedUser.User, db, configuration); cleanupService.PurgeUser(discordAuthedUser.User, db);
} }
else else
{ {
@@ -168,9 +177,9 @@ public class DiscordBot : IHostedService
user.IsAdmin = true; user.IsAdmin = true;
} }
if (configuration.GetValue<bool>("PurgeUnusedAccounts")) if (_configuration.GetValue<bool>("PurgeUnusedAccounts"))
{ {
var purgedDays = configuration.GetValue<int>("PurgeUnusedAccountsPeriodInDays"); var purgedDays = _configuration.GetValue<int>("PurgeUnusedAccountsPeriodInDays");
user.LastLoggedIn = DateTime.UtcNow - TimeSpan.FromDays(purgedDays) + TimeSpan.FromDays(1); user.LastLoggedIn = DateTime.UtcNow - TimeSpan.FromDays(purgedDays) + TimeSpan.FromDays(1);
} }
@@ -360,7 +369,7 @@ public class DiscordBot : IHostedService
{ {
if (!string.IsNullOrEmpty(authToken)) if (!string.IsNullOrEmpty(authToken))
{ {
authToken = configuration.GetValue<string>("DiscordBotToken"); authToken = _configuration.GetValue<string>("DiscordBotToken");
await discordClient.LoginAsync(TokenType.Bot, authToken).ConfigureAwait(false); await discordClient.LoginAsync(TokenType.Bot, authToken).ConfigureAwait(false);
await discordClient.StartAsync().ConfigureAwait(false); await discordClient.StartAsync().ConfigureAwait(false);
@@ -423,7 +432,7 @@ public class DiscordBot : IHostedService
await discordClient.StopAsync().ConfigureAwait(false); await discordClient.StopAsync().ConfigureAwait(false);
} }
public static string GenerateRandomString(int length, string allowableChars = null) public static string GenerateRandomString(int length, string? allowableChars = null)
{ {
if (string.IsNullOrEmpty(allowableChars)) if (string.IsNullOrEmpty(allowableChars))
allowableChars = @"ABCDEFGHJKLMNPQRSTUVWXYZ0123456789"; allowableChars = @"ABCDEFGHJKLMNPQRSTUVWXYZ0123456789";

8
MareSynchronosServer/MareSynchronosServices/MareSynchronosServices.csproj
View File

@@ -2,21 +2,13 @@
<PropertyGroup> <PropertyGroup>
<TargetFramework>net6.0</TargetFramework> <TargetFramework>net6.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Discord.Net" Version="3.7.2" /> <PackageReference Include="Discord.Net" Version="3.7.2" />
<PackageReference Include="EFCore.NamingConventions" Version="6.0.0" /> <PackageReference Include="EFCore.NamingConventions" Version="6.0.0" />
<PackageReference Include="Grpc.AspNetCore" Version="2.47.0" /> <PackageReference Include="Grpc.AspNetCore" Version="2.47.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="6.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.8">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="6.0.0" /> <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="6.0.0" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="6.0.6" />
<PackageReference Include="prometheus-net.AspNetCore" Version="6.0.0" /> <PackageReference Include="prometheus-net.AspNetCore" Version="6.0.0" />
</ItemGroup> </ItemGroup>

7
MareSynchronosServer/MareSynchronosServices/Metrics/MareMetrics.cs
View File

@@ -1,7 +1,11 @@
using MareSynchronosShared.Data; using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Prometheus; using Prometheus;
using System;
using System.Collections.Generic;
using System.Linq;
namespace MareSynchronosServices.Metrics; namespace MareSynchronosServices.Metrics;
@@ -16,7 +20,6 @@ public class MareMetrics
gauges[MetricsAPI.GaugePairs].IncTo(dbContext.ClientPairs.Count()); gauges[MetricsAPI.GaugePairs].IncTo(dbContext.ClientPairs.Count());
gauges[MetricsAPI.GaugePairsPaused].IncTo(dbContext.ClientPairs.Count(p => p.IsPaused)); gauges[MetricsAPI.GaugePairsPaused].IncTo(dbContext.ClientPairs.Count(p => p.IsPaused));
gauges[MetricsAPI.GaugeFilesTotal].IncTo(dbContext.Files.Count()); gauges[MetricsAPI.GaugeFilesTotal].IncTo(dbContext.Files.Count());
gauges[MetricsAPI.GaugeFilesTotalSize].IncTo(Directory.EnumerateFiles(configuration["CacheDirectory"]).Sum(f => new FileInfo(f).Length));
} }
private readonly Dictionary<string, Counter> counters = new() private readonly Dictionary<string, Counter> counters = new()

3
MareSynchronosServer/MareSynchronosServices/Program.cs
View File

@@ -1,4 +1,7 @@
using MareSynchronosServices; using MareSynchronosServices;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Hosting;
using System;
public class Program public class Program
{ {

2
MareSynchronosServer/MareSynchronosServices/Services/AuthenticationService.cs
View File

@@ -2,6 +2,8 @@ using Grpc.Core;
using MareSynchronosServices.Authentication; using MareSynchronosServices.Authentication;
using MareSynchronosShared.Data; using MareSynchronosShared.Data;
using MareSynchronosShared.Protos; using MareSynchronosShared.Protos;
using Microsoft.Extensions.Logging;
using System.Threading.Tasks;
namespace MareSynchronosServices.Services namespace MareSynchronosServices.Services
{ {

1
MareSynchronosServer/MareSynchronosServices/Services/MetricsService.cs
View File

@@ -1,6 +1,7 @@
using Grpc.Core; using Grpc.Core;
using MareSynchronosServices.Metrics; using MareSynchronosServices.Metrics;
using MareSynchronosShared.Protos; using MareSynchronosShared.Protos;
using System.Threading.Tasks;
namespace MareSynchronosServices.Services; namespace MareSynchronosServices.Services;

4
MareSynchronosServer/MareSynchronosServices/Startup.cs
View File

@@ -5,7 +5,11 @@ using MareSynchronosServices.Metrics;
using MareSynchronosServices.Services; using MareSynchronosServices.Services;
using MareSynchronosShared.Authentication; using MareSynchronosShared.Authentication;
using MareSynchronosShared.Data; using MareSynchronosShared.Data;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Prometheus; using Prometheus;
namespace MareSynchronosServices; namespace MareSynchronosServices;

15
MareSynchronosServer/MareSynchronosServices/appsettings.json
View File

@@ -16,12 +16,13 @@
} }
} }
}, },
"DbContextPoolSize": 1024, "MareSynchronos": {
"DiscordBotToken": "", "DbContextPoolSize": 1024,
"PurgeUnusedAccounts": true, "DiscordBotToken": "",
"PurgeUnusedAccountsPeriodInDays": 14, "PurgeUnusedAccounts": true,
"CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored "PurgeUnusedAccountsPeriodInDays": 14,
"FailedAuthForTempBan": 5, "FailedAuthForTempBan": 5,
"TempBanDurationInMinutes": 30, "TempBanDurationInMinutes": 30
},
"AllowedHosts": "*" "AllowedHosts": "*"
} }

6
MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj
View File

@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net6.0</TargetFramework> <TargetFramework>net6.0</TargetFramework>
@@ -16,6 +16,10 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="EFCore.NamingConventions" Version="6.0.0" /> <PackageReference Include="EFCore.NamingConventions" Version="6.0.0" />
<PackageReference Include="Grpc.AspNetCore" Version="2.47.0" /> <PackageReference Include="Grpc.AspNetCore" Version="2.47.0" />
<PackageReference Include="Meziantou.Analyzer" Version="1.0.717">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>

30
MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs
View File

@@ -1,5 +1,15 @@
using MareSynchronosShared.Data; using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using static MareSynchronosShared.Protos.MetricsService; using static MareSynchronosShared.Protos.MetricsService;
namespace MareSynchronosStaticFilesServer; namespace MareSynchronosStaticFilesServer;
@@ -17,7 +27,12 @@ public class CleanupService : IHostedService, IDisposable
_metrics = metrics; _metrics = metrics;
_logger = logger; _logger = logger;
_services = services; _services = services;
_configuration = configuration; _configuration = configuration.GetRequiredSection("MareSynchronos");
metrics.SetGauge(new MareSynchronosShared.Protos.SetGaugeRequest()
{
GaugeName = MetricsAPI.GaugeFilesTotalSize,
Value = Directory.EnumerateFiles(_configuration["CacheDirectory"]).Sum(f => new FileInfo(f).Length)
});
} }
public Task StartAsync(CancellationToken cancellationToken) public Task StartAsync(CancellationToken cancellationToken)
@@ -65,6 +80,16 @@ public class CleanupService : IHostedService, IDisposable
fi.Delete(); fi.Delete();
} }
} }
foreach (var file in Directory.EnumerateFiles(cachedir).ToList())
{
FileInfo fi = new(file);
if (!allFiles.Any(f => f.Hash == fi.Name.ToUpperInvariant()))
{
fi.Delete();
_logger.LogInformation("File not in DB, deleting: {fileName}", fi.FullName);
}
}
} }
catch (Exception ex) catch (Exception ex)
{ {
@@ -78,7 +103,8 @@ public class CleanupService : IHostedService, IDisposable
if (cacheSizeLimitInGiB > 0) if (cacheSizeLimitInGiB > 0)
{ {
_logger.LogInformation("Cleaning up files beyond the cache size limit"); _logger.LogInformation("Cleaning up files beyond the cache size limit");
var allLocalFiles = Directory.EnumerateFiles(_configuration["CacheDirectory"]).Select(f => new FileInfo(f)).ToList().OrderBy(f => f.LastAccessTimeUtc).ToList(); var allLocalFiles = Directory.EnumerateFiles(_configuration["CacheDirectory"])
.Select(f => new FileInfo(f)).ToList().OrderBy(f => f.LastAccessTimeUtc).ToList();
var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length); var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
long cacheSizeLimitInBytes = (long)(cacheSizeLimitInGiB * 1024 * 1024 * 1024); long cacheSizeLimitInBytes = (long)(cacheSizeLimitInGiB * 1024 * 1024 * 1024);
HashSet<string> removedHashes = new(); HashSet<string> removedHashes = new();

8
MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs
View File

@@ -3,7 +3,13 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using MareSynchronosShared.Protos; using MareSynchronosShared.Protos;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using System;
using System.IO;
using System.Linq;
using System.Security.Policy; using System.Security.Policy;
using System.Threading.Tasks;
namespace MareSynchronosStaticFilesServer; namespace MareSynchronosStaticFilesServer;
@@ -16,7 +22,7 @@ public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBa
public FileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger<FileService> logger, MetricsService.MetricsServiceClient metricsClient) public FileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger<FileService> logger, MetricsService.MetricsServiceClient metricsClient)
{ {
_basePath = configuration["CacheDirectory"]; _basePath = configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"];
_mareDbContext = mareDbContext; _mareDbContext = mareDbContext;
_logger = logger; _logger = logger;
_metricsClient = metricsClient; _metricsClient = metricsClient;

2
MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj
View File

@@ -2,8 +2,6 @@
<PropertyGroup> <PropertyGroup>
<TargetFramework>net6.0</TargetFramework> <TargetFramework>net6.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>

5
MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs
View File

@@ -1,5 +1,6 @@
using Microsoft.AspNetCore.Http.Connections; using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.SignalR; using Microsoft.Extensions.Hosting;
using System;
namespace MareSynchronosStaticFilesServer; namespace MareSynchronosStaticFilesServer;

15
MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs
View File

@@ -3,8 +3,13 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Protos; using MareSynchronosShared.Protos;
using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.FileProviders;
using System;
namespace MareSynchronosStaticFilesServer; namespace MareSynchronosStaticFilesServer;
@@ -23,13 +28,15 @@ public class Startup
services.AddTransient(_ => Configuration); services.AddTransient(_ => Configuration);
var mareSettings = Configuration.GetRequiredSection("MareSynchronos");
services.AddGrpcClient<AuthService.AuthServiceClient>(c => services.AddGrpcClient<AuthService.AuthServiceClient>(c =>
{ {
c.Address = new Uri(Configuration.GetValue<string>("ServiceAddress")); c.Address = new Uri(mareSettings.GetValue<string>("ServiceAddress"));
}); });
services.AddGrpcClient<MetricsService.MetricsServiceClient>(c => services.AddGrpcClient<MetricsService.MetricsServiceClient>(c =>
{ {
c.Address = new Uri(Configuration.GetValue<string>("ServiceAddress")); c.Address = new Uri(mareSettings.GetValue<string>("ServiceAddress"));
}); });
services.AddDbContextPool<MareDbContext>(options => services.AddDbContextPool<MareDbContext>(options =>
@@ -39,7 +46,7 @@ public class Startup
builder.MigrationsHistoryTable("_efmigrationshistory", "public"); builder.MigrationsHistoryTable("_efmigrationshistory", "public");
}).UseSnakeCaseNamingConvention(); }).UseSnakeCaseNamingConvention();
options.EnableThreadSafetyChecks(false); options.EnableThreadSafetyChecks(false);
}, Configuration.GetValue("DbContextPoolSize", 1024)); }, mareSettings.GetValue("DbContextPoolSize", 1024));
services.AddAuthentication(options => services.AddAuthentication(options =>
{ {
@@ -66,7 +73,7 @@ public class Startup
app.UseStaticFiles(new StaticFileOptions() app.UseStaticFiles(new StaticFileOptions()
{ {
FileProvider = new PhysicalFileProvider(Configuration["CacheDirectory"]), FileProvider = new PhysicalFileProvider(Configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"]),
RequestPath = "/cache", RequestPath = "/cache",
ServeUnknownFileTypes = true ServeUnknownFileTypes = true
}); });

15
MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json
View File

@@ -1,4 +1,7 @@
{ {
"ConnectionStrings": {
"DefaultConnection": "Host=localhost;Port=5432;Database=mare;Username=postgres"
},
"Logging": { "Logging": {
"LogLevel": { "LogLevel": {
"Default": "Information", "Default": "Information",
@@ -12,9 +15,11 @@
} }
} }
}, },
"CacheSizeHardLimitInGiB": -1, "MareSynchronos": {
"UnusedFileRetentionPeriodInDays": 7, "CacheSizeHardLimitInGiB": -1,
"AllowedHosts": "*", "UnusedFileRetentionPeriodInDays": 7,
"CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored "CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored
"ServiceAddress": "http://localhost:5002" "ServiceAddress": "http://localhost:5002"
},
"AllowedHosts": "*"
} }