switch around all iconfiguration usage to ioptions

This commit is contained in:
rootdarkarchon
2022-12-23 01:53:21 +01:00
parent b8d61b38bd
commit 7ee7fdaf48
21 changed files with 218 additions and 99 deletions

View File

@@ -12,8 +12,8 @@ using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.SignalR; using Microsoft.AspNetCore.SignalR;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace MareSynchronosServer.Hubs; namespace MareSynchronosServer.Hubs;
@@ -33,18 +33,18 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
private readonly int _maxGroupUserCount; private readonly int _maxGroupUserCount;
public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient, public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient,
MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService, IConfiguration configuration, IHttpContextAccessor contextAccessor, MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService, IOptions<ServerConfiguration> configuration, IHttpContextAccessor contextAccessor,
GrpcClientIdentificationService clientIdentService) GrpcClientIdentificationService clientIdentService)
{ {
_mareMetrics = mareMetrics; _mareMetrics = mareMetrics;
_fileServiceClient = fileServiceClient; _fileServiceClient = fileServiceClient;
_systemInfoService = systemInfoService; _systemInfoService = systemInfoService;
var config = configuration.GetRequiredSection("MareSynchronos"); var config = configuration.Value;
_cdnFullUri = new Uri(config.GetValue<string>("CdnFullUrl")); _cdnFullUri = config.CdnFullUrl;
_shardName = config.GetValue("ShardName", "Main"); _shardName = config.ShardName;
_maxExistingGroupsByUser = config.GetValue<int>("MaxExistingGroupsByUser", 3); _maxExistingGroupsByUser = config.MaxExistingGroupsByUser;
_maxJoinedGroupsByUser = config.GetValue<int>("MaxJoinedGroupsByUser", 6); _maxJoinedGroupsByUser = config.MaxJoinedGroupsByUser;
_maxGroupUserCount = config.GetValue<int>("MaxGroupUserCount", 100); _maxGroupUserCount = config.MaxGroupUserCount;
_contextAccessor = contextAccessor; _contextAccessor = contextAccessor;
_clientIdentService = clientIdentService; _clientIdentService = clientIdentService;
_logger = new MareHubLogger(this, logger); _logger = new MareHubLogger(this, logger);

View File

@@ -8,6 +8,7 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using MareSynchronosShared.Data; using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using Microsoft.Extensions.Options;
namespace MareSynchronosServer; namespace MareSynchronosServer;
@@ -40,6 +41,11 @@ public class Program
metrics.SetGaugeTo(MetricsAPI.GaugePairs, context.ClientPairs.Count()); metrics.SetGaugeTo(MetricsAPI.GaugePairs, context.ClientPairs.Count());
metrics.SetGaugeTo(MetricsAPI.GaugePairsPaused, context.ClientPairs.Count(p => p.IsPaused)); metrics.SetGaugeTo(MetricsAPI.GaugePairsPaused, context.ClientPairs.Count(p => p.IsPaused));
var options = host.Services.GetService<IOptions<ServerConfiguration>>();
var logger = host.Services.GetService<ILogger<Program>>();
logger.LogInformation("Loaded MareSynchronos Server Configuration");
logger.LogInformation(options.Value.ToString());
} }
if (args.Length == 0 || !string.Equals(args[0], "dry", StringComparison.Ordinal)) if (args.Length == 0 || !string.Equals(args[0], "dry", StringComparison.Ordinal))

View File

@@ -0,0 +1,32 @@
using MareSynchronosShared.Utils;
using System;
using System.Text;
namespace MareSynchronosServer;
public class ServerConfiguration : MareConfigurationAuthBase
{
public Uri CdnFullUrl { get; set; } = null;
public Uri ServiceAddress { get; set; } = null;
public Uri StaticFileServiceAddress { get; set; } = null;
public string RedisConnectionString { get; set; } = string.Empty;
public int MaxExistingGroupsByUser { get; set; } = 3;
public int MaxJoinedGroupsByUser { get; set; } = 6;
public int MaxGroupUserCount { get; set; } = 100;
public string ShardName { get; set; } = string.Empty;
public override string ToString()
{
StringBuilder sb = new();
sb.AppendLine(base.ToString());
sb.AppendLine($"{nameof(ShardName)} => {ShardName}");
sb.AppendLine($"{nameof(CdnFullUrl)} => {CdnFullUrl}");
sb.AppendLine($"{nameof(ServiceAddress)} => {ServiceAddress}");
sb.AppendLine($"{nameof(StaticFileServiceAddress)} => {StaticFileServiceAddress}");
sb.AppendLine($"{nameof(RedisConnectionString)} => {RedisConnectionString}");
sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");
sb.AppendLine($"{nameof(MaxJoinedGroupsByUser)} => {MaxJoinedGroupsByUser}");
sb.AppendLine($"{nameof(MaxGroupUserCount)} => {MaxGroupUserCount}");
return sb.ToString();
}
}

View File

@@ -2,8 +2,8 @@
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using MareSynchronosShared.Protos; using MareSynchronosShared.Protos;
using MareSynchronosShared.Services; using MareSynchronosShared.Services;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System; using System;
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Linq; using System.Linq;
@@ -17,8 +17,8 @@ public class GrpcClientIdentificationService : GrpcBaseService
private readonly string _shardName; private readonly string _shardName;
private readonly ILogger<GrpcClientIdentificationService> _logger; private readonly ILogger<GrpcClientIdentificationService> _logger;
private readonly IdentificationService.IdentificationServiceClient _grpcIdentClient; private readonly IdentificationService.IdentificationServiceClient _grpcIdentClient;
private readonly IdentificationService.IdentificationServiceClient grpcIdentClientStreamOut; private readonly IdentificationService.IdentificationServiceClient _grpcIdentClientStreamOut;
private readonly IdentificationService.IdentificationServiceClient grpcIdentClientStreamIn; private readonly IdentificationService.IdentificationServiceClient _grpcIdentClientStreamIn;
private readonly MareMetrics _metrics; private readonly MareMetrics _metrics;
protected readonly ConcurrentDictionary<string, UidWithIdent> OnlineClients = new(StringComparer.Ordinal); protected readonly ConcurrentDictionary<string, UidWithIdent> OnlineClients = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, UidWithIdent> RemoteCachedIdents = new(StringComparer.Ordinal); private readonly ConcurrentDictionary<string, UidWithIdent> RemoteCachedIdents = new(StringComparer.Ordinal);
@@ -26,14 +26,13 @@ public class GrpcClientIdentificationService : GrpcBaseService
public GrpcClientIdentificationService(ILogger<GrpcClientIdentificationService> logger, IdentificationService.IdentificationServiceClient gprcIdentClient, public GrpcClientIdentificationService(ILogger<GrpcClientIdentificationService> logger, IdentificationService.IdentificationServiceClient gprcIdentClient,
IdentificationService.IdentificationServiceClient gprcIdentClientStreamOut, IdentificationService.IdentificationServiceClient gprcIdentClientStreamOut,
IdentificationService.IdentificationServiceClient gprcIdentClientStreamIn, MareMetrics metrics, IConfiguration configuration) : base(logger) IdentificationService.IdentificationServiceClient gprcIdentClientStreamIn, MareMetrics metrics, IOptions<ServerConfiguration> configuration) : base(logger)
{ {
var config = configuration.GetSection("MareSynchronos"); _shardName = configuration.Value.ShardName;
_shardName = config.GetValue("ShardName", "Main");
_logger = logger; _logger = logger;
_grpcIdentClient = gprcIdentClient; _grpcIdentClient = gprcIdentClient;
this.grpcIdentClientStreamOut = gprcIdentClientStreamOut; _grpcIdentClientStreamOut = gprcIdentClientStreamOut;
this.grpcIdentClientStreamIn = gprcIdentClientStreamIn; _grpcIdentClientStreamIn = gprcIdentClientStreamIn;
_metrics = metrics; _metrics = metrics;
} }
@@ -136,7 +135,7 @@ public class GrpcClientIdentificationService : GrpcBaseService
{ {
try try
{ {
using var stream = grpcIdentClientStreamOut.SendStreamIdentStatusChange(cancellationToken: cts); using var stream = _grpcIdentClientStreamOut.SendStreamIdentStatusChange(cancellationToken: cts);
_logger.LogInformation("Starting Send Online Client Data stream"); _logger.LogInformation("Starting Send Online Client Data stream");
await stream.RequestStream.WriteAsync(new IdentChangeMessage() await stream.RequestStream.WriteAsync(new IdentChangeMessage()
{ {
@@ -169,7 +168,7 @@ public class GrpcClientIdentificationService : GrpcBaseService
{ {
try try
{ {
using var stream = grpcIdentClientStreamIn.ReceiveStreamIdentStatusChange(new ServerMessage() using var stream = _grpcIdentClientStreamIn.ReceiveStreamIdentStatusChange(new ServerMessage()
{ {
ServerId = _shardName, ServerId = _shardName,
}); });

View File

@@ -8,7 +8,6 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using Microsoft.AspNetCore.SignalR; using Microsoft.AspNetCore.SignalR;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
@@ -25,7 +24,7 @@ public class SystemInfoService : IHostedService, IDisposable
private Timer _timer; private Timer _timer;
public SystemInfoDto SystemInfoDto { get; private set; } = new(); public SystemInfoDto SystemInfoDto { get; private set; } = new();
public SystemInfoService(MareMetrics mareMetrics, IConfiguration configuration, IServiceProvider services, GrpcClientIdentificationService clientIdentService, ILogger<SystemInfoService> logger, IHubContext<MareHub, IMareHub> hubContext) public SystemInfoService(MareMetrics mareMetrics, IServiceProvider services, GrpcClientIdentificationService clientIdentService, ILogger<SystemInfoService> logger, IHubContext<MareHub, IMareHub> hubContext)
{ {
_mareMetrics = mareMetrics; _mareMetrics = mareMetrics;
_services = services; _services = services;

View File

@@ -24,6 +24,7 @@ using System.Net.Http;
using MareSynchronosServer.Utils; using MareSynchronosServer.Utils;
using MareSynchronosServer.RequirementHandlers; using MareSynchronosServer.RequirementHandlers;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using MareSynchronosShared.Utils;
namespace MareSynchronosServer; namespace MareSynchronosServer;
@@ -40,6 +41,8 @@ public class Startup
{ {
services.AddHttpContextAccessor(); services.AddHttpContextAccessor();
services.Configure<ServerConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
services.Configure<MareConfigurationAuthBase>(Configuration.GetRequiredSection("MareSynchronos"));
services.Configure<IpRateLimitOptions>(Configuration.GetSection("IpRateLimiting")); services.Configure<IpRateLimitOptions>(Configuration.GetSection("IpRateLimiting"));
services.Configure<IpRateLimitPolicies>(Configuration.GetSection("IpRateLimitPolicies")); services.Configure<IpRateLimitPolicies>(Configuration.GetSection("IpRateLimitPolicies"));
services.AddTransient(_ => Configuration); services.AddTransient(_ => Configuration);
@@ -96,14 +99,14 @@ public class Startup
services.AddGrpcClient<FileService.FileServiceClient>(c => services.AddGrpcClient<FileService.FileServiceClient>(c =>
{ {
c.Address = new Uri(mareConfig.GetValue<string>("StaticFileServiceAddress")); c.Address = new Uri(mareConfig.GetValue<string>(nameof(ServerConfiguration.StaticFileServiceAddress)));
}).ConfigureChannel(c => }).ConfigureChannel(c =>
{ {
c.ServiceConfig = new ServiceConfig { MethodConfigs = { defaultMethodConfig } }; c.ServiceConfig = new ServiceConfig { MethodConfigs = { defaultMethodConfig } };
}); });
services.AddGrpcClient<IdentificationService.IdentificationServiceClient>(c => services.AddGrpcClient<IdentificationService.IdentificationServiceClient>(c =>
{ {
c.Address = new Uri(mareConfig.GetValue<string>("ServiceAddress")); c.Address = new Uri(mareConfig.GetValue<string>(nameof(ServerConfiguration.ServiceAddress)));
}).ConfigureChannel(c => }).ConfigureChannel(c =>
{ {
c.ServiceConfig = new ServiceConfig { MethodConfigs = { noRetryConfig } }; c.ServiceConfig = new ServiceConfig { MethodConfigs = { noRetryConfig } };
@@ -126,7 +129,7 @@ public class Startup
builder.MigrationsAssembly("MareSynchronosShared"); builder.MigrationsAssembly("MareSynchronosShared");
}).UseSnakeCaseNamingConvention(); }).UseSnakeCaseNamingConvention();
options.EnableThreadSafetyChecks(false); options.EnableThreadSafetyChecks(false);
}, mareConfig.GetValue("DbContextPoolSize", 1024)); }, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
services.AddAuthentication(SecretKeyAuthenticationHandler.AuthScheme) services.AddAuthentication(SecretKeyAuthenticationHandler.AuthScheme)
.AddScheme<AuthenticationSchemeOptions, SecretKeyAuthenticationHandler>(SecretKeyAuthenticationHandler.AuthScheme, options => { options.Validate(); }); .AddScheme<AuthenticationSchemeOptions, SecretKeyAuthenticationHandler>(SecretKeyAuthenticationHandler.AuthScheme, options => { options.Validate(); });
@@ -168,7 +171,7 @@ public class Startup
}); });
// add redis related options // add redis related options
var redis = mareConfig.GetValue("RedisConnectionString", string.Empty); var redis = mareConfig.GetValue(nameof(ServerConfiguration.RedisConnectionString), string.Empty);
if (!string.IsNullOrEmpty(redis)) if (!string.IsNullOrEmpty(redis))
{ {
signalRServiceBuilder.AddStackExchangeRedis(redis, options => signalRServiceBuilder.AddStackExchangeRedis(redis, options =>

View File

@@ -3,10 +3,10 @@ using MareSynchronosShared.Metrics;
using MareSynchronosShared.Models; using MareSynchronosShared.Models;
using MareSynchronosShared.Utils; using MareSynchronosShared.Utils;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
@@ -20,15 +20,15 @@ public class CleanupService : IHostedService, IDisposable
private readonly MareMetrics metrics; private readonly MareMetrics metrics;
private readonly ILogger<CleanupService> _logger; private readonly ILogger<CleanupService> _logger;
private readonly IServiceProvider _services; private readonly IServiceProvider _services;
private readonly IConfiguration _configuration; private readonly ServicesConfiguration _configuration;
private Timer? _timer; private Timer? _timer;
public CleanupService(MareMetrics metrics, ILogger<CleanupService> logger, IServiceProvider services, IConfiguration configuration) public CleanupService(MareMetrics metrics, ILogger<CleanupService> logger, IServiceProvider services, IOptions<ServicesConfiguration> configuration)
{ {
this.metrics = metrics; this.metrics = metrics;
_logger = logger; _logger = logger;
_services = services; _services = services;
_configuration = configuration.GetRequiredSection("MareSynchronos"); _configuration = configuration.Value;
} }
public Task StartAsync(CancellationToken cancellationToken) public Task StartAsync(CancellationToken cancellationToken)
@@ -68,17 +68,9 @@ public class CleanupService : IHostedService, IDisposable
try try
{ {
if (!bool.TryParse(_configuration["PurgeUnusedAccounts"], out var purgeUnusedAccounts)) if (_configuration.PurgeUnusedAccounts)
{ {
purgeUnusedAccounts = false; var usersOlderThanDays = _configuration.PurgeUnusedAccountsPeriodInDays;
}
if (purgeUnusedAccounts)
{
if (!int.TryParse(_configuration["PurgeUnusedAccountsPeriodInDays"], out var usersOlderThanDays))
{
usersOlderThanDays = 14;
}
_logger.LogInformation("Cleaning up users older than {usersOlderThanDays} days", usersOlderThanDays); _logger.LogInformation("Cleaning up users older than {usersOlderThanDays} days", usersOlderThanDays);
@@ -160,7 +152,7 @@ public class CleanupService : IHostedService, IDisposable
} }
else else
{ {
_ = await SharedDbFunctions.MigrateOrDeleteGroup(dbContext, userGroupPair.Group, groupPairs, _configuration.GetValue<int>("MaxExistingGroupsByUser", 3)).ConfigureAwait(false); _ = await SharedDbFunctions.MigrateOrDeleteGroup(dbContext, userGroupPair.Group, groupPairs, _configuration.MaxExistingGroupsByUser).ConfigureAwait(false);
} }
} }

View File

@@ -10,10 +10,10 @@ using Discord.WebSocket;
using MareSynchronosServices.Identity; using MareSynchronosServices.Identity;
using MareSynchronosShared.Data; using MareSynchronosShared.Data;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace MareSynchronosServices.Discord; namespace MareSynchronosServices.Discord;
@@ -22,23 +22,20 @@ internal class DiscordBot : IHostedService
private readonly DiscordBotServices _botServices; private readonly DiscordBotServices _botServices;
private readonly IdentityHandler _identityHandler; private readonly IdentityHandler _identityHandler;
private readonly IServiceProvider _services; private readonly IServiceProvider _services;
private readonly IConfiguration _configuration; private readonly ServicesConfiguration _configuration;
private readonly ILogger<DiscordBot> _logger; private readonly ILogger<DiscordBot> _logger;
private string _discordAuthToken = string.Empty;
private readonly DiscordSocketClient _discordClient; private readonly DiscordSocketClient _discordClient;
private CancellationTokenSource? _updateStatusCts; private CancellationTokenSource? _updateStatusCts;
private CancellationTokenSource? _vanityUpdateCts; private CancellationTokenSource? _vanityUpdateCts;
public DiscordBot(DiscordBotServices botServices, IdentityHandler identityHandler, IServiceProvider services, IConfiguration configuration, ILogger<DiscordBot> logger) public DiscordBot(DiscordBotServices botServices, IdentityHandler identityHandler, IServiceProvider services, IOptions<ServicesConfiguration> configuration, ILogger<DiscordBot> logger)
{ {
_botServices = botServices; _botServices = botServices;
_identityHandler = identityHandler; _identityHandler = identityHandler;
_services = services; _services = services;
_configuration = configuration.GetRequiredSection("MareSynchronos"); _configuration = configuration.Value;
_logger = logger; _logger = logger;
_discordAuthToken = _configuration.GetValue<string>("DiscordBotToken");
_discordClient = new(new DiscordSocketConfig() _discordClient = new(new DiscordSocketConfig()
{ {
DefaultRetryMode = RetryMode.AlwaysRetry DefaultRetryMode = RetryMode.AlwaysRetry
@@ -188,11 +185,9 @@ internal class DiscordBot : IHostedService
public async Task StartAsync(CancellationToken cancellationToken) public async Task StartAsync(CancellationToken cancellationToken)
{ {
if (!string.IsNullOrEmpty(_discordAuthToken)) if (!string.IsNullOrEmpty(_configuration.DiscordBotToken))
{ {
_discordAuthToken = _configuration.GetValue<string>("DiscordBotToken"); await _discordClient.LoginAsync(TokenType.Bot, _configuration.DiscordBotToken).ConfigureAwait(false);
await _discordClient.LoginAsync(TokenType.Bot, _discordAuthToken).ConfigureAwait(false);
await _discordClient.StartAsync().ConfigureAwait(false); await _discordClient.StartAsync().ConfigureAwait(false);
_discordClient.Ready += DiscordClient_Ready; _discordClient.Ready += DiscordClient_Ready;
@@ -204,7 +199,7 @@ internal class DiscordBot : IHostedService
public async Task StopAsync(CancellationToken cancellationToken) public async Task StopAsync(CancellationToken cancellationToken)
{ {
if (!string.IsNullOrEmpty(_discordAuthToken)) if (!string.IsNullOrEmpty(_configuration.DiscordBotToken))
{ {
await _botServices.Stop(); await _botServices.Stop();
_updateStatusCts?.Cancel(); _updateStatusCts?.Cancel();

View File

@@ -3,9 +3,9 @@ using System.Threading.Tasks;
using System.Collections.Generic; using System.Collections.Generic;
using System.Collections.Concurrent; using System.Collections.Concurrent;
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using System.Threading; using System.Threading;
using Microsoft.Extensions.Options;
namespace MareSynchronosServices.Discord; namespace MareSynchronosServices.Discord;
@@ -19,15 +19,15 @@ public class DiscordBotServices
public readonly string[] LodestoneServers = new[] { "eu", "na", "jp", "fr", "de" }; public readonly string[] LodestoneServers = new[] { "eu", "na", "jp", "fr", "de" };
private readonly IServiceProvider _serviceProvider; private readonly IServiceProvider _serviceProvider;
public IConfiguration Configuration { get; init; } public ServicesConfiguration Configuration { get; init; }
public ILogger<DiscordBotServices> Logger { get; init; } public ILogger<DiscordBotServices> Logger { get; init; }
public MareMetrics Metrics { get; init; } public MareMetrics Metrics { get; init; }
public Random Random { get; init; } public Random Random { get; init; }
private CancellationTokenSource? verificationTaskCts; private CancellationTokenSource? verificationTaskCts;
public DiscordBotServices(IConfiguration configuration, IServiceProvider serviceProvider, ILogger<DiscordBotServices> logger, MareMetrics metrics) public DiscordBotServices(IOptions<ServicesConfiguration> configuration, IServiceProvider serviceProvider, ILogger<DiscordBotServices> logger, MareMetrics metrics)
{ {
Configuration = configuration.GetRequiredSection("MareSynchronos"); Configuration = configuration.Value;
_serviceProvider = serviceProvider; _serviceProvider = serviceProvider;
Logger = logger; Logger = logger;
Metrics = metrics; Metrics = metrics;

View File

@@ -12,7 +12,6 @@ using Prometheus;
using MareSynchronosShared.Models; using MareSynchronosShared.Models;
using MareSynchronosServices.Identity; using MareSynchronosServices.Identity;
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using Microsoft.Extensions.Configuration;
using System.Net.Http; using System.Net.Http;
using MareSynchronosShared.Utils; using MareSynchronosShared.Utils;
using System.Collections.Generic; using System.Collections.Generic;
@@ -758,9 +757,9 @@ public class MareModule : InteractionModuleBase
user.IsAdmin = true; user.IsAdmin = true;
} }
if (_botServices.Configuration.GetValue<bool>("PurgeUnusedAccounts")) if (_botServices.Configuration.PurgeUnusedAccounts)
{ {
var purgedDays = _botServices.Configuration.GetValue<int>("PurgeUnusedAccountsPeriodInDays"); var purgedDays = _botServices.Configuration.PurgeUnusedAccountsPeriodInDays;
user.LastLoggedIn = DateTime.UtcNow - TimeSpan.FromDays(purgedDays) + TimeSpan.FromDays(1); user.LastLoggedIn = DateTime.UtcNow - TimeSpan.FromDays(purgedDays) + TimeSpan.FromDays(1);
} }

View File

@@ -5,6 +5,7 @@ using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System; using System;
using System.Linq; using System.Linq;
@@ -22,6 +23,11 @@ public class Program
var metrics = services.GetRequiredService<MareMetrics>(); var metrics = services.GetRequiredService<MareMetrics>();
metrics.SetGaugeTo(MetricsAPI.GaugeUsersRegistered, dbContext.Users.Count()); metrics.SetGaugeTo(MetricsAPI.GaugeUsersRegistered, dbContext.Users.Count());
var options = host.Services.GetService<IOptions<ServicesConfiguration>>();
var logger = host.Services.GetService<ILogger<Program>>();
logger.LogInformation("Loaded MareSynchronos Services Configuration");
logger.LogInformation(options.Value.ToString());
} }
host.Run(); host.Run();

View File

@@ -0,0 +1,23 @@
using MareSynchronosShared.Utils;
using System.Text;
namespace MareSynchronosServices;
public class ServicesConfiguration : MareConfigurationBase
{
public string DiscordBotToken { get; set; } = string.Empty;
public bool PurgeUnusedAccounts { get; set; } = false;
public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14;
public int MaxExistingGroupsByUser { get; set; } = 3;
public override string ToString()
{
StringBuilder sb = new();
sb.AppendLine(base.ToString());
sb.AppendLine($"{nameof(DiscordBotToken)} => {DiscordBotToken}");
sb.AppendLine($"{nameof(PurgeUnusedAccounts)} => {PurgeUnusedAccounts}");
sb.AppendLine($"{nameof(PurgeUnusedAccountsPeriodInDays)} => {PurgeUnusedAccountsPeriodInDays}");
sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");
return sb.ToString();
}
}

View File

@@ -10,6 +10,7 @@ using Prometheus;
using System.Collections.Generic; using System.Collections.Generic;
using MareSynchronosServices.Identity; using MareSynchronosServices.Identity;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using MareSynchronosShared.Utils;
namespace MareSynchronosServices; namespace MareSynchronosServices;
@@ -31,7 +32,7 @@ public class Startup
builder.MigrationsHistoryTable("_efmigrationshistory", "public"); builder.MigrationsHistoryTable("_efmigrationshistory", "public");
}).UseSnakeCaseNamingConvention(); }).UseSnakeCaseNamingConvention();
options.EnableThreadSafetyChecks(false); options.EnableThreadSafetyChecks(false);
}, Configuration.GetValue("DbContextPoolSize", 1024)); }, Configuration.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
services.AddSingleton(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string> { services.AddSingleton(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string> {
}, new List<string> }, new List<string>
@@ -39,7 +40,8 @@ public class Startup
MetricsAPI.GaugeUsersRegistered MetricsAPI.GaugeUsersRegistered
})); }));
services.AddTransient(_ => Configuration); services.Configure<ServicesConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
services.AddSingleton(Configuration);
services.AddSingleton<DiscordBotServices>(); services.AddSingleton<DiscordBotServices>();
services.AddSingleton<IdentityHandler>(); services.AddSingleton<IdentityHandler>();
services.AddSingleton<CleanupService>(); services.AddSingleton<CleanupService>();

View File

@@ -3,9 +3,9 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using MareSynchronosShared.Utils; using MareSynchronosShared.Utils;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace MareSynchronosShared.Authentication; namespace MareSynchronosShared.Authentication;
@@ -20,15 +20,13 @@ public class SecretKeyAuthenticatorService
private readonly int _tempBanMinutes; private readonly int _tempBanMinutes;
private readonly List<string> _whitelistedIps; private readonly List<string> _whitelistedIps;
public SecretKeyAuthenticatorService(MareMetrics metrics, IServiceScopeFactory serviceScopeFactory, IConfiguration configuration, ILogger<SecretKeyAuthenticatorService> logger) public SecretKeyAuthenticatorService(MareMetrics metrics, IServiceScopeFactory serviceScopeFactory, IOptions<MareConfigurationAuthBase> configuration, ILogger<SecretKeyAuthenticatorService> logger)
{ {
_logger = logger; _logger = logger;
var config = configuration.GetRequiredSection("MareSynchronos"); var config = configuration.Value;
_failedAttemptsForTempBan = config.GetValue<int>("FailedAuthForTempBan", 5); _failedAttemptsForTempBan = config.FailedAuthForTempBan;
logger.LogInformation("FailedAuthForTempBan: {num}", _failedAttemptsForTempBan); _tempBanMinutes = config.TempBanDurationInMinutes;
_tempBanMinutes = config.GetValue<int>("TempBanDurationInMinutes", 30); _whitelistedIps = config.WhitelistedIps;
logger.LogInformation("TempBanMinutes: {num}", _tempBanMinutes);
_whitelistedIps = config.GetSection("WhitelistedIps").Get<List<string>>();
foreach (var ip in _whitelistedIps) foreach (var ip in _whitelistedIps)
{ {
logger.LogInformation("Whitelisted IP: " + ip); logger.LogInformation("Whitelisted IP: " + ip);

View File

@@ -0,0 +1,32 @@
using System.Text;
namespace MareSynchronosShared.Utils;
public class MareConfigurationBase
{
public int DbContextPoolSize { get; set; } = 100;
public override string ToString()
{
StringBuilder sb = new();
sb.AppendLine($"{nameof(DbContextPoolSize)} => {DbContextPoolSize}");
return sb.ToString();
}
}
public class MareConfigurationAuthBase : MareConfigurationBase
{
public int DbContextPoolSize { get; set; } = 100;
public int FailedAuthForTempBan { get; set; } = 5;
public int TempBanDurationInMinutes { get; set; } = 5;
public List<string> WhitelistedIps { get; set; } = new();
public override string ToString()
{
StringBuilder sb = new();
sb.AppendLine($"{nameof(FailedAuthForTempBan)} => {FailedAuthForTempBan}");
sb.AppendLine($"{nameof(TempBanDurationInMinutes)} => {TempBanDurationInMinutes}");
sb.AppendLine($"{nameof(WhitelistedIps)} => {string.Join(", ", WhitelistedIps)}");
return sb.ToString();
}
}

View File

@@ -1,4 +1,5 @@
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using Microsoft.Extensions.Options;
using System.Collections.Concurrent; using System.Collections.Concurrent;
namespace MareSynchronosStaticFilesServer; namespace MareSynchronosStaticFilesServer;
@@ -13,14 +14,13 @@ public class CachedFileProvider
private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal); private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
private bool IsMainServer => _remoteCacheSourceUri == null; private bool IsMainServer => _remoteCacheSourceUri == null;
public CachedFileProvider(IConfiguration configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics) public CachedFileProvider(IOptions<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics)
{ {
_logger = logger; _logger = logger;
_fileStatisticsService = fileStatisticsService; _fileStatisticsService = fileStatisticsService;
_metrics = metrics; _metrics = metrics;
var configurationSection = configuration.GetRequiredSection("MareSynchronos"); _remoteCacheSourceUri = configuration.Value.RemoteCacheSourceUri;
_remoteCacheSourceUri = configurationSection.GetValue<Uri>("RemoteCacheSourceUri", null); _basePath = configuration.Value.CacheDirectory;
_basePath = configurationSection["CacheDirectory"];
} }
public async Task<FileStream?> GetFileStream(string hash, string auth) public async Task<FileStream?> GetFileStream(string hash, string auth)

View File

@@ -2,7 +2,7 @@
using MareSynchronosShared.Data; using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using MareSynchronosShared.Models; using MareSynchronosShared.Models;
using System.Globalization; using Microsoft.Extensions.Options;
namespace MareSynchronosStaticFilesServer; namespace MareSynchronosStaticFilesServer;
@@ -11,19 +11,19 @@ public class FileCleanupService : IHostedService
private readonly MareMetrics _metrics; private readonly MareMetrics _metrics;
private readonly ILogger<FileCleanupService> _logger; private readonly ILogger<FileCleanupService> _logger;
private readonly IServiceProvider _services; private readonly IServiceProvider _services;
private readonly IConfiguration _configuration; private readonly StaticFilesServerConfiguration _configuration;
private readonly bool _isMainServer; private readonly bool _isMainServer;
private readonly string _cacheDir; private readonly string _cacheDir;
private CancellationTokenSource _cleanupCts; private CancellationTokenSource _cleanupCts;
public FileCleanupService(MareMetrics metrics, ILogger<FileCleanupService> logger, IServiceProvider services, IConfiguration configuration) public FileCleanupService(MareMetrics metrics, ILogger<FileCleanupService> logger, IServiceProvider services, IOptions<StaticFilesServerConfiguration> configuration)
{ {
_metrics = metrics; _metrics = metrics;
_logger = logger; _logger = logger;
_services = services; _services = services;
_configuration = configuration.GetRequiredSection("MareSynchronos"); _configuration = configuration.Value;
_isMainServer = string.IsNullOrEmpty(_configuration.GetValue("RemoteCacheSourceUri", string.Empty)); _isMainServer = _configuration.RemoteCacheSourceUri == null;
_cacheDir = _configuration.GetValue<string>("CacheDirectory"); _cacheDir = _configuration.CacheDirectory;
} }
public Task StartAsync(CancellationToken cancellationToken) public Task StartAsync(CancellationToken cancellationToken)
@@ -67,21 +67,19 @@ public class FileCleanupService : IHostedService
private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct) private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct)
{ {
var cacheSizeLimitInGiB = _configuration.GetValue<double>("CacheSizeHardLimitInGiB", -1); if (_configuration.CacheSizeHardLimitInGiB <= 0)
if (cacheSizeLimitInGiB <= 0)
{ {
return; return;
} }
try try
{ {
_logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", cacheSizeLimitInGiB); _logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", _configuration.CacheSizeHardLimitInGiB);
var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories) var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories)
.Select(f => new FileInfo(f)).ToList() .Select(f => new FileInfo(f)).ToList()
.OrderBy(f => f.LastAccessTimeUtc).ToList(); .OrderBy(f => f.LastAccessTimeUtc).ToList();
var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length); var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(cacheSizeLimitInGiB).Bytes; long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(_configuration.CacheSizeHardLimitInGiB).Bytes;
while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested) while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested)
{ {
var oldestFile = allLocalFiles[0]; var oldestFile = allLocalFiles[0];
@@ -108,18 +106,15 @@ public class FileCleanupService : IHostedService
{ {
try try
{ {
var filesOlderThanDays = _configuration.GetValue("UnusedFileRetentionPeriodInDays", 7); _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", _configuration.UnusedFileRetentionPeriodInDays);
var forcedDeletionHours = _configuration.GetValue("ForcedDeletionOfFilesAfterHours", -1); if (_configuration.ForcedDeletionOfFilesAfterHours > 0)
_logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", filesOlderThanDays);
if (forcedDeletionHours > 0)
{ {
_logger.LogInformation("Cleaning up files written to longer than {hours}h ago", forcedDeletionHours); _logger.LogInformation("Cleaning up files written to longer than {hours}h ago", _configuration.ForcedDeletionOfFilesAfterHours);
} }
// clean up files in DB but not on disk or last access is expired // clean up files in DB but not on disk or last access is expired
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays)); var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(_configuration.UnusedFileRetentionPeriodInDays));
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionHours)); var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(_configuration.ForcedDeletionOfFilesAfterHours));
var allFiles = dbContext.Files.ToList(); var allFiles = dbContext.Files.ToList();
foreach (var fileCache in allFiles.Where(f => f.Uploaded)) foreach (var fileCache in allFiles.Where(f => f.Uploaded))
{ {
@@ -138,7 +133,7 @@ public class FileCleanupService : IHostedService
if (_isMainServer) if (_isMainServer)
dbContext.Files.Remove(fileCache); dbContext.Files.Remove(fileCache);
} }
else if (file != null && forcedDeletionHours > 0 && file.LastWriteTime < prevTimeForcedDeletion) else if (file != null && _configuration.ForcedDeletionOfFilesAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
{ {
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal); _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);

View File

@@ -3,6 +3,7 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using MareSynchronosShared.Protos; using MareSynchronosShared.Protos;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
namespace MareSynchronosStaticFilesServer; namespace MareSynchronosStaticFilesServer;
@@ -13,9 +14,9 @@ public class GrpcFileService : FileService.FileServiceBase
private readonly ILogger<GrpcFileService> _logger; private readonly ILogger<GrpcFileService> _logger;
private readonly MareMetrics _metricsClient; private readonly MareMetrics _metricsClient;
public GrpcFileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger<GrpcFileService> logger, MareMetrics metricsClient) public GrpcFileService(MareDbContext mareDbContext, IOptions<StaticFilesServerConfiguration> configuration, ILogger<GrpcFileService> logger, MareMetrics metricsClient)
{ {
_basePath = configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"]; _basePath = configuration.Value.CacheDirectory;
_mareDbContext = mareDbContext; _mareDbContext = mareDbContext;
_logger = logger; _logger = logger;
_metricsClient = metricsClient; _metricsClient = metricsClient;

View File

@@ -1,3 +1,5 @@
using Microsoft.Extensions.Options;
namespace MareSynchronosStaticFilesServer; namespace MareSynchronosStaticFilesServer;
public class Program public class Program
@@ -7,6 +9,14 @@ public class Program
var hostBuilder = CreateHostBuilder(args); var hostBuilder = CreateHostBuilder(args);
var host = hostBuilder.Build(); var host = hostBuilder.Build();
using (var scope = host.Services.CreateScope())
{
var options = host.Services.GetService<IOptions<StaticFilesServerConfiguration>>();
var logger = host.Services.GetService<ILogger<Program>>();
logger.LogInformation("Loaded MareSynchronos Static Files Server Configuration");
logger.LogInformation(options.Value.ToString());
}
host.Run(); host.Run();
} }

View File

@@ -1,7 +1,7 @@
using Grpc.Net.Client.Configuration;
using MareSynchronosShared.Authentication; using MareSynchronosShared.Authentication;
using MareSynchronosShared.Data; using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using MareSynchronosShared.Utils;
using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Authorization;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
@@ -16,7 +16,7 @@ public class Startup
{ {
Configuration = configuration; Configuration = configuration;
var mareSettings = Configuration.GetRequiredSection("MareSynchronos"); var mareSettings = Configuration.GetRequiredSection("MareSynchronos");
_isMain = string.IsNullOrEmpty(mareSettings.GetValue("RemoteCacheSourceUri", string.Empty)); _isMain = string.IsNullOrEmpty(mareSettings.GetValue(nameof(StaticFilesServerConfiguration.RemoteCacheSourceUri), string.Empty));
} }
public IConfiguration Configuration { get; } public IConfiguration Configuration { get; }
@@ -25,10 +25,12 @@ public class Startup
{ {
services.AddHttpContextAccessor(); services.AddHttpContextAccessor();
services.AddTransient(_ => Configuration);
services.AddLogging(); services.AddLogging();
services.Configure<MareConfigurationAuthBase>(Configuration.GetRequiredSection("MareSynchronos"));
services.Configure<StaticFilesServerConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
services.AddSingleton(Configuration);
var mareSettings = Configuration.GetRequiredSection("MareSynchronos"); var mareSettings = Configuration.GetRequiredSection("MareSynchronos");
services.AddControllers(); services.AddControllers();
@@ -62,7 +64,7 @@ public class Startup
builder.MigrationsHistoryTable("_efmigrationshistory", "public"); builder.MigrationsHistoryTable("_efmigrationshistory", "public");
}).UseSnakeCaseNamingConvention(); }).UseSnakeCaseNamingConvention();
options.EnableThreadSafetyChecks(false); options.EnableThreadSafetyChecks(false);
}, mareSettings.GetValue("DbContextPoolSize", 1024)); }, mareSettings.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
services.AddAuthentication(options => services.AddAuthentication(options =>
{ {

View File

@@ -0,0 +1,25 @@
using MareSynchronosShared.Utils;
using System.Text;
namespace MareSynchronosStaticFilesServer;
public class StaticFilesServerConfiguration : MareConfigurationAuthBase
{
public int ForcedDeletionOfFilesAfterHours { get; set; } = -1;
public double CacheSizeHardLimitInGiB { get; set; } = -1;
public int UnusedFileRetentionPeriodInDays { get; set; } = -1;
public string CacheDirectory { get; set; }
public Uri? RemoteCacheSourceUri { get; set; } = null;
public override string ToString()
{
StringBuilder sb = new();
sb.AppendLine(base.ToString());
sb.AppendLine($"{nameof(ForcedDeletionOfFilesAfterHours)} => {ForcedDeletionOfFilesAfterHours}");
sb.AppendLine($"{nameof(CacheSizeHardLimitInGiB)} => {CacheSizeHardLimitInGiB}");
sb.AppendLine($"{nameof(UnusedFileRetentionPeriodInDays)} => {UnusedFileRetentionPeriodInDays}");
sb.AppendLine($"{nameof(CacheDirectory)} => {CacheDirectory}");
sb.AppendLine($"{nameof(RemoteCacheSourceUri)} => {RemoteCacheSourceUri}");
return sb.ToString();
}
}