t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

switch around all iconfiguration usage to ioptions

Browse Source
This commit is contained in:
rootdarkarchon
2022-12-23 01:53:21 +01:00
parent b8d61b38bd
commit 7ee7fdaf48
21 changed files with 218 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs
View File

@@ -12,8 +12,8 @@ using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.SignalR;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace MareSynchronosServer.Hubs;
@@ -33,18 +33,18 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
private readonly int _maxGroupUserCount;
public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient,
MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService, IConfiguration configuration, IHttpContextAccessor contextAccessor,
MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService, IOptions<ServerConfiguration> configuration, IHttpContextAccessor contextAccessor,
GrpcClientIdentificationService clientIdentService)
{
_mareMetrics = mareMetrics;
_fileServiceClient = fileServiceClient;
_systemInfoService = systemInfoService;
var config = configuration.GetRequiredSection("MareSynchronos");
_cdnFullUri = new Uri(config.GetValue<string>("CdnFullUrl"));
_shardName = config.GetValue("ShardName", "Main");
_maxExistingGroupsByUser = config.GetValue<int>("MaxExistingGroupsByUser", 3);
_maxJoinedGroupsByUser = config.GetValue<int>("MaxJoinedGroupsByUser", 6);
_maxGroupUserCount = config.GetValue<int>("MaxGroupUserCount", 100);
var config = configuration.Value;
_cdnFullUri = config.CdnFullUrl;
_shardName = config.ShardName;
_maxExistingGroupsByUser = config.MaxExistingGroupsByUser;
_maxJoinedGroupsByUser = config.MaxJoinedGroupsByUser;
_maxGroupUserCount = config.MaxGroupUserCount;
_contextAccessor = contextAccessor;
_clientIdentService = clientIdentService;
_logger = new MareHubLogger(this, logger);

6
MareSynchronosServer/MareSynchronosServer/Program.cs
View File

@@ -8,6 +8,7 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using Microsoft.Extensions.Options;
namespace MareSynchronosServer;
@@ -40,6 +41,11 @@ public class Program
metrics.SetGaugeTo(MetricsAPI.GaugePairs, context.ClientPairs.Count());
metrics.SetGaugeTo(MetricsAPI.GaugePairsPaused, context.ClientPairs.Count(p => p.IsPaused));
var options = host.Services.GetService<IOptions<ServerConfiguration>>();
var logger = host.Services.GetService<ILogger<Program>>();
logger.LogInformation("Loaded MareSynchronos Server Configuration");
logger.LogInformation(options.Value.ToString());
}
if (args.Length == 0 || !string.Equals(args[0], "dry", StringComparison.Ordinal))

32
MareSynchronosServer/MareSynchronosServer/ServerConfiguration.cs Normal file
View File

@@ -0,0 +1,32 @@
using MareSynchronosShared.Utils;
using System;
using System.Text;
namespace MareSynchronosServer;
public class ServerConfiguration : MareConfigurationAuthBase
{
public Uri CdnFullUrl { get; set; } = null;
public Uri ServiceAddress { get; set; } = null;
public Uri StaticFileServiceAddress { get; set; } = null;
public string RedisConnectionString { get; set; } = string.Empty;
public int MaxExistingGroupsByUser { get; set; } = 3;
public int MaxJoinedGroupsByUser { get; set; } = 6;
public int MaxGroupUserCount { get; set; } = 100;
public string ShardName { get; set; } = string.Empty;
public override string ToString()
{
StringBuilder sb = new();
sb.AppendLine(base.ToString());
sb.AppendLine($"{nameof(ShardName)} => {ShardName}");
sb.AppendLine($"{nameof(CdnFullUrl)} => {CdnFullUrl}");
sb.AppendLine($"{nameof(ServiceAddress)} => {ServiceAddress}");
sb.AppendLine($"{nameof(StaticFileServiceAddress)} => {StaticFileServiceAddress}");
sb.AppendLine($"{nameof(RedisConnectionString)} => {RedisConnectionString}");
sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");
sb.AppendLine($"{nameof(MaxJoinedGroupsByUser)} => {MaxJoinedGroupsByUser}");
sb.AppendLine($"{nameof(MaxGroupUserCount)} => {MaxGroupUserCount}");
return sb.ToString();
}
}

19
MareSynchronosServer/MareSynchronosServer/Services/GrpcClientIdentificationService.cs
View File

@@ -2,8 +2,8 @@
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Protos;
using MareSynchronosShared.Services;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System;
using System.Collections.Concurrent;
using System.Linq;
@@ -17,8 +17,8 @@ public class GrpcClientIdentificationService : GrpcBaseService
private readonly string _shardName;
private readonly ILogger<GrpcClientIdentificationService> _logger;
private readonly IdentificationService.IdentificationServiceClient _grpcIdentClient;
private readonly IdentificationService.IdentificationServiceClient grpcIdentClientStreamOut;
private readonly IdentificationService.IdentificationServiceClient grpcIdentClientStreamIn;
private readonly IdentificationService.IdentificationServiceClient _grpcIdentClientStreamOut;
private readonly IdentificationService.IdentificationServiceClient _grpcIdentClientStreamIn;
private readonly MareMetrics _metrics;
protected readonly ConcurrentDictionary<string, UidWithIdent> OnlineClients = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, UidWithIdent> RemoteCachedIdents = new(StringComparer.Ordinal);
@@ -26,14 +26,13 @@ public class GrpcClientIdentificationService : GrpcBaseService
public GrpcClientIdentificationService(ILogger<GrpcClientIdentificationService> logger, IdentificationService.IdentificationServiceClient gprcIdentClient,
IdentificationService.IdentificationServiceClient gprcIdentClientStreamOut,
IdentificationService.IdentificationServiceClient gprcIdentClientStreamIn, MareMetrics metrics, IConfiguration configuration) : base(logger)
IdentificationService.IdentificationServiceClient gprcIdentClientStreamIn, MareMetrics metrics, IOptions<ServerConfiguration> configuration) : base(logger)
{
var config = configuration.GetSection("MareSynchronos");
_shardName = config.GetValue("ShardName", "Main");
_shardName = configuration.Value.ShardName;
_logger = logger;
_grpcIdentClient = gprcIdentClient;
this.grpcIdentClientStreamOut = gprcIdentClientStreamOut;
this.grpcIdentClientStreamIn = gprcIdentClientStreamIn;
_grpcIdentClientStreamOut = gprcIdentClientStreamOut;
_grpcIdentClientStreamIn = gprcIdentClientStreamIn;
_metrics = metrics;
}
@@ -136,7 +135,7 @@ public class GrpcClientIdentificationService : GrpcBaseService
{
try
{
using var stream = grpcIdentClientStreamOut.SendStreamIdentStatusChange(cancellationToken: cts);
using var stream = _grpcIdentClientStreamOut.SendStreamIdentStatusChange(cancellationToken: cts);
_logger.LogInformation("Starting Send Online Client Data stream");
await stream.RequestStream.WriteAsync(new IdentChangeMessage()
{
@@ -169,7 +168,7 @@ public class GrpcClientIdentificationService : GrpcBaseService
{
try
{
using var stream = grpcIdentClientStreamIn.ReceiveStreamIdentStatusChange(new ServerMessage()
using var stream = _grpcIdentClientStreamIn.ReceiveStreamIdentStatusChange(new ServerMessage()
{
ServerId = _shardName,
});

3
MareSynchronosServer/MareSynchronosServer/Services/SystemInfoService.cs
View File

@@ -8,7 +8,6 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using Microsoft.AspNetCore.SignalR;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
@@ -25,7 +24,7 @@ public class SystemInfoService : IHostedService, IDisposable
private Timer _timer;
public SystemInfoDto SystemInfoDto { get; private set; } = new();
public SystemInfoService(MareMetrics mareMetrics, IConfiguration configuration, IServiceProvider services, GrpcClientIdentificationService clientIdentService, ILogger<SystemInfoService> logger, IHubContext<MareHub, IMareHub> hubContext)
public SystemInfoService(MareMetrics mareMetrics, IServiceProvider services, GrpcClientIdentificationService clientIdentService, ILogger<SystemInfoService> logger, IHubContext<MareHub, IMareHub> hubContext)
{
_mareMetrics = mareMetrics;
_services = services;

11
MareSynchronosServer/MareSynchronosServer/Startup.cs
View File

@@ -24,6 +24,7 @@ using System.Net.Http;
using MareSynchronosServer.Utils;
using MareSynchronosServer.RequirementHandlers;
using Microsoft.Extensions.Logging;
using MareSynchronosShared.Utils;
namespace MareSynchronosServer;
@@ -40,6 +41,8 @@ public class Startup
{
services.AddHttpContextAccessor();
services.Configure<ServerConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
services.Configure<MareConfigurationAuthBase>(Configuration.GetRequiredSection("MareSynchronos"));
services.Configure<IpRateLimitOptions>(Configuration.GetSection("IpRateLimiting"));
services.Configure<IpRateLimitPolicies>(Configuration.GetSection("IpRateLimitPolicies"));
services.AddTransient(_ => Configuration);
@@ -96,14 +99,14 @@ public class Startup
services.AddGrpcClient<FileService.FileServiceClient>(c =>
{
c.Address = new Uri(mareConfig.GetValue<string>("StaticFileServiceAddress"));
c.Address = new Uri(mareConfig.GetValue<string>(nameof(ServerConfiguration.StaticFileServiceAddress)));
}).ConfigureChannel(c =>
{
c.ServiceConfig = new ServiceConfig { MethodConfigs = { defaultMethodConfig } };
});
services.AddGrpcClient<IdentificationService.IdentificationServiceClient>(c =>
{
c.Address = new Uri(mareConfig.GetValue<string>("ServiceAddress"));
c.Address = new Uri(mareConfig.GetValue<string>(nameof(ServerConfiguration.ServiceAddress)));
}).ConfigureChannel(c =>
{
c.ServiceConfig = new ServiceConfig { MethodConfigs = { noRetryConfig } };
@@ -126,7 +129,7 @@ public class Startup
builder.MigrationsAssembly("MareSynchronosShared");
}).UseSnakeCaseNamingConvention();
options.EnableThreadSafetyChecks(false);
}, mareConfig.GetValue("DbContextPoolSize", 1024));
}, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
services.AddAuthentication(SecretKeyAuthenticationHandler.AuthScheme)
.AddScheme<AuthenticationSchemeOptions, SecretKeyAuthenticationHandler>(SecretKeyAuthenticationHandler.AuthScheme, options => { options.Validate(); });
@@ -168,7 +171,7 @@ public class Startup
});
// add redis related options
var redis = mareConfig.GetValue("RedisConnectionString", string.Empty);
var redis = mareConfig.GetValue(nameof(ServerConfiguration.RedisConnectionString), string.Empty);
if (!string.IsNullOrEmpty(redis))
{
signalRServiceBuilder.AddStackExchangeRedis(redis, options =>

22
MareSynchronosServer/MareSynchronosServices/CleanupService.cs
View File

@@ -3,10 +3,10 @@ using MareSynchronosShared.Metrics;
using MareSynchronosShared.Models;
using MareSynchronosShared.Utils;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System;
using System.Collections.Generic;
using System.Linq;
@@ -20,15 +20,15 @@ public class CleanupService : IHostedService, IDisposable
private readonly MareMetrics metrics;
private readonly ILogger<CleanupService> _logger;
private readonly IServiceProvider _services;
private readonly IConfiguration _configuration;
private readonly ServicesConfiguration _configuration;
private Timer? _timer;
public CleanupService(MareMetrics metrics, ILogger<CleanupService> logger, IServiceProvider services, IConfiguration configuration)
public CleanupService(MareMetrics metrics, ILogger<CleanupService> logger, IServiceProvider services, IOptions<ServicesConfiguration> configuration)
{
this.metrics = metrics;
_logger = logger;
_services = services;
_configuration = configuration.GetRequiredSection("MareSynchronos");
_configuration = configuration.Value;
}
public Task StartAsync(CancellationToken cancellationToken)
@@ -68,17 +68,9 @@ public class CleanupService : IHostedService, IDisposable
try
{
if (!bool.TryParse(_configuration["PurgeUnusedAccounts"], out var purgeUnusedAccounts))
if (_configuration.PurgeUnusedAccounts)
{
purgeUnusedAccounts = false;
}
if (purgeUnusedAccounts)
{
if (!int.TryParse(_configuration["PurgeUnusedAccountsPeriodInDays"], out var usersOlderThanDays))
{
usersOlderThanDays = 14;
}
var usersOlderThanDays = _configuration.PurgeUnusedAccountsPeriodInDays;
_logger.LogInformation("Cleaning up users older than {usersOlderThanDays} days", usersOlderThanDays);
@@ -160,7 +152,7 @@ public class CleanupService : IHostedService, IDisposable
}
else
{
_ = await SharedDbFunctions.MigrateOrDeleteGroup(dbContext, userGroupPair.Group, groupPairs, _configuration.GetValue<int>("MaxExistingGroupsByUser", 3)).ConfigureAwait(false);
_ = await SharedDbFunctions.MigrateOrDeleteGroup(dbContext, userGroupPair.Group, groupPairs, _configuration.MaxExistingGroupsByUser).ConfigureAwait(false);
}
}

19
MareSynchronosServer/MareSynchronosServices/Discord/DiscordBot.cs
View File

@@ -10,10 +10,10 @@ using Discord.WebSocket;
using MareSynchronosServices.Identity;
using MareSynchronosShared.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace MareSynchronosServices.Discord;
@@ -22,23 +22,20 @@ internal class DiscordBot : IHostedService
private readonly DiscordBotServices _botServices;
private readonly IdentityHandler _identityHandler;
private readonly IServiceProvider _services;
private readonly IConfiguration _configuration;
private readonly ServicesConfiguration _configuration;
private readonly ILogger<DiscordBot> _logger;
private string _discordAuthToken = string.Empty;
private readonly DiscordSocketClient _discordClient;
private CancellationTokenSource? _updateStatusCts;
private CancellationTokenSource? _vanityUpdateCts;
public DiscordBot(DiscordBotServices botServices, IdentityHandler identityHandler, IServiceProvider services, IConfiguration configuration, ILogger<DiscordBot> logger)
public DiscordBot(DiscordBotServices botServices, IdentityHandler identityHandler, IServiceProvider services, IOptions<ServicesConfiguration> configuration, ILogger<DiscordBot> logger)
{
_botServices = botServices;
_identityHandler = identityHandler;
_services = services;
_configuration = configuration.GetRequiredSection("MareSynchronos");
_configuration = configuration.Value;
_logger = logger;
_discordAuthToken = _configuration.GetValue<string>("DiscordBotToken");
_discordClient = new(new DiscordSocketConfig()
{
DefaultRetryMode = RetryMode.AlwaysRetry
@@ -188,11 +185,9 @@ internal class DiscordBot : IHostedService
public async Task StartAsync(CancellationToken cancellationToken)
{
if (!string.IsNullOrEmpty(_discordAuthToken))
if (!string.IsNullOrEmpty(_configuration.DiscordBotToken))
{
_discordAuthToken = _configuration.GetValue<string>("DiscordBotToken");
await _discordClient.LoginAsync(TokenType.Bot, _discordAuthToken).ConfigureAwait(false);
await _discordClient.LoginAsync(TokenType.Bot, _configuration.DiscordBotToken).ConfigureAwait(false);
await _discordClient.StartAsync().ConfigureAwait(false);
_discordClient.Ready += DiscordClient_Ready;
@@ -204,7 +199,7 @@ internal class DiscordBot : IHostedService
public async Task StopAsync(CancellationToken cancellationToken)
{
if (!string.IsNullOrEmpty(_discordAuthToken))
if (!string.IsNullOrEmpty(_configuration.DiscordBotToken))
{
await _botServices.Stop();
_updateStatusCts?.Cancel();

8
MareSynchronosServer/MareSynchronosServices/Discord/DiscordBotServices.cs
View File

@@ -3,9 +3,9 @@ using System.Threading.Tasks;
using System.Collections.Generic;
using System.Collections.Concurrent;
using MareSynchronosShared.Metrics;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using System.Threading;
using Microsoft.Extensions.Options;
namespace MareSynchronosServices.Discord;
@@ -19,15 +19,15 @@ public class DiscordBotServices
public readonly string[] LodestoneServers = new[] { "eu", "na", "jp", "fr", "de" };
private readonly IServiceProvider _serviceProvider;
public IConfiguration Configuration { get; init; }
public ServicesConfiguration Configuration { get; init; }
public ILogger<DiscordBotServices> Logger { get; init; }
public MareMetrics Metrics { get; init; }
public Random Random { get; init; }
private CancellationTokenSource? verificationTaskCts;
public DiscordBotServices(IConfiguration configuration, IServiceProvider serviceProvider, ILogger<DiscordBotServices> logger, MareMetrics metrics)
public DiscordBotServices(IOptions<ServicesConfiguration> configuration, IServiceProvider serviceProvider, ILogger<DiscordBotServices> logger, MareMetrics metrics)
{
Configuration = configuration.GetRequiredSection("MareSynchronos");
Configuration = configuration.Value;
_serviceProvider = serviceProvider;
Logger = logger;
Metrics = metrics;

5
MareSynchronosServer/MareSynchronosServices/Discord/MareModule.cs
View File

@@ -12,7 +12,6 @@ using Prometheus;
using MareSynchronosShared.Models;
using MareSynchronosServices.Identity;
using MareSynchronosShared.Metrics;
using Microsoft.Extensions.Configuration;
using System.Net.Http;
using MareSynchronosShared.Utils;
using System.Collections.Generic;
@@ -758,9 +757,9 @@ public class MareModule : InteractionModuleBase
user.IsAdmin = true;
}
if (_botServices.Configuration.GetValue<bool>("PurgeUnusedAccounts"))
if (_botServices.Configuration.PurgeUnusedAccounts)
{
var purgedDays = _botServices.Configuration.GetValue<int>("PurgeUnusedAccountsPeriodInDays");
var purgedDays = _botServices.Configuration.PurgeUnusedAccountsPeriodInDays;
user.LastLoggedIn = DateTime.UtcNow - TimeSpan.FromDays(purgedDays) + TimeSpan.FromDays(1);
}

6
MareSynchronosServer/MareSynchronosServices/Program.cs
View File

@@ -5,6 +5,7 @@ using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System;
using System.Linq;
@@ -22,6 +23,11 @@ public class Program
var metrics = services.GetRequiredService<MareMetrics>();
metrics.SetGaugeTo(MetricsAPI.GaugeUsersRegistered, dbContext.Users.Count());
var options = host.Services.GetService<IOptions<ServicesConfiguration>>();
var logger = host.Services.GetService<ILogger<Program>>();
logger.LogInformation("Loaded MareSynchronos Services Configuration");
logger.LogInformation(options.Value.ToString());
}
host.Run();

23
MareSynchronosServer/MareSynchronosServices/ServicesConfiguration.cs Normal file
View File

@@ -0,0 +1,23 @@
using MareSynchronosShared.Utils;
using System.Text;
namespace MareSynchronosServices;
public class ServicesConfiguration : MareConfigurationBase
{
public string DiscordBotToken { get; set; } = string.Empty;
public bool PurgeUnusedAccounts { get; set; } = false;
public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14;
public int MaxExistingGroupsByUser { get; set; } = 3;
public override string ToString()
{
StringBuilder sb = new();
sb.AppendLine(base.ToString());
sb.AppendLine($"{nameof(DiscordBotToken)} => {DiscordBotToken}");
sb.AppendLine($"{nameof(PurgeUnusedAccounts)} => {PurgeUnusedAccounts}");
sb.AppendLine($"{nameof(PurgeUnusedAccountsPeriodInDays)} => {PurgeUnusedAccountsPeriodInDays}");
sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");
return sb.ToString();
}
}

6
MareSynchronosServer/MareSynchronosServices/Startup.cs
View File

@@ -10,6 +10,7 @@ using Prometheus;
using System.Collections.Generic;
using MareSynchronosServices.Identity;
using Microsoft.Extensions.Logging;
using MareSynchronosShared.Utils;
namespace MareSynchronosServices;
@@ -31,7 +32,7 @@ public class Startup
builder.MigrationsHistoryTable("_efmigrationshistory", "public");
}).UseSnakeCaseNamingConvention();
options.EnableThreadSafetyChecks(false);
}, Configuration.GetValue("DbContextPoolSize", 1024));
}, Configuration.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
services.AddSingleton(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string> {
}, new List<string>
@@ -39,7 +40,8 @@ public class Startup
MetricsAPI.GaugeUsersRegistered
}));
services.AddTransient(_ => Configuration);
services.Configure<ServicesConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
services.AddSingleton(Configuration);
services.AddSingleton<DiscordBotServices>();
services.AddSingleton<IdentityHandler>();
services.AddSingleton<CleanupService>();

14
MareSynchronosServer/MareSynchronosShared/Authentication/SecretKeyAuthenticatorService.cs
View File

@@ -3,9 +3,9 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Utils;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace MareSynchronosShared.Authentication;
@@ -20,15 +20,13 @@ public class SecretKeyAuthenticatorService
private readonly int _tempBanMinutes;
private readonly List<string> _whitelistedIps;
public SecretKeyAuthenticatorService(MareMetrics metrics, IServiceScopeFactory serviceScopeFactory, IConfiguration configuration, ILogger<SecretKeyAuthenticatorService> logger)
public SecretKeyAuthenticatorService(MareMetrics metrics, IServiceScopeFactory serviceScopeFactory, IOptions<MareConfigurationAuthBase> configuration, ILogger<SecretKeyAuthenticatorService> logger)
{
_logger = logger;
var config = configuration.GetRequiredSection("MareSynchronos");
_failedAttemptsForTempBan = config.GetValue<int>("FailedAuthForTempBan", 5);
logger.LogInformation("FailedAuthForTempBan: {num}", _failedAttemptsForTempBan);
_tempBanMinutes = config.GetValue<int>("TempBanDurationInMinutes", 30);
logger.LogInformation("TempBanMinutes: {num}", _tempBanMinutes);
_whitelistedIps = config.GetSection("WhitelistedIps").Get<List<string>>();
var config = configuration.Value;
_failedAttemptsForTempBan = config.FailedAuthForTempBan;
_tempBanMinutes = config.TempBanDurationInMinutes;
_whitelistedIps = config.WhitelistedIps;
foreach (var ip in _whitelistedIps)
{
logger.LogInformation("Whitelisted IP: " + ip);

32
MareSynchronosServer/MareSynchronosShared/Utils/MareConfigurationBase.cs Normal file
View File

@@ -0,0 +1,32 @@
using System.Text;
namespace MareSynchronosShared.Utils;
public class MareConfigurationBase
{
public int DbContextPoolSize { get; set; } = 100;
public override string ToString()
{
StringBuilder sb = new();
sb.AppendLine($"{nameof(DbContextPoolSize)} => {DbContextPoolSize}");
return sb.ToString();
}
}
public class MareConfigurationAuthBase : MareConfigurationBase
{
public int DbContextPoolSize { get; set; } = 100;
public int FailedAuthForTempBan { get; set; } = 5;
public int TempBanDurationInMinutes { get; set; } = 5;
public List<string> WhitelistedIps { get; set; } = new();
public override string ToString()
{
StringBuilder sb = new();
sb.AppendLine($"{nameof(FailedAuthForTempBan)} => {FailedAuthForTempBan}");
sb.AppendLine($"{nameof(TempBanDurationInMinutes)} => {TempBanDurationInMinutes}");
sb.AppendLine($"{nameof(WhitelistedIps)} => {string.Join(", ", WhitelistedIps)}");
return sb.ToString();
}
}

8
MareSynchronosServer/MareSynchronosStaticFilesServer/CachedFileProvider.cs
View File

@@ -1,4 +1,5 @@
using MareSynchronosShared.Metrics;
using Microsoft.Extensions.Options;
using System.Collections.Concurrent;
namespace MareSynchronosStaticFilesServer;
@@ -13,14 +14,13 @@ public class CachedFileProvider
private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
private bool IsMainServer => _remoteCacheSourceUri == null;
public CachedFileProvider(IConfiguration configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics)
public CachedFileProvider(IOptions<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics)
{
_logger = logger;
_fileStatisticsService = fileStatisticsService;
_metrics = metrics;
var configurationSection = configuration.GetRequiredSection("MareSynchronos");
_remoteCacheSourceUri = configurationSection.GetValue<Uri>("RemoteCacheSourceUri", null);
_basePath = configurationSection["CacheDirectory"];
_remoteCacheSourceUri = configuration.Value.RemoteCacheSourceUri;
_basePath = configuration.Value.CacheDirectory;
}
public async Task<FileStream?> GetFileStream(string hash, string auth)

35
MareSynchronosServer/MareSynchronosStaticFilesServer/FileCleanupService.cs
View File

@@ -2,7 +2,7 @@
using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Models;
using System.Globalization;
using Microsoft.Extensions.Options;
namespace MareSynchronosStaticFilesServer;
@@ -11,19 +11,19 @@ public class FileCleanupService : IHostedService
private readonly MareMetrics _metrics;
private readonly ILogger<FileCleanupService> _logger;
private readonly IServiceProvider _services;
private readonly IConfiguration _configuration;
private readonly StaticFilesServerConfiguration _configuration;
private readonly bool _isMainServer;
private readonly string _cacheDir;
private CancellationTokenSource _cleanupCts;
public FileCleanupService(MareMetrics metrics, ILogger<FileCleanupService> logger, IServiceProvider services, IConfiguration configuration)
public FileCleanupService(MareMetrics metrics, ILogger<FileCleanupService> logger, IServiceProvider services, IOptions<StaticFilesServerConfiguration> configuration)
{
_metrics = metrics;
_logger = logger;
_services = services;
_configuration = configuration.GetRequiredSection("MareSynchronos");
_isMainServer = string.IsNullOrEmpty(_configuration.GetValue("RemoteCacheSourceUri", string.Empty));
_cacheDir = _configuration.GetValue<string>("CacheDirectory");
_configuration = configuration.Value;
_isMainServer = _configuration.RemoteCacheSourceUri == null;
_cacheDir = _configuration.CacheDirectory;
}
public Task StartAsync(CancellationToken cancellationToken)
@@ -67,21 +67,19 @@ public class FileCleanupService : IHostedService
private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct)
{
var cacheSizeLimitInGiB = _configuration.GetValue<double>("CacheSizeHardLimitInGiB", -1);
if (cacheSizeLimitInGiB <= 0)
if (_configuration.CacheSizeHardLimitInGiB <= 0)
{
return;
}
try
{
_logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", cacheSizeLimitInGiB);
_logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", _configuration.CacheSizeHardLimitInGiB);
var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories)
.Select(f => new FileInfo(f)).ToList()
.OrderBy(f => f.LastAccessTimeUtc).ToList();
var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(cacheSizeLimitInGiB).Bytes;
long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(_configuration.CacheSizeHardLimitInGiB).Bytes;
while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested)
{
var oldestFile = allLocalFiles[0];
@@ -108,18 +106,15 @@ public class FileCleanupService : IHostedService
{
try
{
var filesOlderThanDays = _configuration.GetValue("UnusedFileRetentionPeriodInDays", 7);
var forcedDeletionHours = _configuration.GetValue("ForcedDeletionOfFilesAfterHours", -1);
_logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", filesOlderThanDays);
if (forcedDeletionHours > 0)
_logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", _configuration.UnusedFileRetentionPeriodInDays);
if (_configuration.ForcedDeletionOfFilesAfterHours > 0)
{
_logger.LogInformation("Cleaning up files written to longer than {hours}h ago", forcedDeletionHours);
_logger.LogInformation("Cleaning up files written to longer than {hours}h ago", _configuration.ForcedDeletionOfFilesAfterHours);
}
// clean up files in DB but not on disk or last access is expired
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays));
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionHours));
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(_configuration.UnusedFileRetentionPeriodInDays));
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(_configuration.ForcedDeletionOfFilesAfterHours));
var allFiles = dbContext.Files.ToList();
foreach (var fileCache in allFiles.Where(f => f.Uploaded))
{
@@ -138,7 +133,7 @@ public class FileCleanupService : IHostedService
if (_isMainServer)
dbContext.Files.Remove(fileCache);
}
else if (file != null && forcedDeletionHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
else if (file != null && _configuration.ForcedDeletionOfFilesAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
{
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);

5
MareSynchronosServer/MareSynchronosStaticFilesServer/GrpcFileService.cs
View File

@@ -3,6 +3,7 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Protos;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
namespace MareSynchronosStaticFilesServer;
@@ -13,9 +14,9 @@ public class GrpcFileService : FileService.FileServiceBase
private readonly ILogger<GrpcFileService> _logger;
private readonly MareMetrics _metricsClient;
public GrpcFileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger<GrpcFileService> logger, MareMetrics metricsClient)
public GrpcFileService(MareDbContext mareDbContext, IOptions<StaticFilesServerConfiguration> configuration, ILogger<GrpcFileService> logger, MareMetrics metricsClient)
{
_basePath = configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"];
_basePath = configuration.Value.CacheDirectory;
_mareDbContext = mareDbContext;
_logger = logger;
_metricsClient = metricsClient;

10
MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs
View File

@@ -1,3 +1,5 @@
using Microsoft.Extensions.Options;
namespace MareSynchronosStaticFilesServer;
public class Program
@@ -7,6 +9,14 @@ public class Program
var hostBuilder = CreateHostBuilder(args);
var host = hostBuilder.Build();
using (var scope = host.Services.CreateScope())
{
var options = host.Services.GetService<IOptions<StaticFilesServerConfiguration>>();
var logger = host.Services.GetService<ILogger<Program>>();
logger.LogInformation("Loaded MareSynchronos Static Files Server Configuration");
logger.LogInformation(options.Value.ToString());
}
host.Run();
}

12
MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs
View File

@@ -1,7 +1,7 @@
using Grpc.Net.Client.Configuration;
using MareSynchronosShared.Authentication;
using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Utils;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authorization;
using Microsoft.EntityFrameworkCore;
@@ -16,7 +16,7 @@ public class Startup
{
Configuration = configuration;
var mareSettings = Configuration.GetRequiredSection("MareSynchronos");
_isMain = string.IsNullOrEmpty(mareSettings.GetValue("RemoteCacheSourceUri", string.Empty));
_isMain = string.IsNullOrEmpty(mareSettings.GetValue(nameof(StaticFilesServerConfiguration.RemoteCacheSourceUri), string.Empty));
}
public IConfiguration Configuration { get; }
@@ -25,10 +25,12 @@ public class Startup
{
services.AddHttpContextAccessor();
services.AddTransient(_ => Configuration);
services.AddLogging();
services.Configure<MareConfigurationAuthBase>(Configuration.GetRequiredSection("MareSynchronos"));
services.Configure<StaticFilesServerConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
services.AddSingleton(Configuration);
var mareSettings = Configuration.GetRequiredSection("MareSynchronos");
services.AddControllers();
@@ -62,7 +64,7 @@ public class Startup
builder.MigrationsHistoryTable("_efmigrationshistory", "public");
}).UseSnakeCaseNamingConvention();
options.EnableThreadSafetyChecks(false);
}, mareSettings.GetValue("DbContextPoolSize", 1024));
}, mareSettings.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
services.AddAuthentication(options =>
{

25
MareSynchronosServer/MareSynchronosStaticFilesServer/StaticFilesServerConfiguration.cs Normal file
View File

@@ -0,0 +1,25 @@
using MareSynchronosShared.Utils;
using System.Text;
namespace MareSynchronosStaticFilesServer;
public class StaticFilesServerConfiguration : MareConfigurationAuthBase
{
public int ForcedDeletionOfFilesAfterHours { get; set; } = -1;
public double CacheSizeHardLimitInGiB { get; set; } = -1;
public int UnusedFileRetentionPeriodInDays { get; set; } = -1;
public string CacheDirectory { get; set; }
public Uri? RemoteCacheSourceUri { get; set; } = null;
public override string ToString()
{
StringBuilder sb = new();
sb.AppendLine(base.ToString());
sb.AppendLine($"{nameof(ForcedDeletionOfFilesAfterHours)} => {ForcedDeletionOfFilesAfterHours}");
sb.AppendLine($"{nameof(CacheSizeHardLimitInGiB)} => {CacheSizeHardLimitInGiB}");
sb.AppendLine($"{nameof(UnusedFileRetentionPeriodInDays)} => {UnusedFileRetentionPeriodInDays}");
sb.AppendLine($"{nameof(CacheDirectory)} => {CacheDirectory}");
sb.AppendLine($"{nameof(RemoteCacheSourceUri)} => {RemoteCacheSourceUri}");
return sb.ToString();
}
}