switch around all iconfiguration usage to ioptions
This commit is contained in:
@@ -12,8 +12,8 @@ using Microsoft.AspNetCore.Authorization;
|
|||||||
using Microsoft.AspNetCore.Http;
|
using Microsoft.AspNetCore.Http;
|
||||||
using Microsoft.AspNetCore.SignalR;
|
using Microsoft.AspNetCore.SignalR;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.Extensions.Configuration;
|
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
namespace MareSynchronosServer.Hubs;
|
namespace MareSynchronosServer.Hubs;
|
||||||
|
|
||||||
@@ -33,18 +33,18 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
|
|||||||
private readonly int _maxGroupUserCount;
|
private readonly int _maxGroupUserCount;
|
||||||
|
|
||||||
public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient,
|
public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient,
|
||||||
MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService, IConfiguration configuration, IHttpContextAccessor contextAccessor,
|
MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService, IOptions<ServerConfiguration> configuration, IHttpContextAccessor contextAccessor,
|
||||||
GrpcClientIdentificationService clientIdentService)
|
GrpcClientIdentificationService clientIdentService)
|
||||||
{
|
{
|
||||||
_mareMetrics = mareMetrics;
|
_mareMetrics = mareMetrics;
|
||||||
_fileServiceClient = fileServiceClient;
|
_fileServiceClient = fileServiceClient;
|
||||||
_systemInfoService = systemInfoService;
|
_systemInfoService = systemInfoService;
|
||||||
var config = configuration.GetRequiredSection("MareSynchronos");
|
var config = configuration.Value;
|
||||||
_cdnFullUri = new Uri(config.GetValue<string>("CdnFullUrl"));
|
_cdnFullUri = config.CdnFullUrl;
|
||||||
_shardName = config.GetValue("ShardName", "Main");
|
_shardName = config.ShardName;
|
||||||
_maxExistingGroupsByUser = config.GetValue<int>("MaxExistingGroupsByUser", 3);
|
_maxExistingGroupsByUser = config.MaxExistingGroupsByUser;
|
||||||
_maxJoinedGroupsByUser = config.GetValue<int>("MaxJoinedGroupsByUser", 6);
|
_maxJoinedGroupsByUser = config.MaxJoinedGroupsByUser;
|
||||||
_maxGroupUserCount = config.GetValue<int>("MaxGroupUserCount", 100);
|
_maxGroupUserCount = config.MaxGroupUserCount;
|
||||||
_contextAccessor = contextAccessor;
|
_contextAccessor = contextAccessor;
|
||||||
_clientIdentService = clientIdentService;
|
_clientIdentService = clientIdentService;
|
||||||
_logger = new MareHubLogger(this, logger);
|
_logger = new MareHubLogger(this, logger);
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ using Microsoft.Extensions.DependencyInjection;
|
|||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using MareSynchronosShared.Data;
|
using MareSynchronosShared.Data;
|
||||||
using MareSynchronosShared.Metrics;
|
using MareSynchronosShared.Metrics;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
namespace MareSynchronosServer;
|
namespace MareSynchronosServer;
|
||||||
|
|
||||||
@@ -40,6 +41,11 @@ public class Program
|
|||||||
|
|
||||||
metrics.SetGaugeTo(MetricsAPI.GaugePairs, context.ClientPairs.Count());
|
metrics.SetGaugeTo(MetricsAPI.GaugePairs, context.ClientPairs.Count());
|
||||||
metrics.SetGaugeTo(MetricsAPI.GaugePairsPaused, context.ClientPairs.Count(p => p.IsPaused));
|
metrics.SetGaugeTo(MetricsAPI.GaugePairsPaused, context.ClientPairs.Count(p => p.IsPaused));
|
||||||
|
|
||||||
|
var options = host.Services.GetService<IOptions<ServerConfiguration>>();
|
||||||
|
var logger = host.Services.GetService<ILogger<Program>>();
|
||||||
|
logger.LogInformation("Loaded MareSynchronos Server Configuration");
|
||||||
|
logger.LogInformation(options.Value.ToString());
|
||||||
}
|
}
|
||||||
|
|
||||||
if (args.Length == 0 || !string.Equals(args[0], "dry", StringComparison.Ordinal))
|
if (args.Length == 0 || !string.Equals(args[0], "dry", StringComparison.Ordinal))
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
using MareSynchronosShared.Utils;
|
||||||
|
using System;
|
||||||
|
using System.Text;
|
||||||
|
|
||||||
|
namespace MareSynchronosServer;
|
||||||
|
|
||||||
|
public class ServerConfiguration : MareConfigurationAuthBase
|
||||||
|
{
|
||||||
|
public Uri CdnFullUrl { get; set; } = null;
|
||||||
|
public Uri ServiceAddress { get; set; } = null;
|
||||||
|
public Uri StaticFileServiceAddress { get; set; } = null;
|
||||||
|
public string RedisConnectionString { get; set; } = string.Empty;
|
||||||
|
public int MaxExistingGroupsByUser { get; set; } = 3;
|
||||||
|
public int MaxJoinedGroupsByUser { get; set; } = 6;
|
||||||
|
public int MaxGroupUserCount { get; set; } = 100;
|
||||||
|
public string ShardName { get; set; } = string.Empty;
|
||||||
|
|
||||||
|
public override string ToString()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new();
|
||||||
|
sb.AppendLine(base.ToString());
|
||||||
|
sb.AppendLine($"{nameof(ShardName)} => {ShardName}");
|
||||||
|
sb.AppendLine($"{nameof(CdnFullUrl)} => {CdnFullUrl}");
|
||||||
|
sb.AppendLine($"{nameof(ServiceAddress)} => {ServiceAddress}");
|
||||||
|
sb.AppendLine($"{nameof(StaticFileServiceAddress)} => {StaticFileServiceAddress}");
|
||||||
|
sb.AppendLine($"{nameof(RedisConnectionString)} => {RedisConnectionString}");
|
||||||
|
sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");
|
||||||
|
sb.AppendLine($"{nameof(MaxJoinedGroupsByUser)} => {MaxJoinedGroupsByUser}");
|
||||||
|
sb.AppendLine($"{nameof(MaxGroupUserCount)} => {MaxGroupUserCount}");
|
||||||
|
return sb.ToString();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,8 +2,8 @@
|
|||||||
using MareSynchronosShared.Metrics;
|
using MareSynchronosShared.Metrics;
|
||||||
using MareSynchronosShared.Protos;
|
using MareSynchronosShared.Protos;
|
||||||
using MareSynchronosShared.Services;
|
using MareSynchronosShared.Services;
|
||||||
using Microsoft.Extensions.Configuration;
|
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
@@ -17,8 +17,8 @@ public class GrpcClientIdentificationService : GrpcBaseService
|
|||||||
private readonly string _shardName;
|
private readonly string _shardName;
|
||||||
private readonly ILogger<GrpcClientIdentificationService> _logger;
|
private readonly ILogger<GrpcClientIdentificationService> _logger;
|
||||||
private readonly IdentificationService.IdentificationServiceClient _grpcIdentClient;
|
private readonly IdentificationService.IdentificationServiceClient _grpcIdentClient;
|
||||||
private readonly IdentificationService.IdentificationServiceClient grpcIdentClientStreamOut;
|
private readonly IdentificationService.IdentificationServiceClient _grpcIdentClientStreamOut;
|
||||||
private readonly IdentificationService.IdentificationServiceClient grpcIdentClientStreamIn;
|
private readonly IdentificationService.IdentificationServiceClient _grpcIdentClientStreamIn;
|
||||||
private readonly MareMetrics _metrics;
|
private readonly MareMetrics _metrics;
|
||||||
protected readonly ConcurrentDictionary<string, UidWithIdent> OnlineClients = new(StringComparer.Ordinal);
|
protected readonly ConcurrentDictionary<string, UidWithIdent> OnlineClients = new(StringComparer.Ordinal);
|
||||||
private readonly ConcurrentDictionary<string, UidWithIdent> RemoteCachedIdents = new(StringComparer.Ordinal);
|
private readonly ConcurrentDictionary<string, UidWithIdent> RemoteCachedIdents = new(StringComparer.Ordinal);
|
||||||
@@ -26,14 +26,13 @@ public class GrpcClientIdentificationService : GrpcBaseService
|
|||||||
|
|
||||||
public GrpcClientIdentificationService(ILogger<GrpcClientIdentificationService> logger, IdentificationService.IdentificationServiceClient gprcIdentClient,
|
public GrpcClientIdentificationService(ILogger<GrpcClientIdentificationService> logger, IdentificationService.IdentificationServiceClient gprcIdentClient,
|
||||||
IdentificationService.IdentificationServiceClient gprcIdentClientStreamOut,
|
IdentificationService.IdentificationServiceClient gprcIdentClientStreamOut,
|
||||||
IdentificationService.IdentificationServiceClient gprcIdentClientStreamIn, MareMetrics metrics, IConfiguration configuration) : base(logger)
|
IdentificationService.IdentificationServiceClient gprcIdentClientStreamIn, MareMetrics metrics, IOptions<ServerConfiguration> configuration) : base(logger)
|
||||||
{
|
{
|
||||||
var config = configuration.GetSection("MareSynchronos");
|
_shardName = configuration.Value.ShardName;
|
||||||
_shardName = config.GetValue("ShardName", "Main");
|
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
_grpcIdentClient = gprcIdentClient;
|
_grpcIdentClient = gprcIdentClient;
|
||||||
this.grpcIdentClientStreamOut = gprcIdentClientStreamOut;
|
_grpcIdentClientStreamOut = gprcIdentClientStreamOut;
|
||||||
this.grpcIdentClientStreamIn = gprcIdentClientStreamIn;
|
_grpcIdentClientStreamIn = gprcIdentClientStreamIn;
|
||||||
_metrics = metrics;
|
_metrics = metrics;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -136,7 +135,7 @@ public class GrpcClientIdentificationService : GrpcBaseService
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
using var stream = grpcIdentClientStreamOut.SendStreamIdentStatusChange(cancellationToken: cts);
|
using var stream = _grpcIdentClientStreamOut.SendStreamIdentStatusChange(cancellationToken: cts);
|
||||||
_logger.LogInformation("Starting Send Online Client Data stream");
|
_logger.LogInformation("Starting Send Online Client Data stream");
|
||||||
await stream.RequestStream.WriteAsync(new IdentChangeMessage()
|
await stream.RequestStream.WriteAsync(new IdentChangeMessage()
|
||||||
{
|
{
|
||||||
@@ -169,7 +168,7 @@ public class GrpcClientIdentificationService : GrpcBaseService
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
using var stream = grpcIdentClientStreamIn.ReceiveStreamIdentStatusChange(new ServerMessage()
|
using var stream = _grpcIdentClientStreamIn.ReceiveStreamIdentStatusChange(new ServerMessage()
|
||||||
{
|
{
|
||||||
ServerId = _shardName,
|
ServerId = _shardName,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ using MareSynchronosShared.Data;
|
|||||||
using MareSynchronosShared.Metrics;
|
using MareSynchronosShared.Metrics;
|
||||||
using Microsoft.AspNetCore.SignalR;
|
using Microsoft.AspNetCore.SignalR;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.Extensions.Configuration;
|
|
||||||
using Microsoft.Extensions.DependencyInjection;
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
using Microsoft.Extensions.Hosting;
|
using Microsoft.Extensions.Hosting;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
@@ -25,7 +24,7 @@ public class SystemInfoService : IHostedService, IDisposable
|
|||||||
private Timer _timer;
|
private Timer _timer;
|
||||||
public SystemInfoDto SystemInfoDto { get; private set; } = new();
|
public SystemInfoDto SystemInfoDto { get; private set; } = new();
|
||||||
|
|
||||||
public SystemInfoService(MareMetrics mareMetrics, IConfiguration configuration, IServiceProvider services, GrpcClientIdentificationService clientIdentService, ILogger<SystemInfoService> logger, IHubContext<MareHub, IMareHub> hubContext)
|
public SystemInfoService(MareMetrics mareMetrics, IServiceProvider services, GrpcClientIdentificationService clientIdentService, ILogger<SystemInfoService> logger, IHubContext<MareHub, IMareHub> hubContext)
|
||||||
{
|
{
|
||||||
_mareMetrics = mareMetrics;
|
_mareMetrics = mareMetrics;
|
||||||
_services = services;
|
_services = services;
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ using System.Net.Http;
|
|||||||
using MareSynchronosServer.Utils;
|
using MareSynchronosServer.Utils;
|
||||||
using MareSynchronosServer.RequirementHandlers;
|
using MareSynchronosServer.RequirementHandlers;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using MareSynchronosShared.Utils;
|
||||||
|
|
||||||
namespace MareSynchronosServer;
|
namespace MareSynchronosServer;
|
||||||
|
|
||||||
@@ -40,6 +41,8 @@ public class Startup
|
|||||||
{
|
{
|
||||||
services.AddHttpContextAccessor();
|
services.AddHttpContextAccessor();
|
||||||
|
|
||||||
|
services.Configure<ServerConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
|
||||||
|
services.Configure<MareConfigurationAuthBase>(Configuration.GetRequiredSection("MareSynchronos"));
|
||||||
services.Configure<IpRateLimitOptions>(Configuration.GetSection("IpRateLimiting"));
|
services.Configure<IpRateLimitOptions>(Configuration.GetSection("IpRateLimiting"));
|
||||||
services.Configure<IpRateLimitPolicies>(Configuration.GetSection("IpRateLimitPolicies"));
|
services.Configure<IpRateLimitPolicies>(Configuration.GetSection("IpRateLimitPolicies"));
|
||||||
services.AddTransient(_ => Configuration);
|
services.AddTransient(_ => Configuration);
|
||||||
@@ -96,14 +99,14 @@ public class Startup
|
|||||||
|
|
||||||
services.AddGrpcClient<FileService.FileServiceClient>(c =>
|
services.AddGrpcClient<FileService.FileServiceClient>(c =>
|
||||||
{
|
{
|
||||||
c.Address = new Uri(mareConfig.GetValue<string>("StaticFileServiceAddress"));
|
c.Address = new Uri(mareConfig.GetValue<string>(nameof(ServerConfiguration.StaticFileServiceAddress)));
|
||||||
}).ConfigureChannel(c =>
|
}).ConfigureChannel(c =>
|
||||||
{
|
{
|
||||||
c.ServiceConfig = new ServiceConfig { MethodConfigs = { defaultMethodConfig } };
|
c.ServiceConfig = new ServiceConfig { MethodConfigs = { defaultMethodConfig } };
|
||||||
});
|
});
|
||||||
services.AddGrpcClient<IdentificationService.IdentificationServiceClient>(c =>
|
services.AddGrpcClient<IdentificationService.IdentificationServiceClient>(c =>
|
||||||
{
|
{
|
||||||
c.Address = new Uri(mareConfig.GetValue<string>("ServiceAddress"));
|
c.Address = new Uri(mareConfig.GetValue<string>(nameof(ServerConfiguration.ServiceAddress)));
|
||||||
}).ConfigureChannel(c =>
|
}).ConfigureChannel(c =>
|
||||||
{
|
{
|
||||||
c.ServiceConfig = new ServiceConfig { MethodConfigs = { noRetryConfig } };
|
c.ServiceConfig = new ServiceConfig { MethodConfigs = { noRetryConfig } };
|
||||||
@@ -126,7 +129,7 @@ public class Startup
|
|||||||
builder.MigrationsAssembly("MareSynchronosShared");
|
builder.MigrationsAssembly("MareSynchronosShared");
|
||||||
}).UseSnakeCaseNamingConvention();
|
}).UseSnakeCaseNamingConvention();
|
||||||
options.EnableThreadSafetyChecks(false);
|
options.EnableThreadSafetyChecks(false);
|
||||||
}, mareConfig.GetValue("DbContextPoolSize", 1024));
|
}, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
|
||||||
|
|
||||||
services.AddAuthentication(SecretKeyAuthenticationHandler.AuthScheme)
|
services.AddAuthentication(SecretKeyAuthenticationHandler.AuthScheme)
|
||||||
.AddScheme<AuthenticationSchemeOptions, SecretKeyAuthenticationHandler>(SecretKeyAuthenticationHandler.AuthScheme, options => { options.Validate(); });
|
.AddScheme<AuthenticationSchemeOptions, SecretKeyAuthenticationHandler>(SecretKeyAuthenticationHandler.AuthScheme, options => { options.Validate(); });
|
||||||
@@ -168,7 +171,7 @@ public class Startup
|
|||||||
});
|
});
|
||||||
|
|
||||||
// add redis related options
|
// add redis related options
|
||||||
var redis = mareConfig.GetValue("RedisConnectionString", string.Empty);
|
var redis = mareConfig.GetValue(nameof(ServerConfiguration.RedisConnectionString), string.Empty);
|
||||||
if (!string.IsNullOrEmpty(redis))
|
if (!string.IsNullOrEmpty(redis))
|
||||||
{
|
{
|
||||||
signalRServiceBuilder.AddStackExchangeRedis(redis, options =>
|
signalRServiceBuilder.AddStackExchangeRedis(redis, options =>
|
||||||
|
|||||||
@@ -3,10 +3,10 @@ using MareSynchronosShared.Metrics;
|
|||||||
using MareSynchronosShared.Models;
|
using MareSynchronosShared.Models;
|
||||||
using MareSynchronosShared.Utils;
|
using MareSynchronosShared.Utils;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.Extensions.Configuration;
|
|
||||||
using Microsoft.Extensions.DependencyInjection;
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
using Microsoft.Extensions.Hosting;
|
using Microsoft.Extensions.Hosting;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
@@ -20,15 +20,15 @@ public class CleanupService : IHostedService, IDisposable
|
|||||||
private readonly MareMetrics metrics;
|
private readonly MareMetrics metrics;
|
||||||
private readonly ILogger<CleanupService> _logger;
|
private readonly ILogger<CleanupService> _logger;
|
||||||
private readonly IServiceProvider _services;
|
private readonly IServiceProvider _services;
|
||||||
private readonly IConfiguration _configuration;
|
private readonly ServicesConfiguration _configuration;
|
||||||
private Timer? _timer;
|
private Timer? _timer;
|
||||||
|
|
||||||
public CleanupService(MareMetrics metrics, ILogger<CleanupService> logger, IServiceProvider services, IConfiguration configuration)
|
public CleanupService(MareMetrics metrics, ILogger<CleanupService> logger, IServiceProvider services, IOptions<ServicesConfiguration> configuration)
|
||||||
{
|
{
|
||||||
this.metrics = metrics;
|
this.metrics = metrics;
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
_services = services;
|
_services = services;
|
||||||
_configuration = configuration.GetRequiredSection("MareSynchronos");
|
_configuration = configuration.Value;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Task StartAsync(CancellationToken cancellationToken)
|
public Task StartAsync(CancellationToken cancellationToken)
|
||||||
@@ -68,17 +68,9 @@ public class CleanupService : IHostedService, IDisposable
|
|||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
if (!bool.TryParse(_configuration["PurgeUnusedAccounts"], out var purgeUnusedAccounts))
|
if (_configuration.PurgeUnusedAccounts)
|
||||||
{
|
{
|
||||||
purgeUnusedAccounts = false;
|
var usersOlderThanDays = _configuration.PurgeUnusedAccountsPeriodInDays;
|
||||||
}
|
|
||||||
|
|
||||||
if (purgeUnusedAccounts)
|
|
||||||
{
|
|
||||||
if (!int.TryParse(_configuration["PurgeUnusedAccountsPeriodInDays"], out var usersOlderThanDays))
|
|
||||||
{
|
|
||||||
usersOlderThanDays = 14;
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogInformation("Cleaning up users older than {usersOlderThanDays} days", usersOlderThanDays);
|
_logger.LogInformation("Cleaning up users older than {usersOlderThanDays} days", usersOlderThanDays);
|
||||||
|
|
||||||
@@ -160,7 +152,7 @@ public class CleanupService : IHostedService, IDisposable
|
|||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
_ = await SharedDbFunctions.MigrateOrDeleteGroup(dbContext, userGroupPair.Group, groupPairs, _configuration.GetValue<int>("MaxExistingGroupsByUser", 3)).ConfigureAwait(false);
|
_ = await SharedDbFunctions.MigrateOrDeleteGroup(dbContext, userGroupPair.Group, groupPairs, _configuration.MaxExistingGroupsByUser).ConfigureAwait(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,10 +10,10 @@ using Discord.WebSocket;
|
|||||||
using MareSynchronosServices.Identity;
|
using MareSynchronosServices.Identity;
|
||||||
using MareSynchronosShared.Data;
|
using MareSynchronosShared.Data;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.Extensions.Configuration;
|
|
||||||
using Microsoft.Extensions.DependencyInjection;
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
using Microsoft.Extensions.Hosting;
|
using Microsoft.Extensions.Hosting;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
namespace MareSynchronosServices.Discord;
|
namespace MareSynchronosServices.Discord;
|
||||||
|
|
||||||
@@ -22,23 +22,20 @@ internal class DiscordBot : IHostedService
|
|||||||
private readonly DiscordBotServices _botServices;
|
private readonly DiscordBotServices _botServices;
|
||||||
private readonly IdentityHandler _identityHandler;
|
private readonly IdentityHandler _identityHandler;
|
||||||
private readonly IServiceProvider _services;
|
private readonly IServiceProvider _services;
|
||||||
private readonly IConfiguration _configuration;
|
private readonly ServicesConfiguration _configuration;
|
||||||
private readonly ILogger<DiscordBot> _logger;
|
private readonly ILogger<DiscordBot> _logger;
|
||||||
private string _discordAuthToken = string.Empty;
|
|
||||||
private readonly DiscordSocketClient _discordClient;
|
private readonly DiscordSocketClient _discordClient;
|
||||||
private CancellationTokenSource? _updateStatusCts;
|
private CancellationTokenSource? _updateStatusCts;
|
||||||
private CancellationTokenSource? _vanityUpdateCts;
|
private CancellationTokenSource? _vanityUpdateCts;
|
||||||
|
|
||||||
public DiscordBot(DiscordBotServices botServices, IdentityHandler identityHandler, IServiceProvider services, IConfiguration configuration, ILogger<DiscordBot> logger)
|
public DiscordBot(DiscordBotServices botServices, IdentityHandler identityHandler, IServiceProvider services, IOptions<ServicesConfiguration> configuration, ILogger<DiscordBot> logger)
|
||||||
{
|
{
|
||||||
_botServices = botServices;
|
_botServices = botServices;
|
||||||
_identityHandler = identityHandler;
|
_identityHandler = identityHandler;
|
||||||
_services = services;
|
_services = services;
|
||||||
_configuration = configuration.GetRequiredSection("MareSynchronos");
|
_configuration = configuration.Value;
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
|
|
||||||
_discordAuthToken = _configuration.GetValue<string>("DiscordBotToken");
|
|
||||||
|
|
||||||
_discordClient = new(new DiscordSocketConfig()
|
_discordClient = new(new DiscordSocketConfig()
|
||||||
{
|
{
|
||||||
DefaultRetryMode = RetryMode.AlwaysRetry
|
DefaultRetryMode = RetryMode.AlwaysRetry
|
||||||
@@ -188,11 +185,9 @@ internal class DiscordBot : IHostedService
|
|||||||
|
|
||||||
public async Task StartAsync(CancellationToken cancellationToken)
|
public async Task StartAsync(CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
if (!string.IsNullOrEmpty(_discordAuthToken))
|
if (!string.IsNullOrEmpty(_configuration.DiscordBotToken))
|
||||||
{
|
{
|
||||||
_discordAuthToken = _configuration.GetValue<string>("DiscordBotToken");
|
await _discordClient.LoginAsync(TokenType.Bot, _configuration.DiscordBotToken).ConfigureAwait(false);
|
||||||
|
|
||||||
await _discordClient.LoginAsync(TokenType.Bot, _discordAuthToken).ConfigureAwait(false);
|
|
||||||
await _discordClient.StartAsync().ConfigureAwait(false);
|
await _discordClient.StartAsync().ConfigureAwait(false);
|
||||||
|
|
||||||
_discordClient.Ready += DiscordClient_Ready;
|
_discordClient.Ready += DiscordClient_Ready;
|
||||||
@@ -204,7 +199,7 @@ internal class DiscordBot : IHostedService
|
|||||||
|
|
||||||
public async Task StopAsync(CancellationToken cancellationToken)
|
public async Task StopAsync(CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
if (!string.IsNullOrEmpty(_discordAuthToken))
|
if (!string.IsNullOrEmpty(_configuration.DiscordBotToken))
|
||||||
{
|
{
|
||||||
await _botServices.Stop();
|
await _botServices.Stop();
|
||||||
_updateStatusCts?.Cancel();
|
_updateStatusCts?.Cancel();
|
||||||
|
|||||||
@@ -3,9 +3,9 @@ using System.Threading.Tasks;
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
using MareSynchronosShared.Metrics;
|
using MareSynchronosShared.Metrics;
|
||||||
using Microsoft.Extensions.Configuration;
|
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using System.Threading;
|
using System.Threading;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
namespace MareSynchronosServices.Discord;
|
namespace MareSynchronosServices.Discord;
|
||||||
|
|
||||||
@@ -19,15 +19,15 @@ public class DiscordBotServices
|
|||||||
public readonly string[] LodestoneServers = new[] { "eu", "na", "jp", "fr", "de" };
|
public readonly string[] LodestoneServers = new[] { "eu", "na", "jp", "fr", "de" };
|
||||||
private readonly IServiceProvider _serviceProvider;
|
private readonly IServiceProvider _serviceProvider;
|
||||||
|
|
||||||
public IConfiguration Configuration { get; init; }
|
public ServicesConfiguration Configuration { get; init; }
|
||||||
public ILogger<DiscordBotServices> Logger { get; init; }
|
public ILogger<DiscordBotServices> Logger { get; init; }
|
||||||
public MareMetrics Metrics { get; init; }
|
public MareMetrics Metrics { get; init; }
|
||||||
public Random Random { get; init; }
|
public Random Random { get; init; }
|
||||||
private CancellationTokenSource? verificationTaskCts;
|
private CancellationTokenSource? verificationTaskCts;
|
||||||
|
|
||||||
public DiscordBotServices(IConfiguration configuration, IServiceProvider serviceProvider, ILogger<DiscordBotServices> logger, MareMetrics metrics)
|
public DiscordBotServices(IOptions<ServicesConfiguration> configuration, IServiceProvider serviceProvider, ILogger<DiscordBotServices> logger, MareMetrics metrics)
|
||||||
{
|
{
|
||||||
Configuration = configuration.GetRequiredSection("MareSynchronos");
|
Configuration = configuration.Value;
|
||||||
_serviceProvider = serviceProvider;
|
_serviceProvider = serviceProvider;
|
||||||
Logger = logger;
|
Logger = logger;
|
||||||
Metrics = metrics;
|
Metrics = metrics;
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ using Prometheus;
|
|||||||
using MareSynchronosShared.Models;
|
using MareSynchronosShared.Models;
|
||||||
using MareSynchronosServices.Identity;
|
using MareSynchronosServices.Identity;
|
||||||
using MareSynchronosShared.Metrics;
|
using MareSynchronosShared.Metrics;
|
||||||
using Microsoft.Extensions.Configuration;
|
|
||||||
using System.Net.Http;
|
using System.Net.Http;
|
||||||
using MareSynchronosShared.Utils;
|
using MareSynchronosShared.Utils;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
@@ -758,9 +757,9 @@ public class MareModule : InteractionModuleBase
|
|||||||
user.IsAdmin = true;
|
user.IsAdmin = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (_botServices.Configuration.GetValue<bool>("PurgeUnusedAccounts"))
|
if (_botServices.Configuration.PurgeUnusedAccounts)
|
||||||
{
|
{
|
||||||
var purgedDays = _botServices.Configuration.GetValue<int>("PurgeUnusedAccountsPeriodInDays");
|
var purgedDays = _botServices.Configuration.PurgeUnusedAccountsPeriodInDays;
|
||||||
user.LastLoggedIn = DateTime.UtcNow - TimeSpan.FromDays(purgedDays) + TimeSpan.FromDays(1);
|
user.LastLoggedIn = DateTime.UtcNow - TimeSpan.FromDays(purgedDays) + TimeSpan.FromDays(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ using Microsoft.AspNetCore.Hosting;
|
|||||||
using Microsoft.Extensions.DependencyInjection;
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
using Microsoft.Extensions.Hosting;
|
using Microsoft.Extensions.Hosting;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
using System;
|
using System;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
@@ -22,6 +23,11 @@ public class Program
|
|||||||
var metrics = services.GetRequiredService<MareMetrics>();
|
var metrics = services.GetRequiredService<MareMetrics>();
|
||||||
|
|
||||||
metrics.SetGaugeTo(MetricsAPI.GaugeUsersRegistered, dbContext.Users.Count());
|
metrics.SetGaugeTo(MetricsAPI.GaugeUsersRegistered, dbContext.Users.Count());
|
||||||
|
|
||||||
|
var options = host.Services.GetService<IOptions<ServicesConfiguration>>();
|
||||||
|
var logger = host.Services.GetService<ILogger<Program>>();
|
||||||
|
logger.LogInformation("Loaded MareSynchronos Services Configuration");
|
||||||
|
logger.LogInformation(options.Value.ToString());
|
||||||
}
|
}
|
||||||
|
|
||||||
host.Run();
|
host.Run();
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
using MareSynchronosShared.Utils;
|
||||||
|
using System.Text;
|
||||||
|
|
||||||
|
namespace MareSynchronosServices;
|
||||||
|
|
||||||
|
public class ServicesConfiguration : MareConfigurationBase
|
||||||
|
{
|
||||||
|
public string DiscordBotToken { get; set; } = string.Empty;
|
||||||
|
public bool PurgeUnusedAccounts { get; set; } = false;
|
||||||
|
public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14;
|
||||||
|
public int MaxExistingGroupsByUser { get; set; } = 3;
|
||||||
|
|
||||||
|
public override string ToString()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new();
|
||||||
|
sb.AppendLine(base.ToString());
|
||||||
|
sb.AppendLine($"{nameof(DiscordBotToken)} => {DiscordBotToken}");
|
||||||
|
sb.AppendLine($"{nameof(PurgeUnusedAccounts)} => {PurgeUnusedAccounts}");
|
||||||
|
sb.AppendLine($"{nameof(PurgeUnusedAccountsPeriodInDays)} => {PurgeUnusedAccountsPeriodInDays}");
|
||||||
|
sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");
|
||||||
|
return sb.ToString();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@ using Prometheus;
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using MareSynchronosServices.Identity;
|
using MareSynchronosServices.Identity;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using MareSynchronosShared.Utils;
|
||||||
|
|
||||||
namespace MareSynchronosServices;
|
namespace MareSynchronosServices;
|
||||||
|
|
||||||
@@ -31,7 +32,7 @@ public class Startup
|
|||||||
builder.MigrationsHistoryTable("_efmigrationshistory", "public");
|
builder.MigrationsHistoryTable("_efmigrationshistory", "public");
|
||||||
}).UseSnakeCaseNamingConvention();
|
}).UseSnakeCaseNamingConvention();
|
||||||
options.EnableThreadSafetyChecks(false);
|
options.EnableThreadSafetyChecks(false);
|
||||||
}, Configuration.GetValue("DbContextPoolSize", 1024));
|
}, Configuration.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
|
||||||
|
|
||||||
services.AddSingleton(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string> {
|
services.AddSingleton(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string> {
|
||||||
}, new List<string>
|
}, new List<string>
|
||||||
@@ -39,7 +40,8 @@ public class Startup
|
|||||||
MetricsAPI.GaugeUsersRegistered
|
MetricsAPI.GaugeUsersRegistered
|
||||||
}));
|
}));
|
||||||
|
|
||||||
services.AddTransient(_ => Configuration);
|
services.Configure<ServicesConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
|
||||||
|
services.AddSingleton(Configuration);
|
||||||
services.AddSingleton<DiscordBotServices>();
|
services.AddSingleton<DiscordBotServices>();
|
||||||
services.AddSingleton<IdentityHandler>();
|
services.AddSingleton<IdentityHandler>();
|
||||||
services.AddSingleton<CleanupService>();
|
services.AddSingleton<CleanupService>();
|
||||||
|
|||||||
@@ -3,9 +3,9 @@ using MareSynchronosShared.Data;
|
|||||||
using MareSynchronosShared.Metrics;
|
using MareSynchronosShared.Metrics;
|
||||||
using MareSynchronosShared.Utils;
|
using MareSynchronosShared.Utils;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.Extensions.Configuration;
|
|
||||||
using Microsoft.Extensions.DependencyInjection;
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
namespace MareSynchronosShared.Authentication;
|
namespace MareSynchronosShared.Authentication;
|
||||||
|
|
||||||
@@ -20,15 +20,13 @@ public class SecretKeyAuthenticatorService
|
|||||||
private readonly int _tempBanMinutes;
|
private readonly int _tempBanMinutes;
|
||||||
private readonly List<string> _whitelistedIps;
|
private readonly List<string> _whitelistedIps;
|
||||||
|
|
||||||
public SecretKeyAuthenticatorService(MareMetrics metrics, IServiceScopeFactory serviceScopeFactory, IConfiguration configuration, ILogger<SecretKeyAuthenticatorService> logger)
|
public SecretKeyAuthenticatorService(MareMetrics metrics, IServiceScopeFactory serviceScopeFactory, IOptions<MareConfigurationAuthBase> configuration, ILogger<SecretKeyAuthenticatorService> logger)
|
||||||
{
|
{
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
var config = configuration.GetRequiredSection("MareSynchronos");
|
var config = configuration.Value;
|
||||||
_failedAttemptsForTempBan = config.GetValue<int>("FailedAuthForTempBan", 5);
|
_failedAttemptsForTempBan = config.FailedAuthForTempBan;
|
||||||
logger.LogInformation("FailedAuthForTempBan: {num}", _failedAttemptsForTempBan);
|
_tempBanMinutes = config.TempBanDurationInMinutes;
|
||||||
_tempBanMinutes = config.GetValue<int>("TempBanDurationInMinutes", 30);
|
_whitelistedIps = config.WhitelistedIps;
|
||||||
logger.LogInformation("TempBanMinutes: {num}", _tempBanMinutes);
|
|
||||||
_whitelistedIps = config.GetSection("WhitelistedIps").Get<List<string>>();
|
|
||||||
foreach (var ip in _whitelistedIps)
|
foreach (var ip in _whitelistedIps)
|
||||||
{
|
{
|
||||||
logger.LogInformation("Whitelisted IP: " + ip);
|
logger.LogInformation("Whitelisted IP: " + ip);
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
using System.Text;
|
||||||
|
|
||||||
|
namespace MareSynchronosShared.Utils;
|
||||||
|
|
||||||
|
public class MareConfigurationBase
|
||||||
|
{
|
||||||
|
public int DbContextPoolSize { get; set; } = 100;
|
||||||
|
|
||||||
|
public override string ToString()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new();
|
||||||
|
sb.AppendLine($"{nameof(DbContextPoolSize)} => {DbContextPoolSize}");
|
||||||
|
return sb.ToString();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class MareConfigurationAuthBase : MareConfigurationBase
|
||||||
|
{
|
||||||
|
public int DbContextPoolSize { get; set; } = 100;
|
||||||
|
public int FailedAuthForTempBan { get; set; } = 5;
|
||||||
|
public int TempBanDurationInMinutes { get; set; } = 5;
|
||||||
|
public List<string> WhitelistedIps { get; set; } = new();
|
||||||
|
|
||||||
|
public override string ToString()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new();
|
||||||
|
sb.AppendLine($"{nameof(FailedAuthForTempBan)} => {FailedAuthForTempBan}");
|
||||||
|
sb.AppendLine($"{nameof(TempBanDurationInMinutes)} => {TempBanDurationInMinutes}");
|
||||||
|
sb.AppendLine($"{nameof(WhitelistedIps)} => {string.Join(", ", WhitelistedIps)}");
|
||||||
|
return sb.ToString();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
using MareSynchronosShared.Metrics;
|
using MareSynchronosShared.Metrics;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
|
|
||||||
namespace MareSynchronosStaticFilesServer;
|
namespace MareSynchronosStaticFilesServer;
|
||||||
@@ -13,14 +14,13 @@ public class CachedFileProvider
|
|||||||
private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
|
private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
|
||||||
private bool IsMainServer => _remoteCacheSourceUri == null;
|
private bool IsMainServer => _remoteCacheSourceUri == null;
|
||||||
|
|
||||||
public CachedFileProvider(IConfiguration configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics)
|
public CachedFileProvider(IOptions<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics)
|
||||||
{
|
{
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
_fileStatisticsService = fileStatisticsService;
|
_fileStatisticsService = fileStatisticsService;
|
||||||
_metrics = metrics;
|
_metrics = metrics;
|
||||||
var configurationSection = configuration.GetRequiredSection("MareSynchronos");
|
_remoteCacheSourceUri = configuration.Value.RemoteCacheSourceUri;
|
||||||
_remoteCacheSourceUri = configurationSection.GetValue<Uri>("RemoteCacheSourceUri", null);
|
_basePath = configuration.Value.CacheDirectory;
|
||||||
_basePath = configurationSection["CacheDirectory"];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<FileStream?> GetFileStream(string hash, string auth)
|
public async Task<FileStream?> GetFileStream(string hash, string auth)
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
using MareSynchronosShared.Data;
|
using MareSynchronosShared.Data;
|
||||||
using MareSynchronosShared.Metrics;
|
using MareSynchronosShared.Metrics;
|
||||||
using MareSynchronosShared.Models;
|
using MareSynchronosShared.Models;
|
||||||
using System.Globalization;
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
namespace MareSynchronosStaticFilesServer;
|
namespace MareSynchronosStaticFilesServer;
|
||||||
|
|
||||||
@@ -11,19 +11,19 @@ public class FileCleanupService : IHostedService
|
|||||||
private readonly MareMetrics _metrics;
|
private readonly MareMetrics _metrics;
|
||||||
private readonly ILogger<FileCleanupService> _logger;
|
private readonly ILogger<FileCleanupService> _logger;
|
||||||
private readonly IServiceProvider _services;
|
private readonly IServiceProvider _services;
|
||||||
private readonly IConfiguration _configuration;
|
private readonly StaticFilesServerConfiguration _configuration;
|
||||||
private readonly bool _isMainServer;
|
private readonly bool _isMainServer;
|
||||||
private readonly string _cacheDir;
|
private readonly string _cacheDir;
|
||||||
private CancellationTokenSource _cleanupCts;
|
private CancellationTokenSource _cleanupCts;
|
||||||
|
|
||||||
public FileCleanupService(MareMetrics metrics, ILogger<FileCleanupService> logger, IServiceProvider services, IConfiguration configuration)
|
public FileCleanupService(MareMetrics metrics, ILogger<FileCleanupService> logger, IServiceProvider services, IOptions<StaticFilesServerConfiguration> configuration)
|
||||||
{
|
{
|
||||||
_metrics = metrics;
|
_metrics = metrics;
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
_services = services;
|
_services = services;
|
||||||
_configuration = configuration.GetRequiredSection("MareSynchronos");
|
_configuration = configuration.Value;
|
||||||
_isMainServer = string.IsNullOrEmpty(_configuration.GetValue("RemoteCacheSourceUri", string.Empty));
|
_isMainServer = _configuration.RemoteCacheSourceUri == null;
|
||||||
_cacheDir = _configuration.GetValue<string>("CacheDirectory");
|
_cacheDir = _configuration.CacheDirectory;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Task StartAsync(CancellationToken cancellationToken)
|
public Task StartAsync(CancellationToken cancellationToken)
|
||||||
@@ -67,21 +67,19 @@ public class FileCleanupService : IHostedService
|
|||||||
|
|
||||||
private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct)
|
private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct)
|
||||||
{
|
{
|
||||||
var cacheSizeLimitInGiB = _configuration.GetValue<double>("CacheSizeHardLimitInGiB", -1);
|
if (_configuration.CacheSizeHardLimitInGiB <= 0)
|
||||||
|
|
||||||
if (cacheSizeLimitInGiB <= 0)
|
|
||||||
{
|
{
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", cacheSizeLimitInGiB);
|
_logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", _configuration.CacheSizeHardLimitInGiB);
|
||||||
var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories)
|
var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories)
|
||||||
.Select(f => new FileInfo(f)).ToList()
|
.Select(f => new FileInfo(f)).ToList()
|
||||||
.OrderBy(f => f.LastAccessTimeUtc).ToList();
|
.OrderBy(f => f.LastAccessTimeUtc).ToList();
|
||||||
var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
|
var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
|
||||||
long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(cacheSizeLimitInGiB).Bytes;
|
long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(_configuration.CacheSizeHardLimitInGiB).Bytes;
|
||||||
while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested)
|
while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
var oldestFile = allLocalFiles[0];
|
var oldestFile = allLocalFiles[0];
|
||||||
@@ -108,18 +106,15 @@ public class FileCleanupService : IHostedService
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var filesOlderThanDays = _configuration.GetValue("UnusedFileRetentionPeriodInDays", 7);
|
_logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", _configuration.UnusedFileRetentionPeriodInDays);
|
||||||
var forcedDeletionHours = _configuration.GetValue("ForcedDeletionOfFilesAfterHours", -1);
|
if (_configuration.ForcedDeletionOfFilesAfterHours > 0)
|
||||||
|
|
||||||
_logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", filesOlderThanDays);
|
|
||||||
if (forcedDeletionHours > 0)
|
|
||||||
{
|
{
|
||||||
_logger.LogInformation("Cleaning up files written to longer than {hours}h ago", forcedDeletionHours);
|
_logger.LogInformation("Cleaning up files written to longer than {hours}h ago", _configuration.ForcedDeletionOfFilesAfterHours);
|
||||||
}
|
}
|
||||||
|
|
||||||
// clean up files in DB but not on disk or last access is expired
|
// clean up files in DB but not on disk or last access is expired
|
||||||
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays));
|
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(_configuration.UnusedFileRetentionPeriodInDays));
|
||||||
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionHours));
|
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(_configuration.ForcedDeletionOfFilesAfterHours));
|
||||||
var allFiles = dbContext.Files.ToList();
|
var allFiles = dbContext.Files.ToList();
|
||||||
foreach (var fileCache in allFiles.Where(f => f.Uploaded))
|
foreach (var fileCache in allFiles.Where(f => f.Uploaded))
|
||||||
{
|
{
|
||||||
@@ -138,7 +133,7 @@ public class FileCleanupService : IHostedService
|
|||||||
if (_isMainServer)
|
if (_isMainServer)
|
||||||
dbContext.Files.Remove(fileCache);
|
dbContext.Files.Remove(fileCache);
|
||||||
}
|
}
|
||||||
else if (file != null && forcedDeletionHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
|
else if (file != null && _configuration.ForcedDeletionOfFilesAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
|
||||||
{
|
{
|
||||||
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
|
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
|
||||||
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
|
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ using MareSynchronosShared.Data;
|
|||||||
using MareSynchronosShared.Metrics;
|
using MareSynchronosShared.Metrics;
|
||||||
using MareSynchronosShared.Protos;
|
using MareSynchronosShared.Protos;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
namespace MareSynchronosStaticFilesServer;
|
namespace MareSynchronosStaticFilesServer;
|
||||||
|
|
||||||
@@ -13,9 +14,9 @@ public class GrpcFileService : FileService.FileServiceBase
|
|||||||
private readonly ILogger<GrpcFileService> _logger;
|
private readonly ILogger<GrpcFileService> _logger;
|
||||||
private readonly MareMetrics _metricsClient;
|
private readonly MareMetrics _metricsClient;
|
||||||
|
|
||||||
public GrpcFileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger<GrpcFileService> logger, MareMetrics metricsClient)
|
public GrpcFileService(MareDbContext mareDbContext, IOptions<StaticFilesServerConfiguration> configuration, ILogger<GrpcFileService> logger, MareMetrics metricsClient)
|
||||||
{
|
{
|
||||||
_basePath = configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"];
|
_basePath = configuration.Value.CacheDirectory;
|
||||||
_mareDbContext = mareDbContext;
|
_mareDbContext = mareDbContext;
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
_metricsClient = metricsClient;
|
_metricsClient = metricsClient;
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
namespace MareSynchronosStaticFilesServer;
|
namespace MareSynchronosStaticFilesServer;
|
||||||
|
|
||||||
public class Program
|
public class Program
|
||||||
@@ -7,6 +9,14 @@ public class Program
|
|||||||
var hostBuilder = CreateHostBuilder(args);
|
var hostBuilder = CreateHostBuilder(args);
|
||||||
var host = hostBuilder.Build();
|
var host = hostBuilder.Build();
|
||||||
|
|
||||||
|
using (var scope = host.Services.CreateScope())
|
||||||
|
{
|
||||||
|
var options = host.Services.GetService<IOptions<StaticFilesServerConfiguration>>();
|
||||||
|
var logger = host.Services.GetService<ILogger<Program>>();
|
||||||
|
logger.LogInformation("Loaded MareSynchronos Static Files Server Configuration");
|
||||||
|
logger.LogInformation(options.Value.ToString());
|
||||||
|
}
|
||||||
|
|
||||||
host.Run();
|
host.Run();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
using Grpc.Net.Client.Configuration;
|
|
||||||
using MareSynchronosShared.Authentication;
|
using MareSynchronosShared.Authentication;
|
||||||
using MareSynchronosShared.Data;
|
using MareSynchronosShared.Data;
|
||||||
using MareSynchronosShared.Metrics;
|
using MareSynchronosShared.Metrics;
|
||||||
|
using MareSynchronosShared.Utils;
|
||||||
using Microsoft.AspNetCore.Authentication;
|
using Microsoft.AspNetCore.Authentication;
|
||||||
using Microsoft.AspNetCore.Authorization;
|
using Microsoft.AspNetCore.Authorization;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
@@ -16,7 +16,7 @@ public class Startup
|
|||||||
{
|
{
|
||||||
Configuration = configuration;
|
Configuration = configuration;
|
||||||
var mareSettings = Configuration.GetRequiredSection("MareSynchronos");
|
var mareSettings = Configuration.GetRequiredSection("MareSynchronos");
|
||||||
_isMain = string.IsNullOrEmpty(mareSettings.GetValue("RemoteCacheSourceUri", string.Empty));
|
_isMain = string.IsNullOrEmpty(mareSettings.GetValue(nameof(StaticFilesServerConfiguration.RemoteCacheSourceUri), string.Empty));
|
||||||
}
|
}
|
||||||
|
|
||||||
public IConfiguration Configuration { get; }
|
public IConfiguration Configuration { get; }
|
||||||
@@ -25,10 +25,12 @@ public class Startup
|
|||||||
{
|
{
|
||||||
services.AddHttpContextAccessor();
|
services.AddHttpContextAccessor();
|
||||||
|
|
||||||
services.AddTransient(_ => Configuration);
|
|
||||||
|
|
||||||
services.AddLogging();
|
services.AddLogging();
|
||||||
|
|
||||||
|
services.Configure<MareConfigurationAuthBase>(Configuration.GetRequiredSection("MareSynchronos"));
|
||||||
|
services.Configure<StaticFilesServerConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
|
||||||
|
services.AddSingleton(Configuration);
|
||||||
|
|
||||||
var mareSettings = Configuration.GetRequiredSection("MareSynchronos");
|
var mareSettings = Configuration.GetRequiredSection("MareSynchronos");
|
||||||
|
|
||||||
services.AddControllers();
|
services.AddControllers();
|
||||||
@@ -62,7 +64,7 @@ public class Startup
|
|||||||
builder.MigrationsHistoryTable("_efmigrationshistory", "public");
|
builder.MigrationsHistoryTable("_efmigrationshistory", "public");
|
||||||
}).UseSnakeCaseNamingConvention();
|
}).UseSnakeCaseNamingConvention();
|
||||||
options.EnableThreadSafetyChecks(false);
|
options.EnableThreadSafetyChecks(false);
|
||||||
}, mareSettings.GetValue("DbContextPoolSize", 1024));
|
}, mareSettings.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
|
||||||
|
|
||||||
services.AddAuthentication(options =>
|
services.AddAuthentication(options =>
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,25 @@
|
|||||||
|
using MareSynchronosShared.Utils;
|
||||||
|
using System.Text;
|
||||||
|
|
||||||
|
namespace MareSynchronosStaticFilesServer;
|
||||||
|
|
||||||
|
public class StaticFilesServerConfiguration : MareConfigurationAuthBase
|
||||||
|
{
|
||||||
|
public int ForcedDeletionOfFilesAfterHours { get; set; } = -1;
|
||||||
|
public double CacheSizeHardLimitInGiB { get; set; } = -1;
|
||||||
|
public int UnusedFileRetentionPeriodInDays { get; set; } = -1;
|
||||||
|
public string CacheDirectory { get; set; }
|
||||||
|
public Uri? RemoteCacheSourceUri { get; set; } = null;
|
||||||
|
|
||||||
|
public override string ToString()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new();
|
||||||
|
sb.AppendLine(base.ToString());
|
||||||
|
sb.AppendLine($"{nameof(ForcedDeletionOfFilesAfterHours)} => {ForcedDeletionOfFilesAfterHours}");
|
||||||
|
sb.AppendLine($"{nameof(CacheSizeHardLimitInGiB)} => {CacheSizeHardLimitInGiB}");
|
||||||
|
sb.AppendLine($"{nameof(UnusedFileRetentionPeriodInDays)} => {UnusedFileRetentionPeriodInDays}");
|
||||||
|
sb.AppendLine($"{nameof(CacheDirectory)} => {CacheDirectory}");
|
||||||
|
sb.AppendLine($"{nameof(RemoteCacheSourceUri)} => {RemoteCacheSourceUri}");
|
||||||
|
return sb.ToString();
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user