t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rework server responsibilities (#18)

Browse Source 
* rework server responsibilities
add remote configuration

* start metrics only when compiled as not debug

* add some more logging to discord bot

* fixes of some casts

* make metrics port configurable, minor fixes

* add docker bullshit

* md formatting

* adjustments to docker stuff

* fix docker json files, fix some stuff in discord bot, add /useradd for Discord bot

* adjust docker configs and fix sharded.bat

* fixes for logs, cache file provider repeat trying to open filestream

Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>
This commit is contained in:
rootdarkarchon
2022-12-27 13:48:05 +01:00
committed by GitHub
parent 7ee7fdaf48
commit 9eb5967935
101 changed files with 2470 additions and 585 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
MareSynchronosServer/MareSynchronosStaticFilesServer/CachedFileProvider.cs
View File

@@ -1,4 +1,5 @@
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Services;
using Microsoft.Extensions.Options;
using System.Collections.Concurrent;
@@ -14,13 +15,13 @@ public class CachedFileProvider
private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
private bool IsMainServer => _remoteCacheSourceUri == null;
public CachedFileProvider(IOptions<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics)
public CachedFileProvider(IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics)
{
_logger = logger;
_fileStatisticsService = fileStatisticsService;
_metrics = metrics;
_remoteCacheSourceUri = configuration.Value.RemoteCacheSourceUri;
_basePath = configuration.Value.CacheDirectory;
_remoteCacheSourceUri = configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.RemoteCacheSourceUri), null);
_basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
}
public async Task<FileStream?> GetFileStream(string hash, string auth)
@@ -79,6 +80,21 @@ public class CachedFileProvider
_fileStatisticsService.LogFile(hash, fi.Length);
return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read);
int attempts = 0;
while (attempts < 5)
{
try
{
return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read);
}
catch (Exception ex)
{
attempts++;
_logger.LogWarning(ex, "Error opening file, retrying");
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
}
throw new IOException("Could not open file " + fi.FullName);
}
}

83
MareSynchronosServer/MareSynchronosStaticFilesServer/FileCleanupService.cs
View File

@@ -2,6 +2,7 @@
using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Models;
using MareSynchronosShared.Services;
using Microsoft.Extensions.Options;
namespace MareSynchronosStaticFilesServer;
@@ -11,19 +12,19 @@ public class FileCleanupService : IHostedService
private readonly MareMetrics _metrics;
private readonly ILogger<FileCleanupService> _logger;
private readonly IServiceProvider _services;
private readonly StaticFilesServerConfiguration _configuration;
private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
private readonly bool _isMainServer;
private readonly string _cacheDir;
private CancellationTokenSource _cleanupCts;
public FileCleanupService(MareMetrics metrics, ILogger<FileCleanupService> logger, IServiceProvider services, IOptions<StaticFilesServerConfiguration> configuration)
public FileCleanupService(MareMetrics metrics, ILogger<FileCleanupService> logger, IServiceProvider services, IConfigurationService<StaticFilesServerConfiguration> configuration)
{
_metrics = metrics;
_logger = logger;
_services = services;
_configuration = configuration.Value;
_isMainServer = _configuration.RemoteCacheSourceUri == null;
_cacheDir = _configuration.CacheDirectory;
_configuration = configuration;
_isMainServer = configuration.IsMain;
_cacheDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
}
public Task StartAsync(CancellationToken cancellationToken)
@@ -60,26 +61,32 @@ public class FileCleanupService : IHostedService
await dbContext.SaveChangesAsync(ct).ConfigureAwait(false);
}
_logger.LogInformation("File Cleanup Complete, next run at {date}", DateTime.Now.Add(TimeSpan.FromMinutes(10)));
await Task.Delay(TimeSpan.FromMinutes(10), ct).ConfigureAwait(false);
var now = DateTime.Now;
TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % 10, 0);
var span = futureTime.AddMinutes(10) - currentTime;
_logger.LogInformation("File Cleanup Complete, next run at {date}", now.Add(span));
await Task.Delay(span, ct).ConfigureAwait(false);
}
}
private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct)
{
if (_configuration.CacheSizeHardLimitInGiB <= 0)
var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);
if (sizeLimit <= 0)
{
return;
}
try
{
_logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", _configuration.CacheSizeHardLimitInGiB);
_logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", sizeLimit);
var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories)
.Select(f => new FileInfo(f)).ToList()
.OrderBy(f => f.LastAccessTimeUtc).ToList();
var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(_configuration.CacheSizeHardLimitInGiB).Bytes;
long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes;
while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested)
{
var oldestFile = allLocalFiles[0];
@@ -106,15 +113,18 @@ public class FileCleanupService : IHostedService
{
try
{
_logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", _configuration.UnusedFileRetentionPeriodInDays);
if (_configuration.ForcedDeletionOfFilesAfterHours > 0)
var unusedRetention = _configuration.GetValueOrDefault<int>(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
var forcedDeletionAfterHours = _configuration.GetValueOrDefault<int>(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
_logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", unusedRetention);
if (forcedDeletionAfterHours > 0)
{
_logger.LogInformation("Cleaning up files written to longer than {hours}h ago", _configuration.ForcedDeletionOfFilesAfterHours);
_logger.LogInformation("Cleaning up files written to longer than {hours}h ago", forcedDeletionAfterHours);
}
// clean up files in DB but not on disk or last access is expired
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(_configuration.UnusedFileRetentionPeriodInDays));
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(_configuration.ForcedDeletionOfFilesAfterHours));
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention));
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
var allFiles = dbContext.Files.ToList();
foreach (var fileCache in allFiles.Where(f => f.Uploaded))
{
@@ -133,7 +143,7 @@ public class FileCleanupService : IHostedService
if (_isMainServer)
dbContext.Files.Remove(fileCache);
}
else if (file != null && _configuration.ForcedDeletionOfFilesAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
{
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
@@ -147,24 +157,7 @@ public class FileCleanupService : IHostedService
}
// clean up files that are on disk but not in DB for some reason
if (_isMainServer)
{
var allFilesHashes = new HashSet<string>(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal);
DirectoryInfo dir = new(_cacheDir);
var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories);
foreach (var file in allFilesInDir)
{
if (!allFilesHashes.Contains(file.Name.ToUpperInvariant()))
{
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
file.Delete();
_logger.LogInformation("File not in DB, deleting: {fileName}", file.Name);
}
ct.ThrowIfCancellationRequested();
}
}
CleanUpOrphanedFiles(allFiles, ct);
}
catch (Exception ex)
{
@@ -172,6 +165,28 @@ public class FileCleanupService : IHostedService
}
}
private void CleanUpOrphanedFiles(List<FileCache> allFiles, CancellationToken ct)
{
if (_isMainServer)
{
var allFilesHashes = new HashSet<string>(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal);
DirectoryInfo dir = new(_cacheDir);
var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories);
foreach (var file in allFilesInDir)
{
if (!allFilesHashes.Contains(file.Name.ToUpperInvariant()))
{
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
file.Delete();
_logger.LogInformation("File not in DB, deleting: {fileName}", file.Name);
}
ct.ThrowIfCancellationRequested();
}
}
}
public Task StopAsync(CancellationToken cancellationToken)
{
_cleanupCts.Cancel();

16
MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs
View File

@@ -51,7 +51,13 @@ public class FileStatisticsService : IHostedService
_pastHourFiles = new(StringComparer.Ordinal);
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHour, 0);
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHourSize, 0);
await Task.Delay(TimeSpan.FromHours(1), _resetCancellationTokenSource.Token).ConfigureAwait(false);
var now = DateTime.UtcNow;
TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
TimeOnly futureTime = new(now.Hour, 0, 0);
var span = futureTime.AddHours(1) - currentTime;
await Task.Delay(span, _resetCancellationTokenSource.Token).ConfigureAwait(false);
}
}
@@ -64,7 +70,13 @@ public class FileStatisticsService : IHostedService
_pastDayFiles = new(StringComparer.Ordinal);
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDay, 0);
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDaySize, 0);
await Task.Delay(TimeSpan.FromDays(1), _resetCancellationTokenSource.Token).ConfigureAwait(false);
var now = DateTime.UtcNow;
TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
TimeOnly futureTime = new(0, 0, 0);
var span = futureTime - currentTime;
await Task.Delay(span, _resetCancellationTokenSource.Token).ConfigureAwait(false);
}
}

2
MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs
View File

@@ -18,7 +18,7 @@ public class FilesController : Controller
[HttpGet("{fileId}")]
public async Task<IActionResult> GetFile(string fileId)
{
var authedUser = HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, ClaimTypes.NameIdentifier, System.StringComparison.Ordinal))?.Value ?? "Unknown";
var authedUser = HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, ClaimTypes.NameIdentifier, StringComparison.Ordinal))?.Value ?? "Unknown";
_logger.LogInformation($"GetFile:{authedUser}:{fileId}");
var fs = await _cachedFileProvider.GetFileStream(fileId, Request.Headers["Authorization"]);

6
MareSynchronosServer/MareSynchronosStaticFilesServer/GrpcFileService.cs
View File

@@ -2,8 +2,8 @@
using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Protos;
using MareSynchronosShared.Services;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
namespace MareSynchronosStaticFilesServer;
@@ -14,9 +14,9 @@ public class GrpcFileService : FileService.FileServiceBase
private readonly ILogger<GrpcFileService> _logger;
private readonly MareMetrics _metricsClient;
public GrpcFileService(MareDbContext mareDbContext, IOptions<StaticFilesServerConfiguration> configuration, ILogger<GrpcFileService> logger, MareMetrics metricsClient)
public GrpcFileService(MareDbContext mareDbContext, IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<GrpcFileService> logger, MareMetrics metricsClient)
{
_basePath = configuration.Value.CacheDirectory;
_basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
_mareDbContext = mareDbContext;
_logger = logger;
_metricsClient = metricsClient;

11
MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs
View File

@@ -1,3 +1,5 @@
using MareSynchronosShared.Services;
using MareSynchronosShared.Utils;
using Microsoft.Extensions.Options;
namespace MareSynchronosStaticFilesServer;
@@ -11,10 +13,13 @@ public class Program
using (var scope = host.Services.CreateScope())
{
var options = host.Services.GetService<IOptions<StaticFilesServerConfiguration>>();
var options = host.Services.GetService<IConfigurationService<StaticFilesServerConfiguration>>();
var optionsServer = host.Services.GetService<IConfigurationService<MareConfigurationAuthBase>>();
var logger = host.Services.GetService<ILogger<Program>>();
logger.LogInformation("Loaded MareSynchronos Static Files Server Configuration");
logger.LogInformation(options.Value.ToString());
logger.LogInformation("Loaded MareSynchronos Static Files Server Configuration (IsMain: {isMain})", options.IsMain);
logger.LogInformation(options.ToString());
logger.LogInformation("Loaded MareSynchronos Server Auth Configuration (IsMain: {isMain})", optionsServer.IsMain);
logger.LogInformation(optionsServer.ToString());
}
host.Run();

75
MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs
View File

@@ -1,10 +1,15 @@
using Grpc.Net.Client.Configuration;
using Grpc.Net.ClientFactory;
using MareSynchronosShared.Authentication;
using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Protos;
using MareSynchronosShared.Services;
using MareSynchronosShared.Utils;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authorization;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
using Prometheus;
namespace MareSynchronosStaticFilesServer;
@@ -27,11 +32,11 @@ public class Startup
services.AddLogging();
services.Configure<MareConfigurationAuthBase>(Configuration.GetRequiredSection("MareSynchronos"));
services.Configure<StaticFilesServerConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
services.Configure<MareConfigurationAuthBase>(Configuration.GetRequiredSection("MareSynchronos"));
services.AddSingleton(Configuration);
var mareSettings = Configuration.GetRequiredSection("MareSynchronos");
var mareConfig = Configuration.GetRequiredSection("MareSynchronos");
services.AddControllers();
@@ -64,7 +69,37 @@ public class Startup
builder.MigrationsHistoryTable("_efmigrationshistory", "public");
}).UseSnakeCaseNamingConvention();
options.EnableThreadSafetyChecks(false);
}, mareSettings.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
}, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
var noRetryConfig = new MethodConfig
{
Names = { MethodName.Default },
RetryPolicy = null
};
services.AddGrpcClient<ConfigurationService.ConfigurationServiceClient>("FileServer", c =>
{
c.Address = new Uri(mareConfig.GetValue<string>(nameof(StaticFilesServerConfiguration.FileServerGrpcAddress)));
}).ConfigureChannel(c =>
{
c.ServiceConfig = new ServiceConfig { MethodConfigs = { noRetryConfig } };
c.HttpHandler = new SocketsHttpHandler()
{
EnableMultipleHttp2Connections = true
};
});
services.AddGrpcClient<ConfigurationService.ConfigurationServiceClient>("MainServer", c =>
{
c.Address = new Uri(mareConfig.GetValue<string>(nameof(StaticFilesServerConfiguration.MainServerGrpcAddress)));
}).ConfigureChannel(c =>
{
c.ServiceConfig = new ServiceConfig { MethodConfigs = { noRetryConfig } };
c.HttpHandler = new SocketsHttpHandler()
{
EnableMultipleHttp2Connections = true
};
});
services.AddAuthentication(options =>
{
@@ -73,10 +108,30 @@ public class Startup
.AddScheme<AuthenticationSchemeOptions, SecretKeyAuthenticationHandler>(SecretKeyAuthenticationHandler.AuthScheme, options => { });
services.AddAuthorization(options => options.FallbackPolicy = new AuthorizationPolicyBuilder().RequireAuthenticatedUser().Build());
services.AddGrpc(o =>
if (_isMain)
{
o.MaxReceiveMessageSize = null;
});
services.AddGrpc(o =>
{
o.MaxReceiveMessageSize = null;
});
services.AddSingleton<IConfigurationService<StaticFilesServerConfiguration>, MareConfigurationServiceServer<StaticFilesServerConfiguration>>();
}
else
{
services.AddSingleton<IConfigurationService<StaticFilesServerConfiguration>>(p => new MareConfigurationServiceClient<StaticFilesServerConfiguration>(
p.GetRequiredService<ILogger<MareConfigurationServiceClient<StaticFilesServerConfiguration>>>(),
p.GetRequiredService<IOptions<StaticFilesServerConfiguration>>(),
p.GetRequiredService<GrpcClientFactory>(),
"FileServer"));
}
services.AddSingleton<IConfigurationService<MareConfigurationAuthBase>>(p =>
new MareConfigurationServiceClient<MareConfigurationAuthBase>(
p.GetRequiredService<ILogger<MareConfigurationServiceClient<MareConfigurationAuthBase>>>(),
p.GetService<IOptions<MareConfigurationAuthBase>>(),
p.GetRequiredService<GrpcClientFactory>(), "MainServer")
);
}
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
@@ -85,7 +140,9 @@ public class Startup
app.UseRouting();
var metricServer = new KestrelMetricServer(4981);
var config = app.ApplicationServices.GetRequiredService<IConfigurationService<MareConfigurationAuthBase>>();
var metricServer = new KestrelMetricServer(config.GetValueOrDefault<int>(nameof(MareConfigurationBase.MetricsPort), 4981));
metricServer.Start();
app.UseHttpMetrics();
@@ -95,8 +152,10 @@ public class Startup
app.UseEndpoints(e =>
{
if(_isMain)
if (_isMain)
{
e.MapGrpcService<GrpcFileService>();
}
e.MapControllers();
});
}

25
MareSynchronosServer/MareSynchronosStaticFilesServer/StaticFilesServerConfiguration.cs
View File

@@ -1,25 +0,0 @@
using MareSynchronosShared.Utils;
using System.Text;
namespace MareSynchronosStaticFilesServer;
public class StaticFilesServerConfiguration : MareConfigurationAuthBase
{
public int ForcedDeletionOfFilesAfterHours { get; set; } = -1;
public double CacheSizeHardLimitInGiB { get; set; } = -1;
public int UnusedFileRetentionPeriodInDays { get; set; } = -1;
public string CacheDirectory { get; set; }
public Uri? RemoteCacheSourceUri { get; set; } = null;
public override string ToString()
{
StringBuilder sb = new();
sb.AppendLine(base.ToString());
sb.AppendLine($"{nameof(ForcedDeletionOfFilesAfterHours)} => {ForcedDeletionOfFilesAfterHours}");
sb.AppendLine($"{nameof(CacheSizeHardLimitInGiB)} => {CacheSizeHardLimitInGiB}");
sb.AppendLine($"{nameof(UnusedFileRetentionPeriodInDays)} => {UnusedFileRetentionPeriodInDays}");
sb.AppendLine($"{nameof(CacheDirectory)} => {CacheDirectory}");
sb.AppendLine($"{nameof(RemoteCacheSourceUri)} => {RemoteCacheSourceUri}");
return sb.ToString();
}
}