t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rename MainFileCleanupService.cs for better git tracking in the next commit

Browse Source
This commit is contained in:
Loporrit
2024-09-01 15:49:39 +00:00
parent fb0905ef84
commit 649f3018b1
3 changed files with 5 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs → MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs
View File

@@ -8,15 +8,15 @@ using Microsoft.EntityFrameworkCore;
namespace MareSynchronosStaticFilesServer.Services; namespace MareSynchronosStaticFilesServer.Services;
public class MainFileCleanupService : IHostedService public class FileCleanupService : IHostedService
{ {
private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration; private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
private readonly ILogger<MainFileCleanupService> _logger; private readonly ILogger<FileCleanupService> _logger;
private readonly MareMetrics _metrics; private readonly MareMetrics _metrics;
private readonly IServiceProvider _services; private readonly IServiceProvider _services;
private CancellationTokenSource _cleanupCts; private CancellationTokenSource _cleanupCts;
public MainFileCleanupService(MareMetrics metrics, ILogger<MainFileCleanupService> logger, public FileCleanupService(MareMetrics metrics, ILogger<FileCleanupService> logger,
IServiceProvider services, IConfigurationService<StaticFilesServerConfiguration> configuration) IServiceProvider services, IConfigurationService<StaticFilesServerConfiguration> configuration)
{ {
_metrics = metrics; _metrics = metrics;

160
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardFileCleanupService.cs
View File

@@ -1,160 +0,0 @@
using ByteSizeLib;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Services;
using Microsoft.EntityFrameworkCore;
namespace MareSynchronosStaticFilesServer.Services;
public class ShardFileCleanupService : IHostedService
{
private readonly string _cacheDir;
private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
private readonly ILogger<MainFileCleanupService> _logger;
private readonly MareMetrics _metrics;
private CancellationTokenSource _cleanupCts;
public ShardFileCleanupService(MareMetrics metrics, ILogger<MainFileCleanupService> logger, IConfigurationService<StaticFilesServerConfiguration> configuration)
{
_metrics = metrics;
_logger = logger;
_configuration = configuration;
_cacheDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
}
public async Task CleanUpTask(CancellationToken ct)
{
_logger.LogInformation("Starting periodic cleanup task");
while (!ct.IsCancellationRequested)
{
try
{
DirectoryInfo dir = new(_cacheDir);
var allFiles = dir.GetFiles("*", SearchOption.AllDirectories);
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFiles.Sum(f => f.Length));
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFiles.Length);
CleanUpOutdatedFiles(ct);
CleanUpFilesBeyondSizeLimit(ct);
}
catch (Exception e)
{
_logger.LogError(e, "Error during cleanup task");
}
var now = DateTime.Now;
TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % 15, 0);
var span = futureTime.AddMinutes(15) - currentTime;
_logger.LogInformation("File Cleanup Complete, next run at {date}", now.Add(span));
await Task.Delay(span, ct).ConfigureAwait(false);
}
}
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Cleanup Service started");
_cleanupCts = new();
_ = CleanUpTask(_cleanupCts.Token);
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken)
{
_cleanupCts.Cancel();
return Task.CompletedTask;
}
private void CleanUpFilesBeyondSizeLimit(CancellationToken ct)
{
var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);
if (sizeLimit <= 0)
{
return;
}
try
{
_logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", sizeLimit);
var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories)
.Where(f => !f.EndsWith("dl", StringComparison.OrdinalIgnoreCase))
.Select(f => new FileInfo(f)).ToList()
.OrderBy(f => f.LastAccessTimeUtc).ToList();
var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes;
while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested)
{
var oldestFile = allLocalFiles[0];
allLocalFiles.Remove(oldestFile);
totalCacheSizeInBytes -= oldestFile.Length;
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, oldestFile.Length);
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
_logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes);
oldestFile.Delete();
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error during cache size limit cleanup");
}
}
private void CleanUpOutdatedFiles(CancellationToken ct)
{
try
{
var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
_logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", unusedRetention);
if (forcedDeletionAfterHours > 0)
{
_logger.LogInformation("Cleaning up files written to longer than {hours}h ago", forcedDeletionAfterHours);
}
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention));
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
DirectoryInfo dir = new(_cacheDir);
var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories)
.Where(f => !f.Name.EndsWith("dl", StringComparison.OrdinalIgnoreCase))
.ToList();
foreach (var file in allFilesInDir)
{
if (file.LastAccessTime < prevTime)
{
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
_logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
file.Delete();
}
else if (forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
{
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
_logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
file.Delete();
}
else if (file.Length == 0 && !string.Equals(file.Extension, ".dl", StringComparison.OrdinalIgnoreCase))
{
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
_logger.LogInformation("File with size 0 deleted: {filename}", file.Name);
file.Delete();
}
ct.ThrowIfCancellationRequested();
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error during file cleanup of old files");
}
}
}

4
MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs
View File

@@ -95,7 +95,7 @@ public class Startup
if (_isMain) if (_isMain)
{ {
services.AddSingleton<IClientReadyMessageService, MainClientReadyMessageService>(); services.AddSingleton<IClientReadyMessageService, MainClientReadyMessageService>();
services.AddHostedService<MainFileCleanupService>(); services.AddHostedService<FileCleanupService>();
services.AddSingleton<IConfigurationService<StaticFilesServerConfiguration>, MareConfigurationServiceServer<StaticFilesServerConfiguration>>(); services.AddSingleton<IConfigurationService<StaticFilesServerConfiguration>, MareConfigurationServiceServer<StaticFilesServerConfiguration>>();
services.AddDbContextPool<MareDbContext>(options => services.AddDbContextPool<MareDbContext>(options =>
{ {
@@ -196,7 +196,7 @@ public class Startup
} }
}); });
// authentication and authorization // authentication and authorization
services.AddOptions<JwtBearerOptions>(JwtBearerDefaults.AuthenticationScheme) services.AddOptions<JwtBearerOptions>(JwtBearerDefaults.AuthenticationScheme)
.Configure<IConfigurationService<MareConfigurationAuthBase>>((o, s) => .Configure<IConfigurationService<MareConfigurationAuthBase>>((o, s) =>
{ {