t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make cleanup check time configurable, add distribution file server address

Browse Source 
handle download errors on shards better

flush filestream

oml
This commit is contained in:
rootdarkarchon
2024-01-17 01:32:11 +01:00
committed by Loporrit
parent 7c6dbafce9
commit a42f553648
4 changed files with 21 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs
View File

@@ -7,6 +7,7 @@ public class StaticFilesServerConfiguration : MareConfigurationBase
{
public bool IsDistributionNode { get; set; } = false;
public Uri? MainFileServerAddress { get; set; } = null;
public Uri? DistributionFileServerAddress { get; set; } = null;
public int ForcedDeletionOfFilesAfterHours { get; set; } = -1;
public double CacheSizeHardLimitInGiB { get; set; } = -1;
public int UnusedFileRetentionPeriodInDays { get; set; } = 14;
@@ -15,6 +16,7 @@ public class StaticFilesServerConfiguration : MareConfigurationBase
public int DownloadTimeoutSeconds { get; set; } = 5;
public int DownloadQueueReleaseSeconds { get; set; } = 15;
public int DownloadQueueClearLimit { get; set; } = 15000;
public int CleanupCheckInMinutes { get; set; } = 15;
[RemoteConfiguration]
public Uri CdnFullUrl { get; set; } = null;
[RemoteConfiguration]

10
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs
View File

@@ -30,7 +30,7 @@ public sealed class CachedFileProvider : IDisposable
_fileStatisticsService = fileStatisticsService;
_metrics = metrics;
_generator = generator;
_remoteCacheSourceUri = configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.MainFileServerAddress), null);
_remoteCacheSourceUri = configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.DistributionFileServerAddress), null);
_isDistributionServer = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false);
_basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
_httpClient = new();
@@ -69,7 +69,8 @@ public sealed class CachedFileProvider : IDisposable
}
var fileName = FilePathUtil.GetFilePath(_basePath, hash);
using var fileStream = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite);
var tempFileName = fileName + ".dl";
var fileStream = new FileStream(tempFileName, FileMode.Create, FileAccess.ReadWrite);
var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024;
var buffer = new byte[bufferSize];
@@ -79,6 +80,9 @@ public sealed class CachedFileProvider : IDisposable
{
await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead)).ConfigureAwait(false);
}
await fileStream.FlushAsync().ConfigureAwait(false);
await fileStream.DisposeAsync().ConfigureAwait(false);
File.Move(tempFileName, fileName, true);
_metrics.IncGauge(MetricsAPI.GaugeFilesTotal);
_metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_basePath, hash).Length);
@@ -90,7 +94,7 @@ public sealed class CachedFileProvider : IDisposable
if (fi == null && IsMainServer) return;
await _downloadSemaphore.WaitAsync().ConfigureAwait(false);
if (fi == null && !_currentTransfers.ContainsKey(hash))
if ((fi == null || (fi?.Length ?? 0) == 0) && !_currentTransfers.ContainsKey(hash))
{
_currentTransfers[hash] = Task.Run(async () =>
{

9
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs
View File

@@ -17,7 +17,8 @@ public class MainFileCleanupService : IHostedService
private readonly IServiceProvider _services;
private CancellationTokenSource _cleanupCts;
public MainFileCleanupService(MareMetrics metrics, ILogger<MainFileCleanupService> logger, IServiceProvider services, IConfigurationService<StaticFilesServerConfiguration> configuration)
public MainFileCleanupService(MareMetrics metrics, ILogger<MainFileCleanupService> logger,
IServiceProvider services, IConfigurationService<StaticFilesServerConfiguration> configuration)
{
_metrics = metrics;
_logger = logger;
@@ -54,11 +55,11 @@ public class MainFileCleanupService : IHostedService
{
_logger.LogError(e, "Error during cleanup task");
}
var cleanupCheckMinutes = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CleanupCheckInMinutes), 15);
var now = DateTime.Now;
TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % 10, 0);
var span = futureTime.AddMinutes(10) - currentTime;
TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % cleanupCheckMinutes, 0);
var span = futureTime.AddMinutes(cleanupCheckMinutes) - currentTime;
_logger.LogInformation("File Cleanup Complete, next run at {date}", now.Add(span));
await Task.Delay(span, ct).ConfigureAwait(false);

10
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardFileCleanupService.cs
View File

@@ -121,7 +121,6 @@ public class ShardFileCleanupService : IHostedService
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
DirectoryInfo dir = new(_cacheDir);
var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories);
int fileCounter = 0;
foreach (var file in allFilesInDir)
{
@@ -139,8 +138,13 @@ public class ShardFileCleanupService : IHostedService
_logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
file.Delete();
}
fileCounter++;
else if (file.Length == 0 && !string.Equals(file.Extension, ".dl", StringComparison.OrdinalIgnoreCase))
{
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
_logger.LogInformation("File with size 0 deleted: {filename}", file.Name);
file.Delete();
}
ct.ThrowIfCancellationRequested();
}