diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs b/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs index 6786d2d..a9ed59b 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs @@ -7,6 +7,7 @@ public class StaticFilesServerConfiguration : MareConfigurationBase { public bool IsDistributionNode { get; set; } = false; public Uri? MainFileServerAddress { get; set; } = null; + public Uri? DistributionFileServerAddress { get; set; } = null; public int ForcedDeletionOfFilesAfterHours { get; set; } = -1; public double CacheSizeHardLimitInGiB { get; set; } = -1; public int UnusedFileRetentionPeriodInDays { get; set; } = 14; @@ -15,6 +16,7 @@ public class StaticFilesServerConfiguration : MareConfigurationBase public int DownloadTimeoutSeconds { get; set; } = 5; public int DownloadQueueReleaseSeconds { get; set; } = 15; public int DownloadQueueClearLimit { get; set; } = 15000; + public int CleanupCheckInMinutes { get; set; } = 15; [RemoteConfiguration] public Uri CdnFullUrl { get; set; } = null; [RemoteConfiguration] diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs index d6d7c5e..289411e 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs @@ -30,7 +30,7 @@ public sealed class CachedFileProvider : IDisposable _fileStatisticsService = fileStatisticsService; _metrics = metrics; _generator = generator; - _remoteCacheSourceUri = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.MainFileServerAddress), null); + _remoteCacheSourceUri = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DistributionFileServerAddress), null); _isDistributionServer = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false); _basePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); _httpClient = new(); @@ -69,7 +69,8 @@ public sealed class CachedFileProvider : IDisposable } var fileName = FilePathUtil.GetFilePath(_basePath, hash); - using var fileStream = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite); + var tempFileName = fileName + ".dl"; + var fileStream = new FileStream(tempFileName, FileMode.Create, FileAccess.ReadWrite); var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024; var buffer = new byte[bufferSize]; @@ -79,6 +80,9 @@ public sealed class CachedFileProvider : IDisposable { await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead)).ConfigureAwait(false); } + await fileStream.FlushAsync().ConfigureAwait(false); + await fileStream.DisposeAsync().ConfigureAwait(false); + File.Move(tempFileName, fileName, true); _metrics.IncGauge(MetricsAPI.GaugeFilesTotal); _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_basePath, hash).Length); @@ -90,7 +94,7 @@ public sealed class CachedFileProvider : IDisposable if (fi == null && IsMainServer) return; await _downloadSemaphore.WaitAsync().ConfigureAwait(false); - if (fi == null && !_currentTransfers.ContainsKey(hash)) + if ((fi == null || (fi?.Length ?? 0) == 0) && !_currentTransfers.ContainsKey(hash)) { _currentTransfers[hash] = Task.Run(async () => { diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs index 6f87fa3..bcd9de2 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs @@ -17,7 +17,8 @@ public class MainFileCleanupService : IHostedService private readonly IServiceProvider _services; private CancellationTokenSource _cleanupCts; - public MainFileCleanupService(MareMetrics metrics, ILogger logger, IServiceProvider services, IConfigurationService configuration) + public MainFileCleanupService(MareMetrics metrics, ILogger logger, + IServiceProvider services, IConfigurationService configuration) { _metrics = metrics; _logger = logger; @@ -54,11 +55,11 @@ public class MainFileCleanupService : IHostedService { _logger.LogError(e, "Error during cleanup task"); } - + var cleanupCheckMinutes = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CleanupCheckInMinutes), 15); var now = DateTime.Now; TimeOnly currentTime = new(now.Hour, now.Minute, now.Second); - TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % 10, 0); - var span = futureTime.AddMinutes(10) - currentTime; + TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % cleanupCheckMinutes, 0); + var span = futureTime.AddMinutes(cleanupCheckMinutes) - currentTime; _logger.LogInformation("File Cleanup Complete, next run at {date}", now.Add(span)); await Task.Delay(span, ct).ConfigureAwait(false); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardFileCleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardFileCleanupService.cs index cb26ddb..549ff38 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardFileCleanupService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardFileCleanupService.cs @@ -121,7 +121,6 @@ public class ShardFileCleanupService : IHostedService var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours)); DirectoryInfo dir = new(_cacheDir); var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories); - int fileCounter = 0; foreach (var file in allFilesInDir) { @@ -139,8 +138,13 @@ public class ShardFileCleanupService : IHostedService _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); file.Delete(); } - - fileCounter++; + else if (file.Length == 0 && !string.Equals(file.Extension, ".dl", StringComparison.OrdinalIgnoreCase)) + { + _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); + _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); + _logger.LogInformation("File with size 0 deleted: {filename}", file.Name); + file.Delete(); + } ct.ThrowIfCancellationRequested(); }