From fb22a267ac531db2d947432ca64b36a391d6cb07 Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Thu, 2 May 2024 14:03:42 +0200 Subject: [PATCH] add central cold storage options for file servers add access times I guess access, not write overwrite times after copying --- .../Metrics/MetricsAPI.cs | 2 + .../Utils/StaticFilesServerConfiguration.cs | 8 ++ .../Controllers/ServerFilesController.cs | 16 ++- .../Services/CachedFileProvider.cs | 68 +++++++++-- .../Services/MainFileCleanupService.cs | 106 ++++++++++++------ .../Startup.cs | 2 + 6 files changed, 151 insertions(+), 51 deletions(-) diff --git a/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs b/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs index cb78e67..d821592 100644 --- a/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs +++ b/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs @@ -12,7 +12,9 @@ public class MetricsAPI public const string GaugePairs = "mare_pairs"; public const string GaugePairsPaused = "mare_pairs_paused"; public const string GaugeFilesTotal = "mare_files"; + public const string GaugeFilesTotalColdStorage = "mare_files_cold"; public const string GaugeFilesTotalSize = "mare_files_size"; + public const string GaugeFilesTotalSizeColdStorage = "mare_files_size_cold"; public const string GaugeFilesDownloadingFromCache = "mare_files_downloading_from_cache"; public const string GaugeFilesTasksWaitingForDownloadFromCache = "mare_files_waiting_for_dl"; public const string CounterUserPushData = "mare_user_push"; diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs b/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs index a9ed59b..977a7f2 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs @@ -17,6 +17,10 @@ public class StaticFilesServerConfiguration : MareConfigurationBase public int DownloadQueueReleaseSeconds { get; set; } = 15; public int DownloadQueueClearLimit { get; set; } = 15000; public int CleanupCheckInMinutes { get; set; } = 15; + public bool UseColdStorage { get; set; } = false; + public string? ColdStorageDirectory { get; set; } = null; + public double ColdStorageSizeHardLimitInGiB { get; set; } = -1; + public double ColdStorageUnusedFileRetentionPeriodInDays { get; set; } = 30; [RemoteConfiguration] public Uri CdnFullUrl { get; set; } = null; [RemoteConfiguration] @@ -28,6 +32,10 @@ public class StaticFilesServerConfiguration : MareConfigurationBase sb.AppendLine($"{nameof(MainFileServerAddress)} => {MainFileServerAddress}"); sb.AppendLine($"{nameof(ForcedDeletionOfFilesAfterHours)} => {ForcedDeletionOfFilesAfterHours}"); sb.AppendLine($"{nameof(CacheSizeHardLimitInGiB)} => {CacheSizeHardLimitInGiB}"); + sb.AppendLine($"{nameof(UseColdStorage)} => {UseColdStorage}"); + sb.AppendLine($"{nameof(ColdStorageDirectory)} => {ColdStorageDirectory}"); + sb.AppendLine($"{nameof(ColdStorageSizeHardLimitInGiB)} => {ColdStorageSizeHardLimitInGiB}"); + sb.AppendLine($"{nameof(ColdStorageUnusedFileRetentionPeriodInDays)} => {ColdStorageUnusedFileRetentionPeriodInDays}"); sb.AppendLine($"{nameof(UnusedFileRetentionPeriodInDays)} => {UnusedFileRetentionPeriodInDays}"); sb.AppendLine($"{nameof(CacheDirectory)} => {CacheDirectory}"); sb.AppendLine($"{nameof(DownloadQueueSize)} => {DownloadQueueSize}"); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs index 64592a8..329b8b9 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs @@ -13,6 +13,7 @@ using MareSynchronosStaticFilesServer.Utils; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; using System.Collections.Concurrent; using System.Security.Cryptography; using System.Security.Policy; @@ -38,7 +39,9 @@ public class ServerFilesController : ControllerBase IHubContext hubContext, MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger) { - _basePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); + _basePath = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false) + ? configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)) + : configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); _cachedFileProvider = cachedFileProvider; _configuration = configuration; _hubContext = hubContext; @@ -50,14 +53,15 @@ public class ServerFilesController : ControllerBase public async Task FilesDeleteAll() { var ownFiles = await _mareDbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == MareUser).ToListAsync().ConfigureAwait(false); + bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); foreach (var dbFile in ownFiles) { var fi = FilePathUtil.GetFileInfoForHash(_basePath, dbFile.Hash); if (fi != null) { - _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1); - _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0); + _metricsClient.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1); + _metricsClient.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0); fi?.Delete(); } @@ -262,8 +266,10 @@ public class ServerFilesController : ControllerBase }).ConfigureAwait(false); await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); - _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotal, 1); - _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotalSize, compressedSize); + bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); + + _metricsClient.IncGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal, 1); + _metricsClient.IncGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, compressedSize); _fileUploadLocks.TryRemove(hash, out _); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs index f4a0009..e59b448 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs @@ -10,12 +10,13 @@ namespace MareSynchronosStaticFilesServer.Services; public sealed class CachedFileProvider : IDisposable { + private readonly IConfigurationService _configuration; private readonly ILogger _logger; private readonly FileStatisticsService _fileStatisticsService; private readonly MareMetrics _metrics; private readonly ServerTokenGenerator _generator; private readonly Uri _remoteCacheSourceUri; - private readonly string _basePath; + private readonly string _hotStoragePath; private readonly ConcurrentDictionary _currentTransfers = new(StringComparer.Ordinal); private readonly HttpClient _httpClient; private readonly SemaphoreSlim _downloadSemaphore = new(1, 1); @@ -24,15 +25,17 @@ public sealed class CachedFileProvider : IDisposable private bool IsMainServer => _remoteCacheSourceUri == null && _isDistributionServer; private bool _isDistributionServer; - public CachedFileProvider(IConfigurationService configuration, ILogger logger, FileStatisticsService fileStatisticsService, MareMetrics metrics, ServerTokenGenerator generator) + public CachedFileProvider(IConfigurationService configuration, ILogger logger, + FileStatisticsService fileStatisticsService, MareMetrics metrics, ServerTokenGenerator generator) { + _configuration = configuration; _logger = logger; _fileStatisticsService = fileStatisticsService; _metrics = metrics; _generator = generator; _remoteCacheSourceUri = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DistributionFileServerAddress), null); _isDistributionServer = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false); - _basePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); + _hotStoragePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); _httpClient = new(); _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer", "1.0.0.0")); } @@ -50,7 +53,12 @@ public sealed class CachedFileProvider : IDisposable private async Task DownloadTask(string hash) { - // download file from remote + var destinationFilePath = FilePathUtil.GetFilePath(_hotStoragePath, hash); + + // first check cold storage + if (TryCopyFromColdStorage(hash, destinationFilePath)) return; + + // if cold storage is not configured or file not found or error is present try to download file from remote var downloadUrl = MareFiles.DistributionGetFullPath(_remoteCacheSourceUri, hash); _logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl); @@ -70,8 +78,7 @@ public sealed class CachedFileProvider : IDisposable return; } - var fileName = FilePathUtil.GetFilePath(_basePath, hash); - var tempFileName = fileName + ".dl"; + var tempFileName = destinationFilePath + ".dl"; var fileStream = new FileStream(tempFileName, FileMode.Create, FileAccess.ReadWrite); var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024; var buffer = new byte[bufferSize]; @@ -84,17 +91,54 @@ public sealed class CachedFileProvider : IDisposable } await fileStream.FlushAsync().ConfigureAwait(false); await fileStream.DisposeAsync().ConfigureAwait(false); - File.Move(tempFileName, fileName, true); + File.Move(tempFileName, destinationFilePath, true); _metrics.IncGauge(MetricsAPI.GaugeFilesTotal); - _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_basePath, hash).Length); + _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash).Length); response.Dispose(); } + private bool TryCopyFromColdStorage(string hash, string destinationFilePath) + { + if (!_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false)) return false; + + string coldStorageDir = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageDirectory), string.Empty); + if (string.IsNullOrEmpty(coldStorageDir)) return false; + + var coldStorageFilePath = FilePathUtil.GetFileInfoForHash(coldStorageDir, hash); + if (coldStorageFilePath == null) return false; + + try + { + _logger.LogDebug("Copying {hash} from cold storage: {path}", hash, coldStorageFilePath); + var tempFileName = destinationFilePath + ".dl"; + File.Copy(coldStorageFilePath.FullName, tempFileName, true); + File.Move(tempFileName, destinationFilePath, true); + coldStorageFilePath.LastAccessTimeUtc = DateTime.UtcNow; + var destinationFile = new FileInfo(destinationFilePath); + destinationFile.LastAccessTimeUtc = DateTime.UtcNow; + destinationFile.CreationTimeUtc = DateTime.UtcNow; + destinationFile.LastWriteTimeUtc = DateTime.UtcNow; + _metrics.IncGauge(MetricsAPI.GaugeFilesTotal); + _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, new FileInfo(destinationFilePath).Length); + return true; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not copy {coldStoragePath} from cold storage", coldStorageFilePath); + } + + return false; + } + public async Task DownloadFileWhenRequired(string hash) { - var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash); - if (fi == null && IsMainServer) return; + var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash); + if (fi == null && IsMainServer) + { + TryCopyFromColdStorage(hash, FilePathUtil.GetFilePath(_hotStoragePath, hash)); + return; + } await _downloadSemaphore.WaitAsync().ConfigureAwait(false); if ((fi == null || (fi?.Length ?? 0) == 0) @@ -124,9 +168,11 @@ public sealed class CachedFileProvider : IDisposable public FileStream? GetLocalFileStream(string hash) { - var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash); + var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash); if (fi == null) return null; + fi.LastAccessTimeUtc = DateTime.UtcNow; + _fileStatisticsService.LogFile(hash, fi.Length); return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs index bcd9de2..def6936 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs @@ -10,7 +10,6 @@ namespace MareSynchronosStaticFilesServer.Services; public class MainFileCleanupService : IHostedService { - private readonly string _cacheDir; private readonly IConfigurationService _configuration; private readonly ILogger _logger; private readonly MareMetrics _metrics; @@ -24,7 +23,6 @@ public class MainFileCleanupService : IHostedService _logger = logger; _services = services; _configuration = configuration; - _cacheDir = _configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); } public async Task CleanUpTask(CancellationToken ct) @@ -35,17 +33,48 @@ public class MainFileCleanupService : IHostedService { try { - DirectoryInfo dir = new(_cacheDir); - var allFiles = dir.GetFiles("*", SearchOption.AllDirectories); - _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFiles.Sum(f => f.Length)); - _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFiles.Length); + var hotStorageDir = _configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); + DirectoryInfo dir = new(hotStorageDir); + var allFilesInHotStorage = dir.GetFiles("*", SearchOption.AllDirectories).ToList(); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFilesInHotStorage.Sum(f => f.Length)); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFilesInHotStorage.Count); using var scope = _services.CreateScope(); using var dbContext = scope.ServiceProvider.GetService()!; - await CleanUpOutdatedFiles(dbContext, ct).ConfigureAwait(false); + bool useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); - CleanUpFilesBeyondSizeLimit(dbContext, ct); + if (useColdStorage) + { + var coldStorageDir = _configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)); + var coldStorageRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60); + var coldStorageSize = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1); + + DirectoryInfo dirColdStorage = new(coldStorageDir); + var allFilesInColdStorageDir = dirColdStorage.GetFiles("*", SearchOption.AllDirectories).ToList(); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, allFilesInColdStorageDir.Sum(f => f.Length)); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, allFilesInColdStorageDir.Count); + + // clean up cold storage + var remainingColdFiles = await CleanUpOutdatedFiles(coldStorageDir, allFilesInColdStorageDir, coldStorageRetention, forcedDeletionAfterHours: -1, + isColdStorage: true, deleteFromDb: true, + dbContext, ct).ConfigureAwait(false); + CleanUpFilesBeyondSizeLimit(remainingColdFiles, coldStorageSize, + isColdStorage: true, deleteFromDb: true, + dbContext, ct); + } + + var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14); + var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1); + var sizeLimit = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1); + + var remainingHotFiles = await CleanUpOutdatedFiles(hotStorageDir, allFilesInHotStorage, unusedRetention, forcedDeletionAfterHours, + isColdStorage: false, deleteFromDb: !useColdStorage, + dbContext, ct).ConfigureAwait(false); + + CleanUpFilesBeyondSizeLimit(remainingHotFiles, sizeLimit, + isColdStorage: false, deleteFromDb: !useColdStorage, + dbContext, ct); CleanUpStuckUploads(dbContext); @@ -84,9 +113,8 @@ public class MainFileCleanupService : IHostedService return Task.CompletedTask; } - private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct) + private void CleanUpFilesBeyondSizeLimit(List files, double sizeLimit, bool isColdStorage, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct) { - var sizeLimit = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1); if (sizeLimit <= 0) { return; @@ -95,8 +123,7 @@ public class MainFileCleanupService : IHostedService try { _logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", sizeLimit); - var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories) - .Select(f => new FileInfo(f)).ToList() + var allLocalFiles = files .OrderBy(f => f.LastAccessTimeUtc).ToList(); var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length); long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes; @@ -105,12 +132,13 @@ public class MainFileCleanupService : IHostedService var oldestFile = allLocalFiles[0]; allLocalFiles.Remove(oldestFile); totalCacheSizeInBytes -= oldestFile.Length; - _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, oldestFile.Length); - _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); + _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, oldestFile.Length); + _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal); _logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes); oldestFile.Delete(); FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() }; - dbContext.Entry(f).State = EntityState.Deleted; + if (deleteFromDb) + dbContext.Entry(f).State = EntityState.Deleted; } } catch (Exception ex) @@ -119,10 +147,10 @@ public class MainFileCleanupService : IHostedService } } - private void CleanUpOrphanedFiles(List allFiles, FileInfo[] allPhysicalFiles, CancellationToken ct) + private List CleanUpOrphanedFiles(List allFiles, List allPhysicalFiles, CancellationToken ct) { var allFilesHashes = new HashSet(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal); - foreach (var file in allPhysicalFiles) + foreach (var file in allPhysicalFiles.ToList()) { if (!allFilesHashes.Contains(file.Name.ToUpperInvariant())) { @@ -134,15 +162,15 @@ public class MainFileCleanupService : IHostedService ct.ThrowIfCancellationRequested(); } + + return allPhysicalFiles; } - private async Task CleanUpOutdatedFiles(MareDbContext dbContext, CancellationToken ct) + private async Task> CleanUpOutdatedFiles(string cacheDir, List allFilesInDir, int unusedRetention, int forcedDeletionAfterHours, + bool isColdStorage, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct) { try { - var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14); - var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1); - _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", unusedRetention); if (forcedDeletionAfterHours > 0) { @@ -152,39 +180,45 @@ public class MainFileCleanupService : IHostedService // clean up files in DB but not on disk or last access is expired var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention)); var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours)); - DirectoryInfo dir = new(_cacheDir); - var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories); + List deletedFiles = new(); var files = dbContext.Files.OrderBy(f => f.Hash); - List allFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false); + List allDbFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false); int fileCounter = 0; - foreach (var fileCache in allFiles.Where(f => f.Uploaded)) + + foreach (var fileCache in allDbFiles.Where(f => f.Uploaded)) { bool fileDeleted = false; - var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash); + var file = FilePathUtil.GetFileInfoForHash(cacheDir, fileCache.Hash); if (file == null) { _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash); - dbContext.Files.Remove(fileCache); fileDeleted = true; + + if (deleteFromDb) + dbContext.Files.Remove(fileCache); } else if (file != null && file.LastAccessTime < prevTime) { - _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); - _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); + _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, file.Length); + _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal); _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); file.Delete(); + deletedFiles.Add(file.FullName); fileDeleted = true; - dbContext.Files.Remove(fileCache); + if (deleteFromDb) + dbContext.Files.Remove(fileCache); } else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion) { - _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); - _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); + _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, file.Length); + _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal); _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); file.Delete(); fileDeleted = true; - dbContext.Files.Remove(fileCache); + deletedFiles.Add(file.FullName); + if (deleteFromDb) + dbContext.Files.Remove(fileCache); } if (!fileDeleted && file != null && fileCache.Size == 0) @@ -201,17 +235,19 @@ public class MainFileCleanupService : IHostedService } // clean up files that are on disk but not in DB for some reason - CleanUpOrphanedFiles(allFiles, allFilesInDir, ct); + return CleanUpOrphanedFiles(allDbFiles, allFilesInDir.Where(f => !deletedFiles.Contains(f.FullName)).ToList(), ct); } catch (Exception ex) { _logger.LogWarning(ex, "Error during file cleanup of old files"); } + + return []; } private void CleanUpStuckUploads(MareDbContext dbContext) { - var pastTime = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10)); + var pastTime = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(20)); var stuckUploads = dbContext.Files.Where(f => !f.Uploaded && f.UploadDate < pastTime); dbContext.Files.RemoveRange(stuckUploads); } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs index 60602c0..c05c55c 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs @@ -60,6 +60,8 @@ public class Startup MetricsAPI.CounterFileRequestSize }, new List { + MetricsAPI.GaugeFilesTotalColdStorage, + MetricsAPI.GaugeFilesTotalSizeColdStorage, MetricsAPI.GaugeFilesTotalSize, MetricsAPI.GaugeFilesTotal, MetricsAPI.GaugeFilesUniquePastDay,