From 584f5356d7efd844b11ca40314cabf75567eedad Mon Sep 17 00:00:00 2001 From: Loporrit <141286461+loporrit@users.noreply.github.com> Date: Sun, 1 Sep 2024 15:44:57 +0000 Subject: [PATCH] Allow shards to use cold storage, revise cleanup service --- .../Controllers/ServerFilesController.cs | 35 +- .../Services/CachedFileProvider.cs | 39 ++- .../Services/FileCleanupService.cs | 302 ++++++++---------- .../Startup.cs | 3 +- 4 files changed, 181 insertions(+), 198 deletions(-) diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs index 329b8b9..16d2bbe 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs @@ -28,6 +28,7 @@ public class ServerFilesController : ControllerBase private static readonly SemaphoreSlim _fileLockDictLock = new(1); private static readonly ConcurrentDictionary _fileUploadLocks = new(StringComparer.Ordinal); private readonly string _basePath; + private readonly string _coldBasePath; private readonly CachedFileProvider _cachedFileProvider; private readonly IConfigurationService _configuration; private readonly IHubContext _hubContext; @@ -39,11 +40,11 @@ public class ServerFilesController : ControllerBase IHubContext hubContext, MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger) { - _basePath = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false) - ? configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)) - : configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); - _cachedFileProvider = cachedFileProvider; _configuration = configuration; + _basePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); + if (_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false)) + _basePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)); + _cachedFileProvider = cachedFileProvider; _hubContext = hubContext; _mareDbContext = mareDbContext; _metricsClient = metricsClient; @@ -53,20 +54,34 @@ public class ServerFilesController : ControllerBase public async Task FilesDeleteAll() { var ownFiles = await _mareDbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == MareUser).ToListAsync().ConfigureAwait(false); - bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); foreach (var dbFile in ownFiles) { var fi = FilePathUtil.GetFileInfoForHash(_basePath, dbFile.Hash); if (fi != null) { - _metricsClient.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1); - _metricsClient.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0); + _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1); + _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0); fi?.Delete(); } } + if (!_coldBasePath.IsNullOrEmpty()) + { + foreach (var dbFile in ownFiles) + { + var fi = FilePathUtil.GetFileInfoForHash(_coldBasePath, dbFile.Hash); + if (fi != null) + { + _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalColdStorage, fi == null ? 0 : 1); + _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSizeColdStorage, fi?.Length ?? 0); + + fi?.Delete(); + } + } + } + _mareDbContext.Files.RemoveRange(ownFiles); await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); @@ -266,10 +281,8 @@ public class ServerFilesController : ControllerBase }).ConfigureAwait(false); await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); - bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); - - _metricsClient.IncGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal, 1); - _metricsClient.IncGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, compressedSize); + _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotal, 1); + _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotalSize, compressedSize); _fileUploadLocks.TryRemove(hash, out _); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs index ffb40e9..020971f 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs @@ -16,7 +16,9 @@ public sealed class CachedFileProvider : IDisposable private readonly MareMetrics _metrics; private readonly ServerTokenGenerator _generator; private readonly Uri _remoteCacheSourceUri; + private readonly bool _useColdStorage; private readonly string _hotStoragePath; + private readonly string _coldStoragePath; private readonly ConcurrentDictionary _currentTransfers = new(StringComparer.Ordinal); private readonly HttpClient _httpClient; private readonly SemaphoreSlim _downloadSemaphore = new(1, 1); @@ -35,7 +37,9 @@ public sealed class CachedFileProvider : IDisposable _generator = generator; _remoteCacheSourceUri = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DistributionFileServerAddress), null); _isDistributionServer = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false); + _useColdStorage = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); _hotStoragePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); + _coldStoragePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)); _httpClient = new(); _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer", "1.0.0.0")); } @@ -53,7 +57,7 @@ public sealed class CachedFileProvider : IDisposable private async Task DownloadTask(string hash) { - var destinationFilePath = FilePathUtil.GetFilePath(_hotStoragePath, hash); + var destinationFilePath = FilePathUtil.GetFilePath(_useColdStorage ? _coldStoragePath : _hotStoragePath, hash); // if cold storage is not configured or file not found or error is present try to download file from remote var downloadUrl = MareFiles.DistributionGetFullPath(_remoteCacheSourceUri, hash); @@ -77,7 +81,7 @@ public sealed class CachedFileProvider : IDisposable var tempFileName = destinationFilePath + ".dl"; var fileStream = new FileStream(tempFileName, FileMode.Create, FileAccess.ReadWrite); - var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024; + var bufferSize = 4096; var buffer = new byte[bufferSize]; var bytesRead = 0; @@ -90,19 +94,18 @@ public sealed class CachedFileProvider : IDisposable await fileStream.DisposeAsync().ConfigureAwait(false); File.Move(tempFileName, destinationFilePath, true); - _metrics.IncGauge(MetricsAPI.GaugeFilesTotal); - _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash).Length); + _metrics.IncGauge(_useColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal); + _metrics.IncGauge(_useColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, new FileInfo(destinationFilePath).Length); response.Dispose(); } private bool TryCopyFromColdStorage(string hash, string destinationFilePath) { - if (!_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false)) return false; + if (!_useColdStorage) return false; - string coldStorageDir = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageDirectory), string.Empty); - if (string.IsNullOrEmpty(coldStorageDir)) return false; + if (string.IsNullOrEmpty(_coldStoragePath)) return false; - var coldStorageFilePath = FilePathUtil.GetFileInfoForHash(coldStorageDir, hash); + var coldStorageFilePath = FilePathUtil.GetFileInfoForHash(_coldStoragePath, hash); if (coldStorageFilePath == null) return false; try @@ -131,16 +134,20 @@ public sealed class CachedFileProvider : IDisposable public async Task DownloadFileWhenRequired(string hash) { var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash); - if (fi == null) - { - if (TryCopyFromColdStorage(hash, FilePathUtil.GetFilePath(_hotStoragePath, hash))) - return; - } + + if (fi != null && fi.Length != 0) + return; + + // first check cold storage + if (TryCopyFromColdStorage(hash, FilePathUtil.GetFilePath(_hotStoragePath, hash))) + return; + + // no distribution server configured to download from + if (_remoteCacheSourceUri == null) + return; await _downloadSemaphore.WaitAsync().ConfigureAwait(false); - if ((fi == null || (fi?.Length ?? 0) == 0) - && (!_currentTransfers.TryGetValue(hash, out var downloadTask) - || (downloadTask?.IsCompleted ?? true))) + if (!_currentTransfers.TryGetValue(hash, out var downloadTask) || (downloadTask?.IsCompleted ?? true)) { _currentTransfers[hash] = Task.Run(async () => { diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs index 611556a..f65e6c5 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs @@ -14,8 +14,35 @@ public class FileCleanupService : IHostedService private readonly ILogger _logger; private readonly MareMetrics _metrics; private readonly IServiceProvider _services; + + private readonly string _hotStoragePath; + private readonly string _coldStoragePath; + private readonly bool _isMain = false; + private readonly bool _isDistributionNode = false; + private readonly bool _useColdStorage = false; + private CancellationTokenSource _cleanupCts; + private int HotStorageRetention => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14); + private double HotStorageSize => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1.0); + + private int ColdStorageRetention => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60); + private double ColdStorageSize => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1.0); + + private int ForcedDeletionAfterHours => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1); + private int CleanupCheckMinutes => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CleanupCheckInMinutes), 15); + + private List GetAllHotFiles() => new DirectoryInfo(_hotStoragePath).GetFiles("*", SearchOption.AllDirectories) + .Where(f => f != null && f.Name.Length == 40) + .OrderBy(f => f.LastAccessTimeUtc).ToList(); + + private List GetAllColdFiles() => new DirectoryInfo(_coldStoragePath).GetFiles("*", SearchOption.AllDirectories) + .Where(f => f != null && f.Name.Length == 40) + .OrderBy(f => f.LastAccessTimeUtc).ToList(); + + private List GetTempFiles() => new DirectoryInfo(_useColdStorage ? _coldStoragePath : _hotStoragePath).GetFiles("*", SearchOption.AllDirectories) + .Where(f => f != null && (f.Name.EndsWith(".dl", StringComparison.InvariantCultureIgnoreCase) || f.Name.EndsWith(".tmp", StringComparison.InvariantCultureIgnoreCase))).ToList(); + public FileCleanupService(MareMetrics metrics, ILogger logger, IServiceProvider services, IConfigurationService configuration) { @@ -23,6 +50,11 @@ public class FileCleanupService : IHostedService _logger = logger; _services = services; _configuration = configuration; + _useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); + _hotStoragePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); + _coldStoragePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)); + _isDistributionNode = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false); + _isMain = configuration.GetValue(nameof(StaticFilesServerConfiguration.MainFileServerAddress)) == null && _isDistributionNode; } public Task StartAsync(CancellationToken cancellationToken) @@ -45,18 +77,18 @@ public class FileCleanupService : IHostedService return Task.CompletedTask; } - private List CleanUpFilesBeyondSizeLimit(List files, double sizeLimit, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct) + private List CleanUpFilesBeyondSizeLimit(List files, double sizeLimit, CancellationToken ct) { + var removedFiles = new List(); if (sizeLimit <= 0) { - return []; + return removedFiles; } try { _logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", sizeLimit); - var allLocalFiles = files - .OrderBy(f => f.LastAccessTimeUtc).ToList(); + var allLocalFiles = files; var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length); long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes; while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Count != 0 && !ct.IsCancellationRequested) @@ -66,30 +98,24 @@ public class FileCleanupService : IHostedService totalCacheSizeInBytes -= oldestFile.Length; _logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes); oldestFile.Delete(); - FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() }; - if (deleteFromDb) - dbContext.Entry(f).State = EntityState.Deleted; + removedFiles.Add(oldestFile.Name); } - - return allLocalFiles; + files.RemoveAll(f => removedFiles.Contains(f.Name, StringComparer.InvariantCultureIgnoreCase)); } catch (Exception ex) { _logger.LogWarning(ex, "Error during cache size limit cleanup"); } - return []; + return removedFiles; } - private List CleanUpOrphanedFiles(List allFiles, List allPhysicalFiles, CancellationToken ct) + private void CleanUpOrphanedFiles(HashSet allDbFileHashes, List allPhysicalFiles, CancellationToken ct) { - var allFilesHashes = new HashSet(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal); foreach (var file in allPhysicalFiles.ToList()) { - if (!allFilesHashes.Contains(file.Name.ToUpperInvariant())) + if (!allDbFileHashes.Contains(file.Name.ToUpperInvariant())) { - _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); - _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); file.Delete(); _logger.LogInformation("File not in DB, deleting: {fileName}", file.Name); allPhysicalFiles.Remove(file); @@ -97,13 +123,11 @@ public class FileCleanupService : IHostedService ct.ThrowIfCancellationRequested(); } - - return allPhysicalFiles; } - private async Task> CleanUpOutdatedFiles(string dir, List allFilesInDir, int unusedRetention, int forcedDeletionAfterHours, - bool deleteFromDb, MareDbContext dbContext, CancellationToken ct) + private List CleanUpOutdatedFiles(List files, int unusedRetention, int forcedDeletionAfterHours, CancellationToken ct) { + var removedFiles = new List(); try { _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", unusedRetention); @@ -112,37 +136,42 @@ public class FileCleanupService : IHostedService _logger.LogInformation("Cleaning up files written to longer than {hours}h ago", forcedDeletionAfterHours); } - // clean up files in DB but not on disk or last access is expired - var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention)); - var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours)); - List allDbFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false); - List removedFileHashes; + var lastAccessCutoffTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention)); + var forcedDeletionCutoffTime = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours)); - if (!deleteFromDb) + foreach (var file in files) { - removedFileHashes = CleanupViaFiles(allFilesInDir, forcedDeletionAfterHours, prevTime, prevTimeForcedDeletion, ct); - } - else - { - removedFileHashes = await CleanupViaDb(dir, forcedDeletionAfterHours, dbContext, prevTime, prevTimeForcedDeletion, allDbFiles, ct).ConfigureAwait(false); - } + if (file.LastAccessTime < lastAccessCutoffTime) + { + _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); + file.Delete(); + removedFiles.Add(file.Name); + } + else if (forcedDeletionAfterHours > 0 && file.LastWriteTime < forcedDeletionCutoffTime) + { + _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); + file.Delete(); + removedFiles.Add(file.Name); + } - // clean up files that are on disk but not in DB anymore - return CleanUpOrphanedFiles(allDbFiles, allFilesInDir.Where(c => !removedFileHashes.Contains(c.Name, StringComparer.OrdinalIgnoreCase)).ToList(), ct); + ct.ThrowIfCancellationRequested(); + } + files.RemoveAll(f => removedFiles.Contains(f.Name, StringComparer.InvariantCultureIgnoreCase)); } catch (Exception ex) { _logger.LogWarning(ex, "Error during file cleanup of old files"); } - return []; + return removedFiles; } - private void CleanUpStuckUploads(MareDbContext dbContext) + private void CleanUpTempFiles() { var pastTime = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(20)); - var stuckUploads = dbContext.Files.Where(f => !f.Uploaded && f.UploadDate < pastTime); - dbContext.Files.RemoveRange(stuckUploads); + var tempFiles = GetTempFiles(); + foreach (var tempFile in tempFiles.Where(f => f.LastWriteTimeUtc < pastTime)) + tempFile.Delete(); } private async Task CleanUpTask(CancellationToken ct) @@ -152,61 +181,87 @@ public class FileCleanupService : IHostedService try { using var scope = _services.CreateScope(); - using var dbContext = scope.ServiceProvider.GetService()!; + using var dbContext = _isMain ? scope.ServiceProvider.GetService()! : null; - bool useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); + HashSet allDbFileHashes = null; - if (useColdStorage) + // Database operations only performed on main server + if (_isMain) { - var coldStorageDir = _configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)); - - DirectoryInfo dirColdStorage = new(coldStorageDir); - var allFilesInColdStorageDir = dirColdStorage.GetFiles("*", SearchOption.AllDirectories).ToList(); - - var coldStorageRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60); - var coldStorageSize = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1); - - // clean up cold storage - var remainingColdFiles = await CleanUpOutdatedFiles(coldStorageDir, allFilesInColdStorageDir, coldStorageRetention, forcedDeletionAfterHours: -1, - deleteFromDb: true, dbContext: dbContext, - ct: ct).ConfigureAwait(false); - var finalRemainingColdFiles = CleanUpFilesBeyondSizeLimit(remainingColdFiles, coldStorageSize, - deleteFromDb: true, dbContext: dbContext, - ct: ct); - - _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, finalRemainingColdFiles.Sum(f => f.Length)); - _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, finalRemainingColdFiles.Count); + var allDbFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false); + allDbFileHashes = new HashSet(allDbFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal); } - var hotStorageDir = _configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); - DirectoryInfo dirHotStorage = new(hotStorageDir); - var allFilesInHotStorage = dirHotStorage.GetFiles("*", SearchOption.AllDirectories).ToList(); + if (_useColdStorage) + { + var coldFiles = GetAllColdFiles(); + var removedColdFiles = new List(); - var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14); - var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1); - var sizeLimit = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1); + removedColdFiles.AddRange( + CleanUpOutdatedFiles(coldFiles, ColdStorageRetention, ForcedDeletionAfterHours, ct) + ); + removedColdFiles.AddRange( + CleanUpFilesBeyondSizeLimit(coldFiles, ColdStorageSize, ct) + ); - var remainingHotFiles = await CleanUpOutdatedFiles(hotStorageDir, allFilesInHotStorage, unusedRetention, forcedDeletionAfterHours, - deleteFromDb: !useColdStorage, dbContext: dbContext, - ct: ct).ConfigureAwait(false); + // Remove cold storage files are deleted from the database, if we are the main file server + if (_isMain) + { + dbContext.Files.RemoveRange( + dbContext.Files.Where(f => removedColdFiles.Contains(f.Hash)) + ); + allDbFileHashes.ExceptWith(removedColdFiles); + CleanUpOrphanedFiles(allDbFileHashes, coldFiles, ct); + } - var finalRemainingHotFiles = CleanUpFilesBeyondSizeLimit(remainingHotFiles, sizeLimit, - deleteFromDb: !useColdStorage, dbContext: dbContext, - ct: ct); + // Remove hot copies of files now that the authoritative copy is gone + foreach (var removedFile in removedColdFiles) + { + var hotFile = FilePathUtil.GetFileInfoForHash(_hotStoragePath, removedFile); + hotFile?.Delete(); + } - CleanUpStuckUploads(dbContext); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, coldFiles.Sum(f => { try { return f.Length; } catch { return 0; } })); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, coldFiles.Count); + } - await dbContext.SaveChangesAsync(ct).ConfigureAwait(false); + var hotFiles = GetAllHotFiles(); + var removedHotFiles = new List(); - _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, finalRemainingHotFiles.Sum(f => { try { return f.Length; } catch { return 0; } })); - _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, finalRemainingHotFiles.Count); + removedHotFiles.AddRange( + CleanUpOutdatedFiles(hotFiles, HotStorageRetention, forcedDeletionAfterHours: _useColdStorage ? ForcedDeletionAfterHours : -1, ct) + ); + removedHotFiles.AddRange( + CleanUpFilesBeyondSizeLimit(hotFiles, HotStorageSize, ct) + ); + + if (_isMain) + { + // If cold storage is not active, then "hot" files are deleted from the database instead + if (!_useColdStorage) + { + dbContext.Files.RemoveRange( + dbContext.Files.Where(f => removedHotFiles.Contains(f.Hash)) + ); + allDbFileHashes.ExceptWith(removedHotFiles); + } + + CleanUpOrphanedFiles(allDbFileHashes, hotFiles, ct); + + await dbContext.SaveChangesAsync(ct).ConfigureAwait(false); + } + + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, hotFiles.Sum(f => { try { return f.Length; } catch { return 0; } })); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, hotFiles.Count); + + CleanUpTempFiles(); } catch (Exception e) { _logger.LogError(e, "Error during cleanup task"); } - var cleanupCheckMinutes = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CleanupCheckInMinutes), 15); + var cleanupCheckMinutes = CleanupCheckMinutes; var now = DateTime.Now; TimeOnly currentTime = new(now.Hour, now.Minute, now.Second); TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % cleanupCheckMinutes, 0); @@ -216,109 +271,18 @@ public class FileCleanupService : IHostedService await Task.Delay(span, ct).ConfigureAwait(false); } } - private async Task> CleanupViaDb(string dir, int forcedDeletionAfterHours, - MareDbContext dbContext, DateTime lastAccessCutoffTime, DateTime forcedDeletionCutoffTime, List allDbFiles, CancellationToken ct) - { - int fileCounter = 0; - List removedFileHashes = new(); - foreach (var fileCache in allDbFiles.Where(f => f.Uploaded)) - { - bool deleteCurrentFile = false; - var file = FilePathUtil.GetFileInfoForHash(dir, fileCache.Hash); - if (file == null) - { - _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash); - deleteCurrentFile = true; - } - else if (file != null && file.LastAccessTime < lastAccessCutoffTime) - { - _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); - deleteCurrentFile = true; - } - else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < forcedDeletionCutoffTime) - { - _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); - deleteCurrentFile = true; - } - - // do actual deletion of file and remove also from db if needed - if (deleteCurrentFile) - { - if (file != null) file.Delete(); - - removedFileHashes.Add(fileCache.Hash); - - dbContext.Files.Remove(fileCache); - } - - // only used if file in db has no size for whatever reason - if (!deleteCurrentFile && file != null && fileCache.Size == 0) - { - _logger.LogInformation("Setting File Size of " + fileCache.Hash + " to " + file.Length); - fileCache.Size = file.Length; - // commit every 1000 files to db - if (fileCounter % 1000 == 0) - await dbContext.SaveChangesAsync().ConfigureAwait(false); - } - - fileCounter++; - - ct.ThrowIfCancellationRequested(); - } - - return removedFileHashes; - } - - private List CleanupViaFiles(List allFilesInDir, int forcedDeletionAfterHours, - DateTime lastAccessCutoffTime, DateTime forcedDeletionCutoffTime, CancellationToken ct) - { - List removedFileHashes = new List(); - - foreach (var file in allFilesInDir) - { - bool deleteCurrentFile = false; - if (file != null && file.LastAccessTime < lastAccessCutoffTime) - { - _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); - deleteCurrentFile = true; - } - else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < forcedDeletionCutoffTime) - { - _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); - deleteCurrentFile = true; - } - - if (deleteCurrentFile) - { - if (file != null) file.Delete(); - - removedFileHashes.Add(file.Name); - } - - ct.ThrowIfCancellationRequested(); - } - - return removedFileHashes; - } private void InitializeGauges() { - bool useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); - - if (useColdStorage) + if (_useColdStorage) { - var coldStorageDir = _configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)); - - DirectoryInfo dirColdStorage = new(coldStorageDir); - var allFilesInColdStorageDir = dirColdStorage.GetFiles("*", SearchOption.AllDirectories).ToList(); + var allFilesInColdStorageDir = GetAllColdFiles(); _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, allFilesInColdStorageDir.Sum(f => f.Length)); _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, allFilesInColdStorageDir.Count); } - var hotStorageDir = _configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); - DirectoryInfo dirHotStorage = new(hotStorageDir); - var allFilesInHotStorage = dirHotStorage.GetFiles("*", SearchOption.AllDirectories).ToList(); + var allFilesInHotStorage = GetAllHotFiles(); _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFilesInHotStorage.Sum(f => { try { return f.Length; } catch { return 0; } })); _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFilesInHotStorage.Count); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs index fd8676e..b55ac8e 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs @@ -82,6 +82,7 @@ public class Startup // generic services services.AddSingleton(); + services.AddHostedService(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -95,7 +96,6 @@ public class Startup if (_isMain) { services.AddSingleton(); - services.AddHostedService(); services.AddSingleton, MareConfigurationServiceServer>(); services.AddDbContextPool(options => { @@ -171,7 +171,6 @@ public class Startup else { services.AddSingleton(); - services.AddHostedService(); services.AddSingleton, MareConfigurationServiceClient>(); services.AddHostedService(p => (MareConfigurationServiceClient)p.GetService>()); }