diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs index def6936..e293c11 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs @@ -33,37 +33,37 @@ public class MainFileCleanupService : IHostedService { try { - var hotStorageDir = _configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); - DirectoryInfo dir = new(hotStorageDir); - var allFilesInHotStorage = dir.GetFiles("*", SearchOption.AllDirectories).ToList(); - _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFilesInHotStorage.Sum(f => f.Length)); - _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFilesInHotStorage.Count); - using var scope = _services.CreateScope(); using var dbContext = scope.ServiceProvider.GetService()!; - bool useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); + bool useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); if (useColdStorage) { var coldStorageDir = _configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)); - var coldStorageRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60); - var coldStorageSize = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1); DirectoryInfo dirColdStorage = new(coldStorageDir); var allFilesInColdStorageDir = dirColdStorage.GetFiles("*", SearchOption.AllDirectories).ToList(); - _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, allFilesInColdStorageDir.Sum(f => f.Length)); - _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, allFilesInColdStorageDir.Count); + + var coldStorageRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60); + var coldStorageSize = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1); // clean up cold storage var remainingColdFiles = await CleanUpOutdatedFiles(coldStorageDir, allFilesInColdStorageDir, coldStorageRetention, forcedDeletionAfterHours: -1, isColdStorage: true, deleteFromDb: true, dbContext, ct).ConfigureAwait(false); - CleanUpFilesBeyondSizeLimit(remainingColdFiles, coldStorageSize, + var finalRemainingColdFiles = CleanUpFilesBeyondSizeLimit(remainingColdFiles, coldStorageSize, isColdStorage: true, deleteFromDb: true, dbContext, ct); + + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, finalRemainingColdFiles.Sum(f => f.Length)); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, finalRemainingColdFiles.Count); } + var hotStorageDir = _configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); + DirectoryInfo dirHotStorage = new(hotStorageDir); + var allFilesInHotStorage = dirHotStorage.GetFiles("*", SearchOption.AllDirectories).ToList(); + var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14); var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1); var sizeLimit = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1); @@ -72,18 +72,22 @@ public class MainFileCleanupService : IHostedService isColdStorage: false, deleteFromDb: !useColdStorage, dbContext, ct).ConfigureAwait(false); - CleanUpFilesBeyondSizeLimit(remainingHotFiles, sizeLimit, + var finalRemainingHotFiles = CleanUpFilesBeyondSizeLimit(remainingHotFiles, sizeLimit, isColdStorage: false, deleteFromDb: !useColdStorage, dbContext, ct); CleanUpStuckUploads(dbContext); await dbContext.SaveChangesAsync(ct).ConfigureAwait(false); + + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, finalRemainingHotFiles.Sum(f => { try { return f.Length; } catch { return 0; } })); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, finalRemainingHotFiles.Count); } catch (Exception e) { _logger.LogError(e, "Error during cleanup task"); } + var cleanupCheckMinutes = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CleanupCheckInMinutes), 15); var now = DateTime.Now; TimeOnly currentTime = new(now.Hour, now.Minute, now.Second); @@ -113,11 +117,11 @@ public class MainFileCleanupService : IHostedService return Task.CompletedTask; } - private void CleanUpFilesBeyondSizeLimit(List files, double sizeLimit, bool isColdStorage, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct) + private List CleanUpFilesBeyondSizeLimit(List files, double sizeLimit, bool isColdStorage, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct) { if (sizeLimit <= 0) { - return; + return []; } try @@ -127,24 +131,26 @@ public class MainFileCleanupService : IHostedService .OrderBy(f => f.LastAccessTimeUtc).ToList(); var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length); long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes; - while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested) + while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Count != 0 && !ct.IsCancellationRequested) { var oldestFile = allLocalFiles[0]; - allLocalFiles.Remove(oldestFile); + allLocalFiles.RemoveAt(0); totalCacheSizeInBytes -= oldestFile.Length; - _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, oldestFile.Length); - _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal); _logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes); oldestFile.Delete(); FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() }; if (deleteFromDb) dbContext.Entry(f).State = EntityState.Deleted; } + + return allLocalFiles; } catch (Exception ex) { _logger.LogWarning(ex, "Error during cache size limit cleanup"); } + + return []; } private List CleanUpOrphanedFiles(List allFiles, List allPhysicalFiles, CancellationToken ct) @@ -158,6 +164,7 @@ public class MainFileCleanupService : IHostedService _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); file.Delete(); _logger.LogInformation("File not in DB, deleting: {fileName}", file.Name); + allPhysicalFiles.Remove(file); } ct.ThrowIfCancellationRequested(); @@ -166,7 +173,7 @@ public class MainFileCleanupService : IHostedService return allPhysicalFiles; } - private async Task> CleanUpOutdatedFiles(string cacheDir, List allFilesInDir, int unusedRetention, int forcedDeletionAfterHours, + private async Task> CleanUpOutdatedFiles(string dir, List allFilesInDir, int unusedRetention, int forcedDeletionAfterHours, bool isColdStorage, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct) { try @@ -180,53 +187,49 @@ public class MainFileCleanupService : IHostedService // clean up files in DB but not on disk or last access is expired var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention)); var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours)); - List deletedFiles = new(); - var files = dbContext.Files.OrderBy(f => f.Hash); List allDbFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false); + List removedFileHashes = new List(); int fileCounter = 0; foreach (var fileCache in allDbFiles.Where(f => f.Uploaded)) { - bool fileDeleted = false; - - var file = FilePathUtil.GetFileInfoForHash(cacheDir, fileCache.Hash); + bool deleteCurrentFile = false; + var file = FilePathUtil.GetFileInfoForHash(dir, fileCache.Hash); if (file == null) { _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash); - fileDeleted = true; - - if (deleteFromDb) - dbContext.Files.Remove(fileCache); + deleteCurrentFile = true; } else if (file != null && file.LastAccessTime < prevTime) { - _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, file.Length); - _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal); _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); - file.Delete(); - deletedFiles.Add(file.FullName); - fileDeleted = true; - if (deleteFromDb) - dbContext.Files.Remove(fileCache); + deleteCurrentFile = true; } else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion) { - _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, file.Length); - _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal); _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); - file.Delete(); - fileDeleted = true; - deletedFiles.Add(file.FullName); + deleteCurrentFile = true; + } + + // do actual deletion of file and remove also from db if needed + if (deleteCurrentFile) + { + if (file != null) file.Delete(); + + removedFileHashes.Add(fileCache.Hash); + if (deleteFromDb) dbContext.Files.Remove(fileCache); } - if (!fileDeleted && file != null && fileCache.Size == 0) + // only used if file in db has no size for whatever reason + if (!deleteCurrentFile && file != null && fileCache.Size == 0) { _logger.LogInformation("Setting File Size of " + fileCache.Hash + " to " + file.Length); fileCache.Size = file.Length; // commit every 1000 files to db - if (fileCounter % 1000 == 0) await dbContext.SaveChangesAsync().ConfigureAwait(false); + if (fileCounter % 1000 == 0) + await dbContext.SaveChangesAsync().ConfigureAwait(false); } fileCounter++; @@ -235,7 +238,7 @@ public class MainFileCleanupService : IHostedService } // clean up files that are on disk but not in DB for some reason - return CleanUpOrphanedFiles(allDbFiles, allFilesInDir.Where(f => !deletedFiles.Contains(f.FullName)).ToList(), ct); + return CleanUpOrphanedFiles(allDbFiles, allFilesInDir.Where(c => !removedFileHashes.Contains(c.Name, StringComparer.OrdinalIgnoreCase)).ToList(), ct); } catch (Exception ex) {