add central cold storage options for file servers

add access times I guess

access, not write

overwrite times after copying

MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs

@@ -12,7 +12,9 @@ public class MetricsAPI
     public const string GaugePairs = "mare_pairs";
     public const string GaugePairsPaused = "mare_pairs_paused";
     public const string GaugeFilesTotal = "mare_files";
+    public const string GaugeFilesTotalColdStorage = "mare_files_cold";
     public const string GaugeFilesTotalSize = "mare_files_size";
+    public const string GaugeFilesTotalSizeColdStorage = "mare_files_size_cold";
     public const string GaugeFilesDownloadingFromCache = "mare_files_downloading_from_cache";
     public const string GaugeFilesTasksWaitingForDownloadFromCache = "mare_files_waiting_for_dl";
     public const string CounterUserPushData = "mare_user_push";

MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs

@@ -17,6 +17,10 @@ public class StaticFilesServerConfiguration : MareConfigurationBase
     public int DownloadQueueReleaseSeconds { get; set; } = 15;
     public int DownloadQueueClearLimit { get; set; } = 15000;
     public int CleanupCheckInMinutes { get; set; } = 15;
+    public bool UseColdStorage { get; set; } = false;
+    public string? ColdStorageDirectory { get; set; } = null;
+    public double ColdStorageSizeHardLimitInGiB { get; set; } = -1;
+    public double ColdStorageUnusedFileRetentionPeriodInDays { get; set; } = 30;
     [RemoteConfiguration]
     public Uri CdnFullUrl { get; set; } = null;
     [RemoteConfiguration]
@@ -28,6 +32,10 @@ public class StaticFilesServerConfiguration : MareConfigurationBase
         sb.AppendLine($"{nameof(MainFileServerAddress)} => {MainFileServerAddress}");
         sb.AppendLine($"{nameof(ForcedDeletionOfFilesAfterHours)} => {ForcedDeletionOfFilesAfterHours}");
         sb.AppendLine($"{nameof(CacheSizeHardLimitInGiB)} => {CacheSizeHardLimitInGiB}");
+        sb.AppendLine($"{nameof(UseColdStorage)} => {UseColdStorage}");
+        sb.AppendLine($"{nameof(ColdStorageDirectory)} => {ColdStorageDirectory}");
+        sb.AppendLine($"{nameof(ColdStorageSizeHardLimitInGiB)} => {ColdStorageSizeHardLimitInGiB}");
+        sb.AppendLine($"{nameof(ColdStorageUnusedFileRetentionPeriodInDays)} => {ColdStorageUnusedFileRetentionPeriodInDays}");
         sb.AppendLine($"{nameof(UnusedFileRetentionPeriodInDays)} => {UnusedFileRetentionPeriodInDays}");
         sb.AppendLine($"{nameof(CacheDirectory)} => {CacheDirectory}");
         sb.AppendLine($"{nameof(DownloadQueueSize)} => {DownloadQueueSize}");

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs

@@ -13,6 +13,7 @@ using MareSynchronosStaticFilesServer.Utils;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.SignalR;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Configuration;
 using System.Collections.Concurrent;
 using System.Security.Cryptography;
 using System.Security.Policy;
@@ -38,7 +39,9 @@ public class ServerFilesController : ControllerBase
         IHubContext<MareHub> hubContext,
         MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger)
     {
-        _basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
+        _basePath = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false)
+            ? configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory))
+            : configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
         _cachedFileProvider = cachedFileProvider;
         _configuration = configuration;
         _hubContext = hubContext;
@@ -50,14 +53,15 @@ public class ServerFilesController : ControllerBase
     public async Task<IActionResult> FilesDeleteAll()
     {
         var ownFiles = await _mareDbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == MareUser).ToListAsync().ConfigureAwait(false);
+        bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
 
         foreach (var dbFile in ownFiles)
         {
             var fi = FilePathUtil.GetFileInfoForHash(_basePath, dbFile.Hash);
             if (fi != null)
             {
-                _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1);
+                _metricsClient.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1);
-                _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0);
+                _metricsClient.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0);
 
                 fi?.Delete();
             }
@@ -262,8 +266,10 @@ public class ServerFilesController : ControllerBase
             }).ConfigureAwait(false);
             await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
 
-            _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotal, 1);
+            bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
-            _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotalSize, compressedSize);
+
+            _metricsClient.IncGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal, 1);
+            _metricsClient.IncGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, compressedSize);
 
             _fileUploadLocks.TryRemove(hash, out _);
 

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs

@@ -10,12 +10,13 @@ namespace MareSynchronosStaticFilesServer.Services;
 
 public sealed class CachedFileProvider : IDisposable
 {
+    private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
     private readonly ILogger<CachedFileProvider> _logger;
     private readonly FileStatisticsService _fileStatisticsService;
     private readonly MareMetrics _metrics;
     private readonly ServerTokenGenerator _generator;
     private readonly Uri _remoteCacheSourceUri;
-    private readonly string _basePath;
+    private readonly string _hotStoragePath;
     private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
     private readonly HttpClient _httpClient;
     private readonly SemaphoreSlim _downloadSemaphore = new(1, 1);
@@ -24,15 +25,17 @@ public sealed class CachedFileProvider : IDisposable
     private bool IsMainServer => _remoteCacheSourceUri == null && _isDistributionServer;
     private bool _isDistributionServer;
 
-    public CachedFileProvider(IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics, ServerTokenGenerator generator)
+    public CachedFileProvider(IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger,
+        FileStatisticsService fileStatisticsService, MareMetrics metrics, ServerTokenGenerator generator)
     {
+        _configuration = configuration;
         _logger = logger;
         _fileStatisticsService = fileStatisticsService;
         _metrics = metrics;
         _generator = generator;
         _remoteCacheSourceUri = configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.DistributionFileServerAddress), null);
         _isDistributionServer = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false);
-        _basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
+        _hotStoragePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
         _httpClient = new();
         _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer", "1.0.0.0"));
     }
@@ -50,7 +53,12 @@ public sealed class CachedFileProvider : IDisposable
 
     private async Task DownloadTask(string hash)
     {
-        // download file from remote
+        var destinationFilePath = FilePathUtil.GetFilePath(_hotStoragePath, hash);
+
+        // first check cold storage
+        if (TryCopyFromColdStorage(hash, destinationFilePath)) return;
+
+        // if cold storage is not configured or file not found or error is present try to download file from remote
         var downloadUrl = MareFiles.DistributionGetFullPath(_remoteCacheSourceUri, hash);
         _logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl);
 
@@ -70,8 +78,7 @@ public sealed class CachedFileProvider : IDisposable
             return;
         }
 
-        var fileName = FilePathUtil.GetFilePath(_basePath, hash);
+        var tempFileName = destinationFilePath + ".dl";
-        var tempFileName = fileName + ".dl";
         var fileStream = new FileStream(tempFileName, FileMode.Create, FileAccess.ReadWrite);
         var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024;
         var buffer = new byte[bufferSize];
@@ -84,17 +91,54 @@ public sealed class CachedFileProvider : IDisposable
         }
         await fileStream.FlushAsync().ConfigureAwait(false);
         await fileStream.DisposeAsync().ConfigureAwait(false);
-        File.Move(tempFileName, fileName, true);
+        File.Move(tempFileName, destinationFilePath, true);
 
         _metrics.IncGauge(MetricsAPI.GaugeFilesTotal);
-        _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_basePath, hash).Length);
+        _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash).Length);
         response.Dispose();
     }
 
+    private bool TryCopyFromColdStorage(string hash, string destinationFilePath)
+    {
+        if (!_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false)) return false;
+
+        string coldStorageDir = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageDirectory), string.Empty);
+        if (string.IsNullOrEmpty(coldStorageDir)) return false;
+
+        var coldStorageFilePath = FilePathUtil.GetFileInfoForHash(coldStorageDir, hash);
+        if (coldStorageFilePath == null) return false;
+
+        try
+        {
+            _logger.LogDebug("Copying {hash} from cold storage: {path}", hash, coldStorageFilePath);
+            var tempFileName = destinationFilePath + ".dl";
+            File.Copy(coldStorageFilePath.FullName, tempFileName, true);
+            File.Move(tempFileName, destinationFilePath, true);
+            coldStorageFilePath.LastAccessTimeUtc = DateTime.UtcNow;
+            var destinationFile = new FileInfo(destinationFilePath);
+            destinationFile.LastAccessTimeUtc = DateTime.UtcNow;
+            destinationFile.CreationTimeUtc = DateTime.UtcNow;
+            destinationFile.LastWriteTimeUtc = DateTime.UtcNow;
+            _metrics.IncGauge(MetricsAPI.GaugeFilesTotal);
+            _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, new FileInfo(destinationFilePath).Length);
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Could not copy {coldStoragePath} from cold storage", coldStorageFilePath);
+        }
+
+        return false;
+    }
+
     public async Task DownloadFileWhenRequired(string hash)
     {
-        var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
+        var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash);
-        if (fi == null && IsMainServer) return;
+        if (fi == null && IsMainServer)
+        {
+            TryCopyFromColdStorage(hash, FilePathUtil.GetFilePath(_hotStoragePath, hash));
+            return;
+        }
 
         await _downloadSemaphore.WaitAsync().ConfigureAwait(false);
         if ((fi == null || (fi?.Length ?? 0) == 0)
@@ -124,9 +168,11 @@ public sealed class CachedFileProvider : IDisposable
 
     public FileStream? GetLocalFileStream(string hash)
     {
-        var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
+        var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash);
         if (fi == null) return null;
 
+        fi.LastAccessTimeUtc = DateTime.UtcNow;
+
         _fileStatisticsService.LogFile(hash, fi.Length);
 
         return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read);

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs

@@ -10,7 +10,6 @@ namespace MareSynchronosStaticFilesServer.Services;
 
 public class MainFileCleanupService : IHostedService
 {
-    private readonly string _cacheDir;
     private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
     private readonly ILogger<MainFileCleanupService> _logger;
     private readonly MareMetrics _metrics;
@@ -24,7 +23,6 @@ public class MainFileCleanupService : IHostedService
         _logger = logger;
         _services = services;
         _configuration = configuration;
-        _cacheDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
     }
 
     public async Task CleanUpTask(CancellationToken ct)
@@ -35,17 +33,48 @@ public class MainFileCleanupService : IHostedService
         {
             try
             {
-                DirectoryInfo dir = new(_cacheDir);
+                var hotStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
-                var allFiles = dir.GetFiles("*", SearchOption.AllDirectories);
+                DirectoryInfo dir = new(hotStorageDir);
-                _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFiles.Sum(f => f.Length));
+                var allFilesInHotStorage = dir.GetFiles("*", SearchOption.AllDirectories).ToList();
-                _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFiles.Length);
+                _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFilesInHotStorage.Sum(f => f.Length));
+                _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFilesInHotStorage.Count);
 
                 using var scope = _services.CreateScope();
                 using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
 
-                await CleanUpOutdatedFiles(dbContext, ct).ConfigureAwait(false);
+                bool useColdStorage = _configuration.GetValueOrDefault<bool>(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
 
-                CleanUpFilesBeyondSizeLimit(dbContext, ct);
+                if (useColdStorage)
+                {
+                    var coldStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
+                    var coldStorageRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60);
+                    var coldStorageSize = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1);
+
+                    DirectoryInfo dirColdStorage = new(coldStorageDir);
+                    var allFilesInColdStorageDir = dirColdStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
+                    _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, allFilesInColdStorageDir.Sum(f => f.Length));
+                    _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, allFilesInColdStorageDir.Count);
+
+                    // clean up cold storage
+                    var remainingColdFiles = await CleanUpOutdatedFiles(coldStorageDir, allFilesInColdStorageDir, coldStorageRetention, forcedDeletionAfterHours: -1,
+                        isColdStorage: true, deleteFromDb: true,
+                        dbContext, ct).ConfigureAwait(false);
+                    CleanUpFilesBeyondSizeLimit(remainingColdFiles, coldStorageSize,
+                        isColdStorage: true, deleteFromDb: true,
+                        dbContext, ct);
+                }
+
+                var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
+                var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
+                var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);
+
+                var remainingHotFiles = await CleanUpOutdatedFiles(hotStorageDir, allFilesInHotStorage, unusedRetention, forcedDeletionAfterHours,
+                    isColdStorage: false, deleteFromDb: !useColdStorage,
+                    dbContext, ct).ConfigureAwait(false);
+
+                CleanUpFilesBeyondSizeLimit(remainingHotFiles, sizeLimit,
+                    isColdStorage: false, deleteFromDb: !useColdStorage,
+                    dbContext, ct);
 
                 CleanUpStuckUploads(dbContext);
 
@@ -84,9 +113,8 @@ public class MainFileCleanupService : IHostedService
         return Task.CompletedTask;
     }
 
-    private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct)
+    private void CleanUpFilesBeyondSizeLimit(List<FileInfo> files, double sizeLimit, bool isColdStorage, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct)
     {
-        var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);
         if (sizeLimit <= 0)
         {
             return;
@@ -95,8 +123,7 @@ public class MainFileCleanupService : IHostedService
         try
         {
             _logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", sizeLimit);
-            var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories)
+            var allLocalFiles = files
-                .Select(f => new FileInfo(f)).ToList()
                 .OrderBy(f => f.LastAccessTimeUtc).ToList();
             var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
             long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes;
@@ -105,12 +132,13 @@ public class MainFileCleanupService : IHostedService
                 var oldestFile = allLocalFiles[0];
                 allLocalFiles.Remove(oldestFile);
                 totalCacheSizeInBytes -= oldestFile.Length;
-                _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, oldestFile.Length);
+                _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, oldestFile.Length);
-                _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
+                _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal);
                 _logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes);
                 oldestFile.Delete();
                 FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() };
-                dbContext.Entry(f).State = EntityState.Deleted;
+                if (deleteFromDb)
+                    dbContext.Entry(f).State = EntityState.Deleted;
             }
         }
         catch (Exception ex)
@@ -119,10 +147,10 @@ public class MainFileCleanupService : IHostedService
         }
     }
 
-    private void CleanUpOrphanedFiles(List<FileCache> allFiles, FileInfo[] allPhysicalFiles, CancellationToken ct)
+    private List<FileInfo> CleanUpOrphanedFiles(List<FileCache> allFiles, List<FileInfo> allPhysicalFiles, CancellationToken ct)
     {
         var allFilesHashes = new HashSet<string>(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal);
-        foreach (var file in allPhysicalFiles)
+        foreach (var file in allPhysicalFiles.ToList())
         {
             if (!allFilesHashes.Contains(file.Name.ToUpperInvariant()))
             {
@@ -134,15 +162,15 @@ public class MainFileCleanupService : IHostedService
 
             ct.ThrowIfCancellationRequested();
         }
+
+        return allPhysicalFiles;
     }
 
-    private async Task CleanUpOutdatedFiles(MareDbContext dbContext, CancellationToken ct)
+    private async Task<List<FileInfo>> CleanUpOutdatedFiles(string cacheDir, List<FileInfo> allFilesInDir, int unusedRetention, int forcedDeletionAfterHours,
+        bool isColdStorage, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct)
     {
         try
         {
-            var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
-            var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
-
             _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", unusedRetention);
             if (forcedDeletionAfterHours > 0)
             {
@@ -152,39 +180,45 @@ public class MainFileCleanupService : IHostedService
             // clean up files in DB but not on disk or last access is expired
             var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention));
             var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
-            DirectoryInfo dir = new(_cacheDir);
+            List<string> deletedFiles = new();
-            var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories);
             var files = dbContext.Files.OrderBy(f => f.Hash);
-            List<FileCache> allFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false);
+            List<FileCache> allDbFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false);
             int fileCounter = 0;
-            foreach (var fileCache in allFiles.Where(f => f.Uploaded))
+
+            foreach (var fileCache in allDbFiles.Where(f => f.Uploaded))
             {
                 bool fileDeleted = false;
 
-                var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash);
+                var file = FilePathUtil.GetFileInfoForHash(cacheDir, fileCache.Hash);
                 if (file == null)
                 {
                     _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash);
-                    dbContext.Files.Remove(fileCache);
                     fileDeleted = true;
+
+                    if (deleteFromDb)
+                        dbContext.Files.Remove(fileCache);
                 }
                 else if (file != null && file.LastAccessTime < prevTime)
                 {
-                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
+                    _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, file.Length);
-                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
+                    _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal);
                     _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
                     file.Delete();
+                    deletedFiles.Add(file.FullName);
                     fileDeleted = true;
-                    dbContext.Files.Remove(fileCache);
+                    if (deleteFromDb)
+                        dbContext.Files.Remove(fileCache);
                 }
                 else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
                 {
-                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
+                    _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, file.Length);
-                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
+                    _metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal);
                     _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
                     file.Delete();
                     fileDeleted = true;
-                    dbContext.Files.Remove(fileCache);
+                    deletedFiles.Add(file.FullName);
+                    if (deleteFromDb)
+                        dbContext.Files.Remove(fileCache);
                 }
 
                 if (!fileDeleted && file != null && fileCache.Size == 0)
@@ -201,17 +235,19 @@ public class MainFileCleanupService : IHostedService
             }
 
             // clean up files that are on disk but not in DB for some reason
-            CleanUpOrphanedFiles(allFiles, allFilesInDir, ct);
+            return CleanUpOrphanedFiles(allDbFiles, allFilesInDir.Where(f => !deletedFiles.Contains(f.FullName)).ToList(), ct);
         }
         catch (Exception ex)
         {
             _logger.LogWarning(ex, "Error during file cleanup of old files");
         }
+
+        return [];
     }
 
     private void CleanUpStuckUploads(MareDbContext dbContext)
     {
-        var pastTime = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10));
+        var pastTime = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(20));
         var stuckUploads = dbContext.Files.Where(f => !f.Uploaded && f.UploadDate < pastTime);
         dbContext.Files.RemoveRange(stuckUploads);
     }

MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs

@@ -60,6 +60,8 @@ public class Startup
             MetricsAPI.CounterFileRequestSize
         }, new List<string>
         {
+            MetricsAPI.GaugeFilesTotalColdStorage,
+            MetricsAPI.GaugeFilesTotalSizeColdStorage,
             MetricsAPI.GaugeFilesTotalSize,
             MetricsAPI.GaugeFilesTotal,
             MetricsAPI.GaugeFilesUniquePastDay,