Allow shards to use cold storage, revise cleanup service

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs

@@ -28,6 +28,7 @@ public class ServerFilesController : ControllerBase
     private static readonly SemaphoreSlim _fileLockDictLock = new(1);
     private static readonly ConcurrentDictionary<string, SemaphoreSlim> _fileUploadLocks = new(StringComparer.Ordinal);
     private readonly string _basePath;
+    private readonly string _coldBasePath;
     private readonly CachedFileProvider _cachedFileProvider;
     private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
     private readonly IHubContext<MareHub> _hubContext;
@@ -39,11 +40,11 @@ public class ServerFilesController : ControllerBase
         IHubContext<MareHub> hubContext,
         MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger)
     {
-        _basePath = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false)
-            ? configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory))
-            : configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
-        _cachedFileProvider = cachedFileProvider;
         _configuration = configuration;
+        _basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
+        if (_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false))
+            _basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
+        _cachedFileProvider = cachedFileProvider;
         _hubContext = hubContext;
         _mareDbContext = mareDbContext;
         _metricsClient = metricsClient;
@@ -53,20 +54,34 @@ public class ServerFilesController : ControllerBase
     public async Task<IActionResult> FilesDeleteAll()
     {
         var ownFiles = await _mareDbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == MareUser).ToListAsync().ConfigureAwait(false);
-        bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
 
         foreach (var dbFile in ownFiles)
         {
             var fi = FilePathUtil.GetFileInfoForHash(_basePath, dbFile.Hash);
             if (fi != null)
             {
-                _metricsClient.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1);
+                _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1);
-                _metricsClient.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0);
+                _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0);
 
                 fi?.Delete();
             }
         }
 
+        if (!_coldBasePath.IsNullOrEmpty())
+        {
+            foreach (var dbFile in ownFiles)
+            {
+                var fi = FilePathUtil.GetFileInfoForHash(_coldBasePath, dbFile.Hash);
+                if (fi != null)
+                {
+                    _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalColdStorage, fi == null ? 0 : 1);
+                    _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSizeColdStorage, fi?.Length ?? 0);
+
+                    fi?.Delete();
+                }
+            }
+        }
+
         _mareDbContext.Files.RemoveRange(ownFiles);
         await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
 
@@ -266,10 +281,8 @@ public class ServerFilesController : ControllerBase
             }).ConfigureAwait(false);
             await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
 
-            bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
+            _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotal, 1);
-
+            _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotalSize, compressedSize);
-            _metricsClient.IncGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal, 1);
-            _metricsClient.IncGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, compressedSize);
 
             _fileUploadLocks.TryRemove(hash, out _);
 

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs

@@ -16,7 +16,9 @@ public sealed class CachedFileProvider : IDisposable
     private readonly MareMetrics _metrics;
     private readonly ServerTokenGenerator _generator;
     private readonly Uri _remoteCacheSourceUri;
+    private readonly bool _useColdStorage;
     private readonly string _hotStoragePath;
+    private readonly string _coldStoragePath;
     private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
     private readonly HttpClient _httpClient;
     private readonly SemaphoreSlim _downloadSemaphore = new(1, 1);
@@ -35,7 +37,9 @@ public sealed class CachedFileProvider : IDisposable
         _generator = generator;
         _remoteCacheSourceUri = configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.DistributionFileServerAddress), null);
         _isDistributionServer = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false);
+        _useColdStorage = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
         _hotStoragePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
+        _coldStoragePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
         _httpClient = new();
         _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer", "1.0.0.0"));
     }
@@ -53,7 +57,7 @@ public sealed class CachedFileProvider : IDisposable
 
     private async Task DownloadTask(string hash)
     {
-        var destinationFilePath = FilePathUtil.GetFilePath(_hotStoragePath, hash);
+        var destinationFilePath = FilePathUtil.GetFilePath(_useColdStorage ? _coldStoragePath : _hotStoragePath, hash);
 
         // if cold storage is not configured or file not found or error is present try to download file from remote
         var downloadUrl = MareFiles.DistributionGetFullPath(_remoteCacheSourceUri, hash);
@@ -77,7 +81,7 @@ public sealed class CachedFileProvider : IDisposable
 
         var tempFileName = destinationFilePath + ".dl";
         var fileStream = new FileStream(tempFileName, FileMode.Create, FileAccess.ReadWrite);
-        var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024;
+        var bufferSize = 4096;
         var buffer = new byte[bufferSize];
 
         var bytesRead = 0;
@@ -90,19 +94,18 @@ public sealed class CachedFileProvider : IDisposable
         await fileStream.DisposeAsync().ConfigureAwait(false);
         File.Move(tempFileName, destinationFilePath, true);
 
-        _metrics.IncGauge(MetricsAPI.GaugeFilesTotal);
+        _metrics.IncGauge(_useColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal);
-        _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash).Length);
+        _metrics.IncGauge(_useColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, new FileInfo(destinationFilePath).Length);
         response.Dispose();
     }
 
     private bool TryCopyFromColdStorage(string hash, string destinationFilePath)
     {
-        if (!_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false)) return false;
+        if (!_useColdStorage) return false;
 
-        string coldStorageDir = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageDirectory), string.Empty);
+        if (string.IsNullOrEmpty(_coldStoragePath)) return false;
-        if (string.IsNullOrEmpty(coldStorageDir)) return false;
 
-        var coldStorageFilePath = FilePathUtil.GetFileInfoForHash(coldStorageDir, hash);
+        var coldStorageFilePath = FilePathUtil.GetFileInfoForHash(_coldStoragePath, hash);
         if (coldStorageFilePath == null) return false;
 
         try
@@ -131,16 +134,20 @@ public sealed class CachedFileProvider : IDisposable
     public async Task DownloadFileWhenRequired(string hash)
     {
         var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash);
-        if (fi == null)
+
-        {
+        if (fi != null && fi.Length != 0)
-            if (TryCopyFromColdStorage(hash, FilePathUtil.GetFilePath(_hotStoragePath, hash)))
+            return;
-                return;
+
-        }
+        // first check cold storage
+        if (TryCopyFromColdStorage(hash, FilePathUtil.GetFilePath(_hotStoragePath, hash)))
+            return;
+
+        // no distribution server configured to download from
+        if (_remoteCacheSourceUri == null)
+            return;
 
         await _downloadSemaphore.WaitAsync().ConfigureAwait(false);
-        if ((fi == null || (fi?.Length ?? 0) == 0)
+        if (!_currentTransfers.TryGetValue(hash, out var downloadTask) || (downloadTask?.IsCompleted ?? true))
-            && (!_currentTransfers.TryGetValue(hash, out var downloadTask)
-                || (downloadTask?.IsCompleted ?? true)))
         {
             _currentTransfers[hash] = Task.Run(async () =>
             {

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs

@@ -14,8 +14,35 @@ public class FileCleanupService : IHostedService
     private readonly ILogger<FileCleanupService> _logger;
     private readonly MareMetrics _metrics;
     private readonly IServiceProvider _services;
+
+    private readonly string _hotStoragePath;
+    private readonly string _coldStoragePath;
+    private readonly bool _isMain = false;
+    private readonly bool _isDistributionNode = false;
+    private readonly bool _useColdStorage = false;
+
     private CancellationTokenSource _cleanupCts;
 
+    private int HotStorageRetention => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
+    private double HotStorageSize => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1.0);
+
+    private int ColdStorageRetention => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60);
+    private double ColdStorageSize => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1.0);
+
+    private int ForcedDeletionAfterHours => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
+    private int CleanupCheckMinutes => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CleanupCheckInMinutes), 15);
+
+    private List<FileInfo> GetAllHotFiles() => new DirectoryInfo(_hotStoragePath).GetFiles("*", SearchOption.AllDirectories)
+                .Where(f => f != null && f.Name.Length == 40)
+                .OrderBy(f => f.LastAccessTimeUtc).ToList();
+
+    private List<FileInfo> GetAllColdFiles() => new DirectoryInfo(_coldStoragePath).GetFiles("*", SearchOption.AllDirectories)
+                .Where(f => f != null && f.Name.Length == 40)
+                .OrderBy(f => f.LastAccessTimeUtc).ToList();
+
+    private List<FileInfo> GetTempFiles() => new DirectoryInfo(_useColdStorage ? _coldStoragePath : _hotStoragePath).GetFiles("*", SearchOption.AllDirectories)
+                .Where(f => f != null && (f.Name.EndsWith(".dl", StringComparison.InvariantCultureIgnoreCase) || f.Name.EndsWith(".tmp", StringComparison.InvariantCultureIgnoreCase))).ToList();
+
     public FileCleanupService(MareMetrics metrics, ILogger<FileCleanupService> logger,
         IServiceProvider services, IConfigurationService<StaticFilesServerConfiguration> configuration)
     {
@@ -23,6 +50,11 @@ public class FileCleanupService : IHostedService
         _logger = logger;
         _services = services;
         _configuration = configuration;
+        _useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
+        _hotStoragePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
+        _coldStoragePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
+        _isDistributionNode = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false);
+        _isMain = configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.MainFileServerAddress)) == null && _isDistributionNode;
     }
 
     public Task StartAsync(CancellationToken cancellationToken)
@@ -45,18 +77,18 @@ public class FileCleanupService : IHostedService
         return Task.CompletedTask;
     }
 
-    private List<FileInfo> CleanUpFilesBeyondSizeLimit(List<FileInfo> files, double sizeLimit, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct)
+    private List<string> CleanUpFilesBeyondSizeLimit(List<FileInfo> files, double sizeLimit, CancellationToken ct)
     {
+        var removedFiles = new List<string>();
         if (sizeLimit <= 0)
         {
-            return [];
+            return removedFiles;
         }
 
         try
         {
             _logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", sizeLimit);
-            var allLocalFiles = files
+            var allLocalFiles = files;
-                .OrderBy(f => f.LastAccessTimeUtc).ToList();
             var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
             long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes;
             while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Count != 0 && !ct.IsCancellationRequested)
@@ -66,30 +98,24 @@ public class FileCleanupService : IHostedService
                 totalCacheSizeInBytes -= oldestFile.Length;
                 _logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes);
                 oldestFile.Delete();
-                FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() };
+                removedFiles.Add(oldestFile.Name);
-                if (deleteFromDb)
-                    dbContext.Entry(f).State = EntityState.Deleted;
             }
-
+            files.RemoveAll(f => removedFiles.Contains(f.Name, StringComparer.InvariantCultureIgnoreCase));
-            return allLocalFiles;
         }
         catch (Exception ex)
         {
             _logger.LogWarning(ex, "Error during cache size limit cleanup");
         }
 
-        return [];
+        return removedFiles;
     }
 
-    private List<FileInfo> CleanUpOrphanedFiles(List<FileCache> allFiles, List<FileInfo> allPhysicalFiles, CancellationToken ct)
+    private void CleanUpOrphanedFiles(HashSet<string> allDbFileHashes, List<FileInfo> allPhysicalFiles, CancellationToken ct)
     {
-        var allFilesHashes = new HashSet<string>(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal);
         foreach (var file in allPhysicalFiles.ToList())
         {
-            if (!allFilesHashes.Contains(file.Name.ToUpperInvariant()))
+            if (!allDbFileHashes.Contains(file.Name.ToUpperInvariant()))
             {
-                _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
-                _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
                 file.Delete();
                 _logger.LogInformation("File not in DB, deleting: {fileName}", file.Name);
                 allPhysicalFiles.Remove(file);
@@ -97,13 +123,11 @@ public class FileCleanupService : IHostedService
 
             ct.ThrowIfCancellationRequested();
         }
-
-        return allPhysicalFiles;
     }
 
-    private async Task<List<FileInfo>> CleanUpOutdatedFiles(string dir, List<FileInfo> allFilesInDir, int unusedRetention, int forcedDeletionAfterHours,
+    private List<string> CleanUpOutdatedFiles(List<FileInfo> files, int unusedRetention, int forcedDeletionAfterHours, CancellationToken ct)
-        bool deleteFromDb, MareDbContext dbContext, CancellationToken ct)
     {
+        var removedFiles = new List<string>();
         try
         {
             _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", unusedRetention);
@@ -112,37 +136,42 @@ public class FileCleanupService : IHostedService
                 _logger.LogInformation("Cleaning up files written to longer than {hours}h ago", forcedDeletionAfterHours);
             }
 
-            // clean up files in DB but not on disk or last access is expired
+            var lastAccessCutoffTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention));
-            var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention));
+            var forcedDeletionCutoffTime = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
-            var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
-            List<FileCache> allDbFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false);
-            List<string> removedFileHashes;
 
-            if (!deleteFromDb)
+            foreach (var file in files)
             {
-                removedFileHashes = CleanupViaFiles(allFilesInDir, forcedDeletionAfterHours, prevTime, prevTimeForcedDeletion, ct);
+                if (file.LastAccessTime < lastAccessCutoffTime)
-            }
+                {
-            else
+                    _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
-            {
+                    file.Delete();
-                removedFileHashes = await CleanupViaDb(dir, forcedDeletionAfterHours, dbContext, prevTime, prevTimeForcedDeletion, allDbFiles, ct).ConfigureAwait(false);
+                    removedFiles.Add(file.Name);
-            }
+                }
+                else if (forcedDeletionAfterHours > 0 && file.LastWriteTime < forcedDeletionCutoffTime)
+                {
+                    _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
+                    file.Delete();
+                    removedFiles.Add(file.Name);
+                }
 
-            // clean up files that are on disk but not in DB anymore
+                ct.ThrowIfCancellationRequested();
-            return CleanUpOrphanedFiles(allDbFiles, allFilesInDir.Where(c => !removedFileHashes.Contains(c.Name, StringComparer.OrdinalIgnoreCase)).ToList(), ct);
+            }
+            files.RemoveAll(f => removedFiles.Contains(f.Name, StringComparer.InvariantCultureIgnoreCase));
         }
         catch (Exception ex)
         {
             _logger.LogWarning(ex, "Error during file cleanup of old files");
         }
 
-        return [];
+        return removedFiles;
     }
 
-    private void CleanUpStuckUploads(MareDbContext dbContext)
+    private void CleanUpTempFiles()
     {
         var pastTime = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(20));
-        var stuckUploads = dbContext.Files.Where(f => !f.Uploaded && f.UploadDate < pastTime);
+        var tempFiles = GetTempFiles();
-        dbContext.Files.RemoveRange(stuckUploads);
+        foreach (var tempFile in tempFiles.Where(f => f.LastWriteTimeUtc < pastTime))
+            tempFile.Delete();
     }
 
     private async Task CleanUpTask(CancellationToken ct)
@@ -152,61 +181,87 @@ public class FileCleanupService : IHostedService
             try
             {
                 using var scope = _services.CreateScope();
-                using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
+                using var dbContext = _isMain ? scope.ServiceProvider.GetService<MareDbContext>()! : null;
 
-                bool useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
+                HashSet<string> allDbFileHashes = null;
 
-                if (useColdStorage)
+                // Database operations only performed on main server
+                if (_isMain)
                 {
-                    var coldStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
+                    var allDbFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false);
-
+                    allDbFileHashes = new HashSet<string>(allDbFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal);
-                    DirectoryInfo dirColdStorage = new(coldStorageDir);
-                    var allFilesInColdStorageDir = dirColdStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
-
-                    var coldStorageRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60);
-                    var coldStorageSize = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1);
-
-                    // clean up cold storage
-                    var remainingColdFiles = await CleanUpOutdatedFiles(coldStorageDir, allFilesInColdStorageDir, coldStorageRetention, forcedDeletionAfterHours: -1,
-                        deleteFromDb: true, dbContext: dbContext,
-                        ct: ct).ConfigureAwait(false);
-                    var finalRemainingColdFiles = CleanUpFilesBeyondSizeLimit(remainingColdFiles, coldStorageSize,
-                        deleteFromDb: true, dbContext: dbContext,
-                        ct: ct);
-
-                    _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, finalRemainingColdFiles.Sum(f => f.Length));
-                    _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, finalRemainingColdFiles.Count);
                 }
 
-                var hotStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
+                if (_useColdStorage)
-                DirectoryInfo dirHotStorage = new(hotStorageDir);
+                {
-                var allFilesInHotStorage = dirHotStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
+                    var coldFiles = GetAllColdFiles();
+                    var removedColdFiles = new List<string>();
 
-                var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
+                    removedColdFiles.AddRange(
-                var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
+                        CleanUpOutdatedFiles(coldFiles, ColdStorageRetention, ForcedDeletionAfterHours, ct)
-                var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);
+                    );
+                    removedColdFiles.AddRange(
+                        CleanUpFilesBeyondSizeLimit(coldFiles, ColdStorageSize, ct)
+                    );
 
-                var remainingHotFiles = await CleanUpOutdatedFiles(hotStorageDir, allFilesInHotStorage, unusedRetention, forcedDeletionAfterHours,
+                    // Remove cold storage files are deleted from the database, if we are the main file server
-                    deleteFromDb: !useColdStorage, dbContext: dbContext,
+                    if (_isMain)
-                    ct: ct).ConfigureAwait(false);
+                    {
+                        dbContext.Files.RemoveRange(
+                            dbContext.Files.Where(f => removedColdFiles.Contains(f.Hash))
+                        );
+                        allDbFileHashes.ExceptWith(removedColdFiles);
+                        CleanUpOrphanedFiles(allDbFileHashes, coldFiles, ct);
+                    }
 
-                var finalRemainingHotFiles = CleanUpFilesBeyondSizeLimit(remainingHotFiles, sizeLimit,
+                    // Remove hot copies of files now that the authoritative copy is gone
-                    deleteFromDb: !useColdStorage, dbContext: dbContext,
+                    foreach (var removedFile in removedColdFiles)
-                    ct: ct);
+                    {
+                        var hotFile = FilePathUtil.GetFileInfoForHash(_hotStoragePath, removedFile);
+                        hotFile?.Delete();
+                    }
 
-                CleanUpStuckUploads(dbContext);
+                    _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, coldFiles.Sum(f => { try { return f.Length; } catch { return 0; } }));
+                    _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, coldFiles.Count);
+                }
 
-                await dbContext.SaveChangesAsync(ct).ConfigureAwait(false);
+                var hotFiles = GetAllHotFiles();
+                var removedHotFiles = new List<string>();
 
-                _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, finalRemainingHotFiles.Sum(f => { try { return f.Length; } catch { return 0; } }));
+                removedHotFiles.AddRange(
-                _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, finalRemainingHotFiles.Count);
+                    CleanUpOutdatedFiles(hotFiles, HotStorageRetention, forcedDeletionAfterHours: _useColdStorage ? ForcedDeletionAfterHours : -1, ct)
+                );
+                removedHotFiles.AddRange(
+                    CleanUpFilesBeyondSizeLimit(hotFiles, HotStorageSize, ct)
+                );
+
+                if (_isMain)
+                {
+                    // If cold storage is not active, then "hot" files are deleted from the database instead
+                    if (!_useColdStorage)
+                    {
+                        dbContext.Files.RemoveRange(
+                            dbContext.Files.Where(f => removedHotFiles.Contains(f.Hash))
+                        );
+                        allDbFileHashes.ExceptWith(removedHotFiles);
+                    }
+
+                    CleanUpOrphanedFiles(allDbFileHashes, hotFiles, ct);
+
+                    await dbContext.SaveChangesAsync(ct).ConfigureAwait(false);
+                }
+
+                _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, hotFiles.Sum(f => { try { return f.Length; } catch { return 0; } }));
+                _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, hotFiles.Count);
+
+                CleanUpTempFiles();
             }
             catch (Exception e)
             {
                 _logger.LogError(e, "Error during cleanup task");
             }
 
-            var cleanupCheckMinutes = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CleanupCheckInMinutes), 15);
+            var cleanupCheckMinutes = CleanupCheckMinutes;
             var now = DateTime.Now;
             TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
             TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % cleanupCheckMinutes, 0);
@@ -216,109 +271,18 @@ public class FileCleanupService : IHostedService
             await Task.Delay(span, ct).ConfigureAwait(false);
         }
     }
-    private async Task<List<string>> CleanupViaDb(string dir, int forcedDeletionAfterHours,
-        MareDbContext dbContext, DateTime lastAccessCutoffTime, DateTime forcedDeletionCutoffTime, List<FileCache> allDbFiles, CancellationToken ct)
-    {
-        int fileCounter = 0;
-        List<string> removedFileHashes = new();
-        foreach (var fileCache in allDbFiles.Where(f => f.Uploaded))
-        {
-            bool deleteCurrentFile = false;
-            var file = FilePathUtil.GetFileInfoForHash(dir, fileCache.Hash);
-            if (file == null)
-            {
-                _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash);
-                deleteCurrentFile = true;
-            }
-            else if (file != null && file.LastAccessTime < lastAccessCutoffTime)
-            {
-                _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
-                deleteCurrentFile = true;
-            }
-            else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < forcedDeletionCutoffTime)
-            {
-                _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
-                deleteCurrentFile = true;
-            }
-
-            // do actual deletion of file and remove also from db if needed
-            if (deleteCurrentFile)
-            {
-                if (file != null) file.Delete();
-
-                removedFileHashes.Add(fileCache.Hash);
-
-                dbContext.Files.Remove(fileCache);
-            }
-
-            // only used if file in db has no size for whatever reason
-            if (!deleteCurrentFile && file != null && fileCache.Size == 0)
-            {
-                _logger.LogInformation("Setting File Size of " + fileCache.Hash + " to " + file.Length);
-                fileCache.Size = file.Length;
-                // commit every 1000 files to db
-                if (fileCounter % 1000 == 0)
-                    await dbContext.SaveChangesAsync().ConfigureAwait(false);
-            }
-
-            fileCounter++;
-
-            ct.ThrowIfCancellationRequested();
-        }
-
-        return removedFileHashes;
-    }
-
-    private List<string> CleanupViaFiles(List<FileInfo> allFilesInDir, int forcedDeletionAfterHours,
-        DateTime lastAccessCutoffTime, DateTime forcedDeletionCutoffTime, CancellationToken ct)
-    {
-        List<string> removedFileHashes = new List<string>();
-
-        foreach (var file in allFilesInDir)
-        {
-            bool deleteCurrentFile = false;
-            if (file != null && file.LastAccessTime < lastAccessCutoffTime)
-            {
-                _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
-                deleteCurrentFile = true;
-            }
-            else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < forcedDeletionCutoffTime)
-            {
-                _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
-                deleteCurrentFile = true;
-            }
-
-            if (deleteCurrentFile)
-            {
-                if (file != null) file.Delete();
-
-                removedFileHashes.Add(file.Name);
-            }
-
-            ct.ThrowIfCancellationRequested();
-        }
-
-        return removedFileHashes;
-    }
 
     private void InitializeGauges()
     {
-        bool useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
+        if (_useColdStorage)
-
-        if (useColdStorage)
         {
-            var coldStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
+            var allFilesInColdStorageDir = GetAllColdFiles();
-
-            DirectoryInfo dirColdStorage = new(coldStorageDir);
-            var allFilesInColdStorageDir = dirColdStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
 
             _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, allFilesInColdStorageDir.Sum(f => f.Length));
             _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, allFilesInColdStorageDir.Count);
         }
 
-        var hotStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
+        var allFilesInHotStorage = GetAllHotFiles();
-        DirectoryInfo dirHotStorage = new(hotStorageDir);
-        var allFilesInHotStorage = dirHotStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
 
         _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFilesInHotStorage.Sum(f => { try { return f.Length; } catch { return 0; } }));
         _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFilesInHotStorage.Count);

MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs

@@ -82,6 +82,7 @@ public class Startup
 
         // generic services
         services.AddSingleton<CachedFileProvider>();
+        services.AddHostedService<FileCleanupService>();
         services.AddSingleton<FileStatisticsService>();
         services.AddSingleton<RequestFileStreamResultFactory>();
         services.AddSingleton<ServerTokenGenerator>();
@@ -95,7 +96,6 @@ public class Startup
         if (_isMain)
         {
             services.AddSingleton<IClientReadyMessageService, MainClientReadyMessageService>();
-            services.AddHostedService<FileCleanupService>();
             services.AddSingleton<IConfigurationService<StaticFilesServerConfiguration>, MareConfigurationServiceServer<StaticFilesServerConfiguration>>();
             services.AddDbContextPool<MareDbContext>(options =>
             {
@@ -171,7 +171,6 @@ public class Startup
         else
         {
             services.AddSingleton<IClientReadyMessageService, ShardClientReadyMessageService>();
-            services.AddHostedService<ShardFileCleanupService>();
             services.AddSingleton<IConfigurationService<StaticFilesServerConfiguration>, MareConfigurationServiceClient<StaticFilesServerConfiguration>>();
             services.AddHostedService(p => (MareConfigurationServiceClient<StaticFilesServerConfiguration>)p.GetService<IConfigurationService<StaticFilesServerConfiguration>>());
         }