using MareSynchronosServices.Authentication; using MareSynchronosServices.Metrics; using MareSynchronosShared.Data; using MareSynchronosShared.Metrics; using MareSynchronosShared.Models; using MareSynchronosShared.Protos; using Microsoft.EntityFrameworkCore; using MetricsService = MareSynchronosShared.Protos.MetricsService; namespace MareSynchronosServices { public class CleanupService : IHostedService, IDisposable { private readonly MareMetrics metrics; private readonly SecretKeyAuthenticationHandler _authService; private readonly ILogger _logger; private readonly IServiceProvider _services; private readonly IConfiguration _configuration; private Timer _timer; public CleanupService(MareMetrics metrics, SecretKeyAuthenticationHandler authService, ILogger logger, IServiceProvider services, IConfiguration configuration) { this.metrics = metrics; _authService = authService; _logger = logger; _services = services; _configuration = configuration; } public Task StartAsync(CancellationToken cancellationToken) { _logger.LogInformation("Cleanup Service started"); _timer = new Timer(CleanUp, null, TimeSpan.Zero, TimeSpan.FromMinutes(10)); return Task.CompletedTask; } private void CleanUp(object state) { if (!int.TryParse(_configuration["UnusedFileRetentionPeriodInDays"], out var filesOlderThanDays)) { filesOlderThanDays = 7; } using var scope = _services.CreateScope(); using var dbContext = scope.ServiceProvider.GetService()!; _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", filesOlderThanDays); try { var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays)); var allFiles = dbContext.Files.ToList(); var cachedir = _configuration["CacheDirectory"]; foreach (var file in allFiles.Where(f => f.Uploaded)) { var fileName = Path.Combine(cachedir, file.Hash); var fi = new FileInfo(fileName); if (!fi.Exists) { _logger.LogInformation("File does not exist anymore: {fileName}", fileName); dbContext.Files.Remove(file); } else if (fi.LastAccessTime < prevTime) { metrics.DecGaugeBy(MetricsAPI.GaugeFilesTotalSize, fi.Length); metrics.DecGaugeBy(MetricsAPI.GaugeFilesTotal, 1); _logger.LogInformation("File outdated: {fileName}", fileName); dbContext.Files.Remove(file); fi.Delete(); } } } catch (Exception ex) { _logger.LogWarning(ex, "Error during file cleanup"); } var cacheSizeLimitInGiB = _configuration.GetValue("CacheSizeHardLimitInGiB", -1); try { if (cacheSizeLimitInGiB > 0) { _logger.LogInformation("Cleaning up files beyond the cache size limit"); var allLocalFiles = Directory.EnumerateFiles(_configuration["CacheDirectory"]).Select(f => new FileInfo(f)).ToList().OrderBy(f => f.LastAccessTimeUtc).ToList(); var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length); long cacheSizeLimitInBytes = (long)(cacheSizeLimitInGiB * 1024 * 1024 * 1024); HashSet removedHashes = new(); while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any()) { var oldestFile = allLocalFiles.First(); removedHashes.Add(oldestFile.Name.ToLower()); allLocalFiles.Remove(oldestFile); totalCacheSizeInBytes -= oldestFile.Length; metrics.DecGaugeBy(MetricsAPI.GaugeFilesTotalSize, oldestFile.Length); metrics.DecGaugeBy(MetricsAPI.GaugeFilesTotal, 1); oldestFile.Delete(); } dbContext.Files.RemoveRange(dbContext.Files.Where(f => removedHashes.Contains(f.Hash.ToLower()))); } } catch (Exception ex) { _logger.LogWarning(ex, "Error during cache size limit cleanup"); } try { _logger.LogInformation($"Cleaning up expired lodestone authentications"); var lodestoneAuths = dbContext.LodeStoneAuth.Include(u => u.User).Where(a => a.StartedAt != null).ToList(); List expiredAuths = new List(); foreach (var auth in lodestoneAuths) { if (auth.StartedAt < DateTime.UtcNow - TimeSpan.FromMinutes(15)) { expiredAuths.Add(auth); } } dbContext.RemoveRange(expiredAuths.Select(a => a.User)); dbContext.RemoveRange(expiredAuths); } catch (Exception ex) { _logger.LogWarning(ex, "Error during expired auths cleanup"); } try { if (!bool.TryParse(_configuration["PurgeUnusedAccounts"], out var purgeUnusedAccounts)) { purgeUnusedAccounts = false; } if (purgeUnusedAccounts) { if (!int.TryParse(_configuration["PurgeUnusedAccountsPeriodInDays"], out var usersOlderThanDays)) { usersOlderThanDays = 14; } _logger.LogInformation("Cleaning up users older than {usersOlderThanDays} days", usersOlderThanDays); var allUsers = dbContext.Users.ToList(); List usersToRemove = new(); foreach (var user in allUsers) { if (user.LastLoggedIn < (DateTime.UtcNow - TimeSpan.FromDays(usersOlderThanDays))) { _logger.LogInformation("User outdated: {userUID}", user.UID); usersToRemove.Add(user); } } foreach (var user in usersToRemove) { PurgeUser(user, dbContext, _configuration); } } _logger.LogInformation("Cleaning up unauthorized users"); } catch (Exception ex) { _logger.LogWarning(ex, "Error during user purge"); } _authService.ClearUnauthorizedUsers(); _logger.LogInformation($"Cleanup complete"); dbContext.SaveChanges(); } public void PurgeUser(User user, MareDbContext dbContext, IConfiguration _configuration) { var lodestone = dbContext.LodeStoneAuth.SingleOrDefault(a => a.User.UID == user.UID); if (lodestone != null) { dbContext.Remove(lodestone); } _authService.RemoveAuthentication(user.UID); var auth = dbContext.Auth.Single(a => a.UserUID == user.UID); var userFiles = dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == user.UID).ToList(); foreach (var file in userFiles) { var fi = new FileInfo(Path.Combine(_configuration["CacheDirectory"], file.Hash)); if (fi.Exists) { metrics.DecGaugeBy(MetricsAPI.GaugeFilesTotalSize, fi.Length); metrics.DecGaugeBy(MetricsAPI.GaugeFilesTotal, 1); fi.Delete(); } } dbContext.Files.RemoveRange(userFiles); var ownPairData = dbContext.ClientPairs.Where(u => u.User.UID == user.UID).ToList(); dbContext.RemoveRange(ownPairData); var otherPairData = dbContext.ClientPairs.Include(u => u.User) .Where(u => u.OtherUser.UID == user.UID).ToList(); metrics.DecGaugeBy(MetricsAPI.GaugePairs, ownPairData.Count + otherPairData.Count); metrics.DecGaugeBy(MetricsAPI.GaugePairsPaused, ownPairData.Count + ownPairData.Count(c => c.IsPaused)); metrics.DecGaugeBy(MetricsAPI.GaugeUsersRegistered, ownPairData.Count + 1); dbContext.RemoveRange(otherPairData); dbContext.Remove(auth); dbContext.Remove(user); } public Task StopAsync(CancellationToken cancellationToken) { _timer?.Change(Timeout.Infinite, 0); return Task.CompletedTask; } public void Dispose() { _timer?.Dispose(); } } }