t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

clean up some code and optinmize cleanup

Browse Source 
speed up cleanup service

fixes for fixes
This commit is contained in:
Stanley Dimant
2024-05-02 16:04:43 +02:00
committed by Loporrit
parent fb22a267ac
commit 09662b477e
1 changed files with 47 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs
View File

@@ -33,37 +33,37 @@ public class MainFileCleanupService : IHostedService
{ {
try try
{ {
var hotStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
DirectoryInfo dir = new(hotStorageDir);
var allFilesInHotStorage = dir.GetFiles("*", SearchOption.AllDirectories).ToList();
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFilesInHotStorage.Sum(f => f.Length));
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFilesInHotStorage.Count);
using var scope = _services.CreateScope(); using var scope = _services.CreateScope();
using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!; using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
bool useColdStorage = _configuration.GetValueOrDefault<bool>(nameof(StaticFilesServerConfiguration.UseColdStorage), false); bool useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
if (useColdStorage) if (useColdStorage)
{ {
var coldStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)); var coldStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
var coldStorageRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60);
var coldStorageSize = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1);
DirectoryInfo dirColdStorage = new(coldStorageDir); DirectoryInfo dirColdStorage = new(coldStorageDir);
var allFilesInColdStorageDir = dirColdStorage.GetFiles("*", SearchOption.AllDirectories).ToList(); var allFilesInColdStorageDir = dirColdStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, allFilesInColdStorageDir.Sum(f => f.Length));
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, allFilesInColdStorageDir.Count); var coldStorageRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60);
var coldStorageSize = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1);
// clean up cold storage // clean up cold storage
var remainingColdFiles = await CleanUpOutdatedFiles(coldStorageDir, allFilesInColdStorageDir, coldStorageRetention, forcedDeletionAfterHours: -1, var remainingColdFiles = await CleanUpOutdatedFiles(coldStorageDir, allFilesInColdStorageDir, coldStorageRetention, forcedDeletionAfterHours: -1,
isColdStorage: true, deleteFromDb: true, isColdStorage: true, deleteFromDb: true,
dbContext, ct).ConfigureAwait(false); dbContext, ct).ConfigureAwait(false);
CleanUpFilesBeyondSizeLimit(remainingColdFiles, coldStorageSize, var finalRemainingColdFiles = CleanUpFilesBeyondSizeLimit(remainingColdFiles, coldStorageSize,
isColdStorage: true, deleteFromDb: true, isColdStorage: true, deleteFromDb: true,
dbContext, ct); dbContext, ct);
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, finalRemainingColdFiles.Sum(f => f.Length));
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, finalRemainingColdFiles.Count);
} }
var hotStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
DirectoryInfo dirHotStorage = new(hotStorageDir);
var allFilesInHotStorage = dirHotStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14); var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1); var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1); var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);
@@ -72,18 +72,22 @@ public class MainFileCleanupService : IHostedService
isColdStorage: false, deleteFromDb: !useColdStorage, isColdStorage: false, deleteFromDb: !useColdStorage,
dbContext, ct).ConfigureAwait(false); dbContext, ct).ConfigureAwait(false);
CleanUpFilesBeyondSizeLimit(remainingHotFiles, sizeLimit, var finalRemainingHotFiles = CleanUpFilesBeyondSizeLimit(remainingHotFiles, sizeLimit,
isColdStorage: false, deleteFromDb: !useColdStorage, isColdStorage: false, deleteFromDb: !useColdStorage,
dbContext, ct); dbContext, ct);
CleanUpStuckUploads(dbContext); CleanUpStuckUploads(dbContext);
await dbContext.SaveChangesAsync(ct).ConfigureAwait(false); await dbContext.SaveChangesAsync(ct).ConfigureAwait(false);
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, finalRemainingHotFiles.Sum(f => { try { return f.Length; } catch { return 0; } }));
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, finalRemainingHotFiles.Count);
} }
catch (Exception e) catch (Exception e)
{ {
_logger.LogError(e, "Error during cleanup task"); _logger.LogError(e, "Error during cleanup task");
} }
var cleanupCheckMinutes = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CleanupCheckInMinutes), 15); var cleanupCheckMinutes = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CleanupCheckInMinutes), 15);
var now = DateTime.Now; var now = DateTime.Now;
TimeOnly currentTime = new(now.Hour, now.Minute, now.Second); TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
@@ -113,11 +117,11 @@ public class MainFileCleanupService : IHostedService
return Task.CompletedTask; return Task.CompletedTask;
} }
private void CleanUpFilesBeyondSizeLimit(List<FileInfo> files, double sizeLimit, bool isColdStorage, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct) private List<FileInfo> CleanUpFilesBeyondSizeLimit(List<FileInfo> files, double sizeLimit, bool isColdStorage, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct)
{ {
if (sizeLimit <= 0) if (sizeLimit <= 0)
{ {
return; return [];
} }
try try
@@ -127,24 +131,26 @@ public class MainFileCleanupService : IHostedService
.OrderBy(f => f.LastAccessTimeUtc).ToList(); .OrderBy(f => f.LastAccessTimeUtc).ToList();
var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length); var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes; long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes;
while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested) while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Count != 0 && !ct.IsCancellationRequested)
{ {
var oldestFile = allLocalFiles[0]; var oldestFile = allLocalFiles[0];
allLocalFiles.Remove(oldestFile); allLocalFiles.RemoveAt(0);
totalCacheSizeInBytes -= oldestFile.Length; totalCacheSizeInBytes -= oldestFile.Length;
_metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, oldestFile.Length);
_metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal);
_logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes); _logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes);
oldestFile.Delete(); oldestFile.Delete();
FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() }; FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() };
if (deleteFromDb) if (deleteFromDb)
dbContext.Entry(f).State = EntityState.Deleted; dbContext.Entry(f).State = EntityState.Deleted;
} }
return allLocalFiles;
} }
catch (Exception ex) catch (Exception ex)
{ {
_logger.LogWarning(ex, "Error during cache size limit cleanup"); _logger.LogWarning(ex, "Error during cache size limit cleanup");
} }
return [];
} }
private List<FileInfo> CleanUpOrphanedFiles(List<FileCache> allFiles, List<FileInfo> allPhysicalFiles, CancellationToken ct) private List<FileInfo> CleanUpOrphanedFiles(List<FileCache> allFiles, List<FileInfo> allPhysicalFiles, CancellationToken ct)
@@ -158,6 +164,7 @@ public class MainFileCleanupService : IHostedService
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal); _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
file.Delete(); file.Delete();
_logger.LogInformation("File not in DB, deleting: {fileName}", file.Name); _logger.LogInformation("File not in DB, deleting: {fileName}", file.Name);
allPhysicalFiles.Remove(file);
} }
ct.ThrowIfCancellationRequested(); ct.ThrowIfCancellationRequested();
@@ -166,7 +173,7 @@ public class MainFileCleanupService : IHostedService
return allPhysicalFiles; return allPhysicalFiles;
} }
private async Task<List<FileInfo>> CleanUpOutdatedFiles(string cacheDir, List<FileInfo> allFilesInDir, int unusedRetention, int forcedDeletionAfterHours, private async Task<List<FileInfo>> CleanUpOutdatedFiles(string dir, List<FileInfo> allFilesInDir, int unusedRetention, int forcedDeletionAfterHours,
bool isColdStorage, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct) bool isColdStorage, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct)
{ {
try try
@@ -180,53 +187,49 @@ public class MainFileCleanupService : IHostedService
// clean up files in DB but not on disk or last access is expired // clean up files in DB but not on disk or last access is expired
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention)); var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention));
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours)); var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
List<string> deletedFiles = new();
var files = dbContext.Files.OrderBy(f => f.Hash);
List<FileCache> allDbFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false); List<FileCache> allDbFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false);
List<string> removedFileHashes = new List<string>();
int fileCounter = 0; int fileCounter = 0;
foreach (var fileCache in allDbFiles.Where(f => f.Uploaded)) foreach (var fileCache in allDbFiles.Where(f => f.Uploaded))
{ {
bool fileDeleted = false; bool deleteCurrentFile = false;
var file = FilePathUtil.GetFileInfoForHash(dir, fileCache.Hash);
var file = FilePathUtil.GetFileInfoForHash(cacheDir, fileCache.Hash);
if (file == null) if (file == null)
{ {
_logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash); _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash);
fileDeleted = true; deleteCurrentFile = true;
if (deleteFromDb)
dbContext.Files.Remove(fileCache);
} }
else if (file != null && file.LastAccessTime < prevTime) else if (file != null && file.LastAccessTime < prevTime)
{ {
_metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, file.Length);
_metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal);
_logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
file.Delete(); deleteCurrentFile = true;
deletedFiles.Add(file.FullName);
fileDeleted = true;
if (deleteFromDb)
dbContext.Files.Remove(fileCache);
} }
else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion) else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
{ {
_metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, file.Length);
_metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal);
_logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
file.Delete(); deleteCurrentFile = true;
fileDeleted = true; }
deletedFiles.Add(file.FullName);
// do actual deletion of file and remove also from db if needed
if (deleteCurrentFile)
{
if (file != null) file.Delete();
removedFileHashes.Add(fileCache.Hash);
if (deleteFromDb) if (deleteFromDb)
dbContext.Files.Remove(fileCache); dbContext.Files.Remove(fileCache);
} }
if (!fileDeleted && file != null && fileCache.Size == 0) // only used if file in db has no size for whatever reason
if (!deleteCurrentFile && file != null && fileCache.Size == 0)
{ {
_logger.LogInformation("Setting File Size of " + fileCache.Hash + " to " + file.Length); _logger.LogInformation("Setting File Size of " + fileCache.Hash + " to " + file.Length);
fileCache.Size = file.Length; fileCache.Size = file.Length;
// commit every 1000 files to db // commit every 1000 files to db
if (fileCounter % 1000 == 0) await dbContext.SaveChangesAsync().ConfigureAwait(false); if (fileCounter % 1000 == 0)
await dbContext.SaveChangesAsync().ConfigureAwait(false);
} }
fileCounter++; fileCounter++;
@@ -235,7 +238,7 @@ public class MainFileCleanupService : IHostedService
} }
// clean up files that are on disk but not in DB for some reason // clean up files that are on disk but not in DB for some reason
return CleanUpOrphanedFiles(allDbFiles, allFilesInDir.Where(f => !deletedFiles.Contains(f.FullName)).ToList(), ct); return CleanUpOrphanedFiles(allDbFiles, allFilesInDir.Where(c => !removedFileHashes.Contains(c.Name, StringComparer.OrdinalIgnoreCase)).ToList(), ct);
} }
catch (Exception ex) catch (Exception ex)
{ {