t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move GetFileSize to database

Browse Source
This commit is contained in:
rootdarkarchon
2022-12-28 04:50:37 +01:00
parent de2defe692
commit 31550b0aeb
8 changed files with 572 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
MareSynchronosServer/MareSynchronosStaticFilesServer/FileCleanupService.cs
View File

@@ -3,7 +3,7 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Models;
using MareSynchronosShared.Services;
using Microsoft.Extensions.Options;
using Microsoft.EntityFrameworkCore;
namespace MareSynchronosStaticFilesServer;
@@ -52,7 +52,7 @@ public class FileCleanupService : IHostedService
using var scope = _services.CreateScope();
using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
CleanUpOutdatedFiles(dbContext, ct);
await CleanUpOutdatedFiles(dbContext, ct).ConfigureAwait(false);
CleanUpFilesBeyondSizeLimit(dbContext, ct);
@@ -109,7 +109,7 @@ public class FileCleanupService : IHostedService
}
}
private void CleanUpOutdatedFiles(MareDbContext dbContext, CancellationToken ct)
private async Task CleanUpOutdatedFiles(MareDbContext dbContext, CancellationToken ct)
{
try
{
@@ -125,14 +125,17 @@ public class FileCleanupService : IHostedService
// clean up files in DB but not on disk or last access is expired
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention));
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
var allFiles = dbContext.Files.ToList();
var allFiles = await dbContext.Files.ToListAsync().ConfigureAwait(false);
int fileCounter = 0;
foreach (var fileCache in allFiles.Where(f => f.Uploaded))
{
var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash);
bool fileDeleted = false;
if (file == null && _isMainServer)
{
_logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash);
dbContext.Files.Remove(fileCache);
fileDeleted = true;
}
else if (file != null && file.LastAccessTime < prevTime)
{
@@ -141,7 +144,10 @@ public class FileCleanupService : IHostedService
_logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
file.Delete();
if (_isMainServer)
{
fileDeleted = true;
dbContext.Files.Remove(fileCache);
}
}
else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
{
@@ -150,9 +156,21 @@ public class FileCleanupService : IHostedService
_logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
file.Delete();
if (_isMainServer)
{
fileDeleted = true;
dbContext.Files.Remove(fileCache);
}
}
if (_isMainServer && !fileDeleted && file != null && fileCache.Size == 0)
{
fileCache.Size = file.Length;
// commit every 1000 files to db
if (fileCounter % 1000 == 0) await dbContext.SaveChangesAsync().ConfigureAwait(false);
}
fileCounter++;
ct.ThrowIfCancellationRequested();
}

13
MareSynchronosServer/MareSynchronosStaticFilesServer/GrpcFileService.cs
View File

@@ -46,6 +46,7 @@ public class GrpcFileService : FileService.FileServiceBase
var fileSize = new FileInfo(filePath).Length;
file.Uploaded = true;
file.Size = fileSize;
await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
@@ -96,16 +97,4 @@ public class GrpcFileService : FileService.FileServiceBase
await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
return new Empty();
}
public override Task<FileSizeResponse> GetFileSizes(FileSizeRequest request, ServerCallContext context)
{
FileSizeResponse response = new();
foreach (var hash in request.Hash.Distinct(StringComparer.Ordinal))
{
FileInfo? fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
response.HashToFileSize.Add(hash, fi?.Length ?? 0);
}
return Task.FromResult(response);
}
}