t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add central cold storage options for file servers

Browse Source 
add access times I guess

access, not write

overwrite times after copying
This commit is contained in:
Stanley Dimant
2024-05-02 14:03:42 +02:00
committed by Loporrit
parent 180d7d504d
commit fb22a267ac
6 changed files with 151 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs
View File

@@ -13,6 +13,7 @@ using MareSynchronosStaticFilesServer.Utils;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.SignalR;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using System.Collections.Concurrent;
using System.Security.Cryptography;
using System.Security.Policy;
@@ -38,7 +39,9 @@ public class ServerFilesController : ControllerBase
IHubContext<MareHub> hubContext,
MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger)
{
_basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
_basePath = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false)
? configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory))
: configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
_cachedFileProvider = cachedFileProvider;
_configuration = configuration;
_hubContext = hubContext;
@@ -50,14 +53,15 @@ public class ServerFilesController : ControllerBase
public async Task<IActionResult> FilesDeleteAll()
{
var ownFiles = await _mareDbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == MareUser).ToListAsync().ConfigureAwait(false);
bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
foreach (var dbFile in ownFiles)
{
var fi = FilePathUtil.GetFileInfoForHash(_basePath, dbFile.Hash);
if (fi != null)
{
_metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1);
_metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0);
_metricsClient.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1);
_metricsClient.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0);
fi?.Delete();
}
@@ -262,8 +266,10 @@ public class ServerFilesController : ControllerBase
}).ConfigureAwait(false);
await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
_metricsClient.IncGauge(MetricsAPI.GaugeFilesTotal, 1);
_metricsClient.IncGauge(MetricsAPI.GaugeFilesTotalSize, compressedSize);
bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
_metricsClient.IncGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal, 1);
_metricsClient.IncGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, compressedSize);
_fileUploadLocks.TryRemove(hash, out _);

68
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs
View File

@@ -10,12 +10,13 @@ namespace MareSynchronosStaticFilesServer.Services;
public sealed class CachedFileProvider : IDisposable
{
private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
private readonly ILogger<CachedFileProvider> _logger;
private readonly FileStatisticsService _fileStatisticsService;
private readonly MareMetrics _metrics;
private readonly ServerTokenGenerator _generator;
private readonly Uri _remoteCacheSourceUri;
private readonly string _basePath;
private readonly string _hotStoragePath;
private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
private readonly HttpClient _httpClient;
private readonly SemaphoreSlim _downloadSemaphore = new(1, 1);
@@ -24,15 +25,17 @@ public sealed class CachedFileProvider : IDisposable
private bool IsMainServer => _remoteCacheSourceUri == null && _isDistributionServer;
private bool _isDistributionServer;
public CachedFileProvider(IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics, ServerTokenGenerator generator)
public CachedFileProvider(IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger,
FileStatisticsService fileStatisticsService, MareMetrics metrics, ServerTokenGenerator generator)
{
_configuration = configuration;
_logger = logger;
_fileStatisticsService = fileStatisticsService;
_metrics = metrics;
_generator = generator;
_remoteCacheSourceUri = configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.DistributionFileServerAddress), null);
_isDistributionServer = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false);
_basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
_hotStoragePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
_httpClient = new();
_httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer", "1.0.0.0"));
}
@@ -50,7 +53,12 @@ public sealed class CachedFileProvider : IDisposable
private async Task DownloadTask(string hash)
{
// download file from remote
var destinationFilePath = FilePathUtil.GetFilePath(_hotStoragePath, hash);
// first check cold storage
if (TryCopyFromColdStorage(hash, destinationFilePath)) return;
// if cold storage is not configured or file not found or error is present try to download file from remote
var downloadUrl = MareFiles.DistributionGetFullPath(_remoteCacheSourceUri, hash);
_logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl);
@@ -70,8 +78,7 @@ public sealed class CachedFileProvider : IDisposable
return;
}
var fileName = FilePathUtil.GetFilePath(_basePath, hash);
var tempFileName = fileName + ".dl";
var tempFileName = destinationFilePath + ".dl";
var fileStream = new FileStream(tempFileName, FileMode.Create, FileAccess.ReadWrite);
var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024;
var buffer = new byte[bufferSize];
@@ -84,17 +91,54 @@ public sealed class CachedFileProvider : IDisposable
}
await fileStream.FlushAsync().ConfigureAwait(false);
await fileStream.DisposeAsync().ConfigureAwait(false);
File.Move(tempFileName, fileName, true);
File.Move(tempFileName, destinationFilePath, true);
_metrics.IncGauge(MetricsAPI.GaugeFilesTotal);
_metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_basePath, hash).Length);
_metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash).Length);
response.Dispose();
}
private bool TryCopyFromColdStorage(string hash, string destinationFilePath)
{
if (!_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false)) return false;
string coldStorageDir = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageDirectory), string.Empty);
if (string.IsNullOrEmpty(coldStorageDir)) return false;
var coldStorageFilePath = FilePathUtil.GetFileInfoForHash(coldStorageDir, hash);
if (coldStorageFilePath == null) return false;
try
{
_logger.LogDebug("Copying {hash} from cold storage: {path}", hash, coldStorageFilePath);
var tempFileName = destinationFilePath + ".dl";
File.Copy(coldStorageFilePath.FullName, tempFileName, true);
File.Move(tempFileName, destinationFilePath, true);
coldStorageFilePath.LastAccessTimeUtc = DateTime.UtcNow;
var destinationFile = new FileInfo(destinationFilePath);
destinationFile.LastAccessTimeUtc = DateTime.UtcNow;
destinationFile.CreationTimeUtc = DateTime.UtcNow;
destinationFile.LastWriteTimeUtc = DateTime.UtcNow;
_metrics.IncGauge(MetricsAPI.GaugeFilesTotal);
_metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, new FileInfo(destinationFilePath).Length);
return true;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Could not copy {coldStoragePath} from cold storage", coldStorageFilePath);
}
return false;
}
public async Task DownloadFileWhenRequired(string hash)
{
var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
if (fi == null && IsMainServer) return;
var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash);
if (fi == null && IsMainServer)
{
TryCopyFromColdStorage(hash, FilePathUtil.GetFilePath(_hotStoragePath, hash));
return;
}
await _downloadSemaphore.WaitAsync().ConfigureAwait(false);
if ((fi == null || (fi?.Length ?? 0) == 0)
@@ -124,9 +168,11 @@ public sealed class CachedFileProvider : IDisposable
public FileStream? GetLocalFileStream(string hash)
{
var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash);
if (fi == null) return null;
fi.LastAccessTimeUtc = DateTime.UtcNow;
_fileStatisticsService.LogFile(hash, fi.Length);
return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read);

106
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs
View File

@@ -10,7 +10,6 @@ namespace MareSynchronosStaticFilesServer.Services;
public class MainFileCleanupService : IHostedService
{
private readonly string _cacheDir;
private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
private readonly ILogger<MainFileCleanupService> _logger;
private readonly MareMetrics _metrics;
@@ -24,7 +23,6 @@ public class MainFileCleanupService : IHostedService
_logger = logger;
_services = services;
_configuration = configuration;
_cacheDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
}
public async Task CleanUpTask(CancellationToken ct)
@@ -35,17 +33,48 @@ public class MainFileCleanupService : IHostedService
{
try
{
DirectoryInfo dir = new(_cacheDir);
var allFiles = dir.GetFiles("*", SearchOption.AllDirectories);
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFiles.Sum(f => f.Length));
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFiles.Length);
var hotStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
DirectoryInfo dir = new(hotStorageDir);
var allFilesInHotStorage = dir.GetFiles("*", SearchOption.AllDirectories).ToList();
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFilesInHotStorage.Sum(f => f.Length));
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFilesInHotStorage.Count);
using var scope = _services.CreateScope();
using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
await CleanUpOutdatedFiles(dbContext, ct).ConfigureAwait(false);
bool useColdStorage = _configuration.GetValueOrDefault<bool>(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
CleanUpFilesBeyondSizeLimit(dbContext, ct);
if (useColdStorage)
{
var coldStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
var coldStorageRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60);
var coldStorageSize = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1);
DirectoryInfo dirColdStorage = new(coldStorageDir);
var allFilesInColdStorageDir = dirColdStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, allFilesInColdStorageDir.Sum(f => f.Length));
_metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, allFilesInColdStorageDir.Count);
// clean up cold storage
var remainingColdFiles = await CleanUpOutdatedFiles(coldStorageDir, allFilesInColdStorageDir, coldStorageRetention, forcedDeletionAfterHours: -1,
isColdStorage: true, deleteFromDb: true,
dbContext, ct).ConfigureAwait(false);
CleanUpFilesBeyondSizeLimit(remainingColdFiles, coldStorageSize,
isColdStorage: true, deleteFromDb: true,
dbContext, ct);
}
var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);
var remainingHotFiles = await CleanUpOutdatedFiles(hotStorageDir, allFilesInHotStorage, unusedRetention, forcedDeletionAfterHours,
isColdStorage: false, deleteFromDb: !useColdStorage,
dbContext, ct).ConfigureAwait(false);
CleanUpFilesBeyondSizeLimit(remainingHotFiles, sizeLimit,
isColdStorage: false, deleteFromDb: !useColdStorage,
dbContext, ct);
CleanUpStuckUploads(dbContext);
@@ -84,9 +113,8 @@ public class MainFileCleanupService : IHostedService
return Task.CompletedTask;
}
private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct)
private void CleanUpFilesBeyondSizeLimit(List<FileInfo> files, double sizeLimit, bool isColdStorage, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct)
{
var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);
if (sizeLimit <= 0)
{
return;
@@ -95,8 +123,7 @@ public class MainFileCleanupService : IHostedService
try
{
_logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", sizeLimit);
var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories)
.Select(f => new FileInfo(f)).ToList()
var allLocalFiles = files
.OrderBy(f => f.LastAccessTimeUtc).ToList();
var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes;
@@ -105,12 +132,13 @@ public class MainFileCleanupService : IHostedService
var oldestFile = allLocalFiles[0];
allLocalFiles.Remove(oldestFile);
totalCacheSizeInBytes -= oldestFile.Length;
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, oldestFile.Length);
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
_metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, oldestFile.Length);
_metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal);
_logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes);
oldestFile.Delete();
FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() };
dbContext.Entry(f).State = EntityState.Deleted;
if (deleteFromDb)
dbContext.Entry(f).State = EntityState.Deleted;
}
}
catch (Exception ex)
@@ -119,10 +147,10 @@ public class MainFileCleanupService : IHostedService
}
}
private void CleanUpOrphanedFiles(List<FileCache> allFiles, FileInfo[] allPhysicalFiles, CancellationToken ct)
private List<FileInfo> CleanUpOrphanedFiles(List<FileCache> allFiles, List<FileInfo> allPhysicalFiles, CancellationToken ct)
{
var allFilesHashes = new HashSet<string>(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal);
foreach (var file in allPhysicalFiles)
foreach (var file in allPhysicalFiles.ToList())
{
if (!allFilesHashes.Contains(file.Name.ToUpperInvariant()))
{
@@ -134,15 +162,15 @@ public class MainFileCleanupService : IHostedService
ct.ThrowIfCancellationRequested();
}
return allPhysicalFiles;
}
private async Task CleanUpOutdatedFiles(MareDbContext dbContext, CancellationToken ct)
private async Task<List<FileInfo>> CleanUpOutdatedFiles(string cacheDir, List<FileInfo> allFilesInDir, int unusedRetention, int forcedDeletionAfterHours,
bool isColdStorage, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct)
{
try
{
var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
_logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", unusedRetention);
if (forcedDeletionAfterHours > 0)
{
@@ -152,39 +180,45 @@ public class MainFileCleanupService : IHostedService
// clean up files in DB but not on disk or last access is expired
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention));
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
DirectoryInfo dir = new(_cacheDir);
var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories);
List<string> deletedFiles = new();
var files = dbContext.Files.OrderBy(f => f.Hash);
List<FileCache> allFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false);
List<FileCache> allDbFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false);
int fileCounter = 0;
foreach (var fileCache in allFiles.Where(f => f.Uploaded))
foreach (var fileCache in allDbFiles.Where(f => f.Uploaded))
{
bool fileDeleted = false;
var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash);
var file = FilePathUtil.GetFileInfoForHash(cacheDir, fileCache.Hash);
if (file == null)
{
_logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash);
dbContext.Files.Remove(fileCache);
fileDeleted = true;
if (deleteFromDb)
dbContext.Files.Remove(fileCache);
}
else if (file != null && file.LastAccessTime < prevTime)
{
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
_metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, file.Length);
_metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal);
_logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
file.Delete();
deletedFiles.Add(file.FullName);
fileDeleted = true;
dbContext.Files.Remove(fileCache);
if (deleteFromDb)
dbContext.Files.Remove(fileCache);
}
else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
{
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
_metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, file.Length);
_metrics.DecGauge(isColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal);
_logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
file.Delete();
fileDeleted = true;
dbContext.Files.Remove(fileCache);
deletedFiles.Add(file.FullName);
if (deleteFromDb)
dbContext.Files.Remove(fileCache);
}
if (!fileDeleted && file != null && fileCache.Size == 0)
@@ -201,17 +235,19 @@ public class MainFileCleanupService : IHostedService
}
// clean up files that are on disk but not in DB for some reason
CleanUpOrphanedFiles(allFiles, allFilesInDir, ct);
return CleanUpOrphanedFiles(allDbFiles, allFilesInDir.Where(f => !deletedFiles.Contains(f.FullName)).ToList(), ct);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error during file cleanup of old files");
}
return [];
}
private void CleanUpStuckUploads(MareDbContext dbContext)
{
var pastTime = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10));
var pastTime = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(20));
var stuckUploads = dbContext.Files.Where(f => !f.Uploaded && f.UploadDate < pastTime);
dbContext.Files.RemoveRange(stuckUploads);
}

2
MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs
View File

@@ -60,6 +60,8 @@ public class Startup
MetricsAPI.CounterFileRequestSize
}, new List<string>
{
MetricsAPI.GaugeFilesTotalColdStorage,
MetricsAPI.GaugeFilesTotalSizeColdStorage,
MetricsAPI.GaugeFilesTotalSize,
MetricsAPI.GaugeFilesTotal,
MetricsAPI.GaugeFilesUniquePastDay,