diff --git a/MareAPI b/MareAPI index cd8934a..b529a10 160000 --- a/MareAPI +++ b/MareAPI @@ -1 +1 @@ -Subproject commit cd8934a4ab37a3549bacf7e7108f83a34403da96 +Subproject commit b529a101ae75bf9f20ae3b47d964e0a69b063754 diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs b/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs index 658af57..6786d2d 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs @@ -5,6 +5,7 @@ namespace MareSynchronosStaticFilesServer; public class StaticFilesServerConfiguration : MareConfigurationBase { + public bool IsDistributionNode { get; set; } = false; public Uri? MainFileServerAddress { get; set; } = null; public int ForcedDeletionOfFilesAfterHours { get; set; } = -1; public double CacheSizeHardLimitInGiB { get; set; } = -1; diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/DistributionController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/DistributionController.cs new file mode 100644 index 0000000..6df7cc3 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/DistributionController.cs @@ -0,0 +1,29 @@ +using MareSynchronos.API.Routes; +using MareSynchronosStaticFilesServer.Services; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace MareSynchronosStaticFilesServer.Controllers; + +[Route(MareFiles.Distribution)] +public class DistributionController : ControllerBase +{ + private readonly CachedFileProvider _cachedFileProvider; + + public DistributionController(ILogger logger, CachedFileProvider cachedFileProvider) : base(logger) + { + _cachedFileProvider = cachedFileProvider; + } + + [HttpGet(MareFiles.Distribution_Get)] + [Authorize(Policy = "Internal")] + public async Task GetFile(string file) + { + _logger.LogInformation($"GetFile:{MareUser}:{file}"); + + var fs = await _cachedFileProvider.GetAndDownloadFileStream(file); + if (fs == null) return NotFound(); + + return File(fs, "application/octet-stream"); + } +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/MainController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/MainController.cs new file mode 100644 index 0000000..7e43ba0 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/MainController.cs @@ -0,0 +1,25 @@ +using MareSynchronos.API.Routes; +using MareSynchronosStaticFilesServer.Services; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace MareSynchronosStaticFilesServer.Controllers; + +[Route(MareFiles.Main)] +public class MainController : ControllerBase +{ + private readonly IClientReadyMessageService _messageService; + + public MainController(ILogger logger, IClientReadyMessageService mareHub) : base(logger) + { + _messageService = mareHub; + } + + [HttpGet(MareFiles.Main_SendReady)] + [Authorize(Policy = "Internal")] + public IActionResult SendReadyToClients(string uid, Guid requestId) + { + _messageService.SendDownloadReady(uid, requestId); + return Ok(); + } +} \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs index 66ce9f8..64592a8 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs @@ -10,7 +10,6 @@ using MareSynchronosShared.Services; using MareSynchronosShared.Utils; using MareSynchronosStaticFilesServer.Services; using MareSynchronosStaticFilesServer.Utils; -using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; @@ -165,18 +164,6 @@ public class ServerFilesController : ControllerBase return Ok(JsonSerializer.Serialize(notCoveredFiles.Values.ToList())); } - [HttpGet(MareFiles.ServerFiles_Get + "/{fileId}")] - [Authorize(Policy = "Internal")] - public IActionResult GetFile(string fileId) - { - _logger.LogInformation($"GetFile:{MareUser}:{fileId}"); - - var fs = _cachedFileProvider.GetLocalFileStream(fileId); - if (fs == null) return NotFound(); - - return File(fs, "application/octet-stream"); - } - [HttpPost(MareFiles.ServerFiles_Upload + "/{hash}")] [RequestSizeLimit(200 * 1024 * 1024)] public async Task UploadFile(string hash, CancellationToken requestAborted) diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs index 6f8fdc0..4fec591 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs @@ -21,7 +21,8 @@ public sealed class CachedFileProvider : IDisposable private readonly SemaphoreSlim _downloadSemaphore = new(1); private bool _disposed; - private bool IsMainServer => _remoteCacheSourceUri == null; + private bool IsMainServer => _remoteCacheSourceUri == null && _isDistributionServer; + private bool _isDistributionServer; public CachedFileProvider(IConfigurationService configuration, ILogger logger, FileStatisticsService fileStatisticsService, MareMetrics metrics, ServerTokenGenerator generator) { @@ -30,8 +31,10 @@ public sealed class CachedFileProvider : IDisposable _metrics = metrics; _generator = generator; _remoteCacheSourceUri = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.MainFileServerAddress), null); + _isDistributionServer = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false); _basePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); _httpClient = new(); + _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer")); } public void Dispose() @@ -48,7 +51,7 @@ public sealed class CachedFileProvider : IDisposable private async Task DownloadTask(string hash) { // download file from remote - var downloadUrl = MareFiles.ServerFilesGetFullPath(_remoteCacheSourceUri, hash); + var downloadUrl = MareFiles.DistributionGetFullPath(_remoteCacheSourceUri, hash); _logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl); using var requestMessage = new HttpRequestMessage(HttpMethod.Get, downloadUrl); @@ -129,12 +132,4 @@ public sealed class CachedFileProvider : IDisposable return GetLocalFileStream(hash); } - - private void ThrowIfDisposed() - { - if (_disposed) - { - throw new ObjectDisposedException(GetType().FullName); - } - } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/IClientReadyMessageService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/IClientReadyMessageService.cs new file mode 100644 index 0000000..932cf51 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/IClientReadyMessageService.cs @@ -0,0 +1,6 @@ +namespace MareSynchronosStaticFilesServer.Services; + +public interface IClientReadyMessageService +{ + void SendDownloadReady(string uid, Guid requestId); +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainClientReadyMessageService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainClientReadyMessageService.cs new file mode 100644 index 0000000..cb8c98f --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainClientReadyMessageService.cs @@ -0,0 +1,26 @@ +using Microsoft.AspNetCore.SignalR; +using MareSynchronos.API.SignalR; +using MareSynchronosServer.Hubs; + +namespace MareSynchronosStaticFilesServer.Services; + +public class MainClientReadyMessageService : IClientReadyMessageService +{ + private readonly ILogger _logger; + private readonly IHubContext _mareHub; + + public MainClientReadyMessageService(ILogger logger, IHubContext mareHub) + { + _logger = logger; + _mareHub = mareHub; + } + + public void SendDownloadReady(string uid, Guid requestId) + { + _ = Task.Run(async () => + { + _logger.LogInformation("Sending Client Ready for {uid}:{requestId} to SignalR", uid, requestId); + await _mareHub.Clients.User(uid).SendAsync(nameof(IMareHub.Client_DownloadReady), requestId).ConfigureAwait(false); + }); + } +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs similarity index 79% rename from MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs rename to MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs index c15f216..6f87fa3 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs @@ -8,23 +8,21 @@ using Microsoft.EntityFrameworkCore; namespace MareSynchronosStaticFilesServer.Services; -public class FileCleanupService : IHostedService +public class MainFileCleanupService : IHostedService { private readonly string _cacheDir; private readonly IConfigurationService _configuration; - private readonly bool _isMainServer; - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly MareMetrics _metrics; private readonly IServiceProvider _services; private CancellationTokenSource _cleanupCts; - public FileCleanupService(MareMetrics metrics, ILogger logger, IServiceProvider services, IConfigurationService configuration) + public MainFileCleanupService(MareMetrics metrics, ILogger logger, IServiceProvider services, IConfigurationService configuration) { _metrics = metrics; _logger = logger; _services = services; _configuration = configuration; - _isMainServer = configuration.IsMain; _cacheDir = _configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); } @@ -48,12 +46,9 @@ public class FileCleanupService : IHostedService CleanUpFilesBeyondSizeLimit(dbContext, ct); - if (_isMainServer) - { - CleanUpStuckUploads(dbContext); + CleanUpStuckUploads(dbContext); - await dbContext.SaveChangesAsync(ct).ConfigureAwait(false); - } + await dbContext.SaveChangesAsync(ct).ConfigureAwait(false); } catch (Exception e) { @@ -113,11 +108,8 @@ public class FileCleanupService : IHostedService _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); _logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes); oldestFile.Delete(); - if (_isMainServer) - { - FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() }; - dbContext.Entry(f).State = EntityState.Deleted; - } + FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() }; + dbContext.Entry(f).State = EntityState.Deleted; } } catch (Exception ex) @@ -128,21 +120,18 @@ public class FileCleanupService : IHostedService private void CleanUpOrphanedFiles(List allFiles, FileInfo[] allPhysicalFiles, CancellationToken ct) { - if (_isMainServer) + var allFilesHashes = new HashSet(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal); + foreach (var file in allPhysicalFiles) { - var allFilesHashes = new HashSet(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal); - foreach (var file in allPhysicalFiles) + if (!allFilesHashes.Contains(file.Name.ToUpperInvariant())) { - if (!allFilesHashes.Contains(file.Name.ToUpperInvariant())) - { - _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); - _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); - file.Delete(); - _logger.LogInformation("File not in DB, deleting: {fileName}", file.Name); - } - - ct.ThrowIfCancellationRequested(); + _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); + _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); + file.Delete(); + _logger.LogInformation("File not in DB, deleting: {fileName}", file.Name); } + + ct.ThrowIfCancellationRequested(); } } @@ -164,13 +153,15 @@ public class FileCleanupService : IHostedService var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours)); DirectoryInfo dir = new(_cacheDir); var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories); - var allFiles = await dbContext.Files.ToListAsync().ConfigureAwait(false); + var files = dbContext.Files.OrderBy(f => f.Hash); + List allFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false); int fileCounter = 0; foreach (var fileCache in allFiles.Where(f => f.Uploaded)) { - var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash); bool fileDeleted = false; - if (file == null && _isMainServer) + + var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash); + if (file == null) { _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash); dbContext.Files.Remove(fileCache); @@ -182,11 +173,8 @@ public class FileCleanupService : IHostedService _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); file.Delete(); - if (_isMainServer) - { - fileDeleted = true; - dbContext.Files.Remove(fileCache); - } + fileDeleted = true; + dbContext.Files.Remove(fileCache); } else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion) { @@ -194,14 +182,11 @@ public class FileCleanupService : IHostedService _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); file.Delete(); - if (_isMainServer) - { - fileDeleted = true; - dbContext.Files.Remove(fileCache); - } + fileDeleted = true; + dbContext.Files.Remove(fileCache); } - if (_isMainServer && !fileDeleted && file != null && fileCache.Size == 0) + if (!fileDeleted && file != null && fileCache.Size == 0) { _logger.LogInformation("Setting File Size of " + fileCache.Hash + " to " + file.Length); fileCache.Size = file.Length; diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs index 690a3de..f7911d6 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs @@ -1,32 +1,27 @@ using MareSynchronosShared.Metrics; using MareSynchronosShared.Services; using MareSynchronosStaticFilesServer.Utils; -using Microsoft.AspNetCore.SignalR; using System.Collections.Concurrent; using System.Timers; -using MareSynchronos.API.SignalR; namespace MareSynchronosStaticFilesServer.Services; public class RequestQueueService : IHostedService { - private record PriorityEntry(bool IsHighPriority, DateTime LastChecked); - - private readonly IHubContext _hubContext; + private readonly IClientReadyMessageService _clientReadyMessageService; private readonly ILogger _logger; private readonly MareMetrics _metrics; private readonly ConcurrentQueue _queue = new(); private readonly ConcurrentQueue _priorityQueue = new(); private readonly int _queueExpirationSeconds; private readonly SemaphoreSlim _queueProcessingSemaphore = new(1); - private readonly SemaphoreSlim _queueSemaphore = new(1); private readonly UserQueueEntry[] _userQueueRequests; private int _queueLimitForReset; private readonly int _queueReleaseSeconds; private System.Timers.Timer _queueTimer; public RequestQueueService(MareMetrics metrics, IConfigurationService configurationService, - ILogger logger, IHubContext hubContext) + ILogger logger, IClientReadyMessageService hubContext) { _userQueueRequests = new UserQueueEntry[configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueSize), 50)]; _queueExpirationSeconds = configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadTimeoutSeconds), 5); @@ -34,7 +29,7 @@ public class RequestQueueService : IHostedService _queueReleaseSeconds = configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueReleaseSeconds), 15); _metrics = metrics; _logger = logger; - _hubContext = hubContext; + _clientReadyMessageService = hubContext; } public void ActivateRequest(Guid request) @@ -125,7 +120,7 @@ public class RequestQueueService : IHostedService { _logger.LogDebug("Dequeueing {req} into {i}: {user} with {file}", userRequest.RequestId, slot, userRequest.User, string.Join(", ", userRequest.FileIds)); _userQueueRequests[slot] = new(userRequest, DateTime.UtcNow.AddSeconds(_queueExpirationSeconds)); - _ = _hubContext.Clients.User(userRequest.User).SendAsync(nameof(IMareHub.Client_DownloadReady), userRequest.RequestId).ConfigureAwait(false); + _clientReadyMessageService.SendDownloadReady(userRequest.User, userRequest.RequestId); } private void ProcessQueue(object src, ElapsedEventArgs e) diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardClientReadyMessageService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardClientReadyMessageService.cs new file mode 100644 index 0000000..9fded2a --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardClientReadyMessageService.cs @@ -0,0 +1,47 @@ +using MareSynchronos.API.Routes; +using MareSynchronosShared.Services; +using MareSynchronosShared.Utils; +using System.Net.Http.Headers; + +namespace MareSynchronosStaticFilesServer.Services; + +public class ShardClientReadyMessageService : IClientReadyMessageService +{ + private readonly ILogger _logger; + private readonly ServerTokenGenerator _tokenGenerator; + private readonly IConfigurationService _configurationService; + private readonly HttpClient _httpClient; + + public ShardClientReadyMessageService(ILogger logger, ServerTokenGenerator tokenGenerator, IConfigurationService configurationService) + { + _logger = logger; + _tokenGenerator = tokenGenerator; + _configurationService = configurationService; + _httpClient = new(); + _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer")); + } + + public void SendDownloadReady(string uid, Guid requestId) + { + var mainUrl = _configurationService.GetValue(nameof(StaticFilesServerConfiguration.MainFileServerAddress)); + var path = MareFiles.MainSendReadyFullPath(mainUrl, uid, requestId); + using HttpRequestMessage msg = new() + { + RequestUri = path + }; + msg.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _tokenGenerator.Token); + + _ = Task.Run(async () => + { + _logger.LogInformation("Sending Client Ready for {uid}:{requestId} to {path}", uid, requestId, path); + try + { + using var result = await _httpClient.SendAsync(msg).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failure to send for {uid}:{requestId}", uid, requestId); + } + }); + } +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardFileCleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardFileCleanupService.cs new file mode 100644 index 0000000..cb26ddb --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardFileCleanupService.cs @@ -0,0 +1,153 @@ +using ByteSizeLib; +using MareSynchronosShared.Metrics; +using MareSynchronosShared.Services; +using Microsoft.EntityFrameworkCore; + +namespace MareSynchronosStaticFilesServer.Services; + +public class ShardFileCleanupService : IHostedService +{ + private readonly string _cacheDir; + private readonly IConfigurationService _configuration; + private readonly ILogger _logger; + private readonly MareMetrics _metrics; + private CancellationTokenSource _cleanupCts; + + public ShardFileCleanupService(MareMetrics metrics, ILogger logger, IConfigurationService configuration) + { + _metrics = metrics; + _logger = logger; + _configuration = configuration; + _cacheDir = _configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); + } + + public async Task CleanUpTask(CancellationToken ct) + { + _logger.LogInformation("Starting periodic cleanup task"); + + while (!ct.IsCancellationRequested) + { + try + { + DirectoryInfo dir = new(_cacheDir); + var allFiles = dir.GetFiles("*", SearchOption.AllDirectories); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFiles.Sum(f => f.Length)); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFiles.Length); + + CleanUpOutdatedFiles(ct); + + CleanUpFilesBeyondSizeLimit(ct); + } + catch (Exception e) + { + _logger.LogError(e, "Error during cleanup task"); + } + + var now = DateTime.Now; + TimeOnly currentTime = new(now.Hour, now.Minute, now.Second); + TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % 15, 0); + var span = futureTime.AddMinutes(15) - currentTime; + + _logger.LogInformation("File Cleanup Complete, next run at {date}", now.Add(span)); + await Task.Delay(span, ct).ConfigureAwait(false); + } + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Cleanup Service started"); + + _cleanupCts = new(); + + _ = CleanUpTask(_cleanupCts.Token); + + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _cleanupCts.Cancel(); + + return Task.CompletedTask; + } + + private void CleanUpFilesBeyondSizeLimit(CancellationToken ct) + { + var sizeLimit = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1); + if (sizeLimit <= 0) + { + return; + } + + try + { + _logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", sizeLimit); + var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories) + .Select(f => new FileInfo(f)).ToList() + .OrderBy(f => f.LastAccessTimeUtc).ToList(); + var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length); + long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes; + while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested) + { + var oldestFile = allLocalFiles[0]; + allLocalFiles.Remove(oldestFile); + totalCacheSizeInBytes -= oldestFile.Length; + _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, oldestFile.Length); + _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); + _logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes); + oldestFile.Delete(); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error during cache size limit cleanup"); + } + } + + private void CleanUpOutdatedFiles(CancellationToken ct) + { + try + { + var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14); + var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1); + + _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", unusedRetention); + if (forcedDeletionAfterHours > 0) + { + _logger.LogInformation("Cleaning up files written to longer than {hours}h ago", forcedDeletionAfterHours); + } + + var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention)); + var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours)); + DirectoryInfo dir = new(_cacheDir); + var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories); + int fileCounter = 0; + + foreach (var file in allFilesInDir) + { + if (file.LastAccessTime < prevTime) + { + _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); + _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); + _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); + file.Delete(); + } + else if (forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion) + { + _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); + _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); + _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); + file.Delete(); + } + + fileCounter++; + + ct.ThrowIfCancellationRequested(); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error during file cleanup of old files"); + } + } +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs index f13d44e..c915a6a 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs @@ -26,6 +26,7 @@ namespace MareSynchronosStaticFilesServer; public class Startup { private bool _isMain; + private bool _isDistributionNode; private readonly ILogger _logger; public Startup(IConfiguration configuration, ILogger logger) @@ -33,7 +34,8 @@ public class Startup Configuration = configuration; _logger = logger; var mareSettings = Configuration.GetRequiredSection("MareSynchronos"); - _isMain = string.IsNullOrEmpty(mareSettings.GetValue(nameof(StaticFilesServerConfiguration.MainFileServerAddress), string.Empty)); + _isDistributionNode = mareSettings.GetValue(nameof(StaticFilesServerConfiguration.IsDistributionNode), false); + _isMain = string.IsNullOrEmpty(mareSettings.GetValue(nameof(StaticFilesServerConfiguration.MainFileServerAddress), string.Empty)) && _isDistributionNode; } public IConfiguration Configuration { get; } @@ -51,6 +53,7 @@ public class Startup var mareConfig = Configuration.GetRequiredSection("MareSynchronos"); + // metrics configuration services.AddSingleton(m => new MareMetrics(m.GetService>(), new List { MetricsAPI.CounterFileRequests, @@ -72,22 +75,124 @@ public class Startup MetricsAPI.GaugeQueueInactive, MetricsAPI.GaugeQueueActive, })); + + // generic services services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - + services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(p => p.GetService()); services.AddHostedService(m => m.GetService()); - services.AddHostedService(); + services.AddSingleton, MareConfigurationServiceClient>(); + services.AddHostedService(p => (MareConfigurationServiceClient)p.GetService>()); - services.AddDbContextPool(options => + // specific services + if (_isMain) { - options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder => + services.AddSingleton(); + services.AddHostedService(); + services.AddSingleton, MareConfigurationServiceServer>(); + services.AddDbContextPool(options => { - builder.MigrationsHistoryTable("_efmigrationshistory", "public"); - }).UseSnakeCaseNamingConvention(); - options.EnableThreadSafetyChecks(false); - }, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024)); + options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder => + { + builder.MigrationsHistoryTable("_efmigrationshistory", "public"); + }).UseSnakeCaseNamingConvention(); + options.EnableThreadSafetyChecks(false); + }, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024)); + var signalRServiceBuilder = services.AddSignalR(hubOptions => + { + hubOptions.MaximumReceiveMessageSize = long.MaxValue; + hubOptions.EnableDetailedErrors = true; + hubOptions.MaximumParallelInvocationsPerClient = 10; + hubOptions.StreamBufferCapacity = 200; + }).AddMessagePackProtocol(opt => + { + var resolver = CompositeResolver.Create(StandardResolverAllowPrivate.Instance, + BuiltinResolver.Instance, + AttributeFormatterResolver.Instance, + // replace enum resolver + DynamicEnumAsStringResolver.Instance, + DynamicGenericResolver.Instance, + DynamicUnionResolver.Instance, + DynamicObjectResolver.Instance, + PrimitiveObjectResolver.Instance, + // final fallback(last priority) + StandardResolver.Instance); + + opt.SerializerOptions = MessagePackSerializerOptions.Standard + .WithCompression(MessagePackCompression.Lz4Block) + .WithResolver(resolver); + }); + + // configure redis for SignalR + var redisConnection = mareConfig.GetValue(nameof(ServerConfiguration.RedisConnectionString), string.Empty); + signalRServiceBuilder.AddStackExchangeRedis(redisConnection, options => { }); + + var options = ConfigurationOptions.Parse(redisConnection); + + var endpoint = options.EndPoints[0]; + string address = ""; + int port = 0; + if (endpoint is DnsEndPoint dnsEndPoint) { address = dnsEndPoint.Host; port = dnsEndPoint.Port; } + if (endpoint is IPEndPoint ipEndPoint) { address = ipEndPoint.Address.ToString(); port = ipEndPoint.Port; } + var redisConfiguration = new RedisConfiguration() + { + AbortOnConnectFail = true, + KeyPrefix = "", + Hosts = new RedisHost[] + { + new RedisHost(){ Host = address, Port = port }, + }, + AllowAdmin = true, + ConnectTimeout = options.ConnectTimeout, + Database = 0, + Ssl = false, + Password = options.Password, + ServerEnumerationStrategy = new ServerEnumerationStrategy() + { + Mode = ServerEnumerationStrategy.ModeOptions.All, + TargetRole = ServerEnumerationStrategy.TargetRoleOptions.Any, + UnreachableServerAction = ServerEnumerationStrategy.UnreachableServerActionOptions.Throw, + }, + MaxValueLength = 1024, + PoolSize = mareConfig.GetValue(nameof(ServerConfiguration.RedisPool), 50), + SyncTimeout = options.SyncTimeout, + }; + + services.AddStackExchangeRedisExtensions(redisConfiguration); + } + else + { + services.AddSingleton(); + services.AddHostedService(); + services.AddSingleton, MareConfigurationServiceClient>(); + services.AddHostedService(p => (MareConfigurationServiceClient)p.GetService>()); + } + + // controller setup + services.AddControllers().ConfigureApplicationPartManager(a => + { + a.FeatureProviders.Remove(a.FeatureProviders.OfType().First()); + if (_isMain) + { + a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(MareStaticFilesServerConfigurationController), + typeof(CacheController), typeof(RequestController), typeof(ServerFilesController), + typeof(DistributionController), typeof(MainController))); + } + else if (_isDistributionNode) + { + a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(CacheController), typeof(RequestController), typeof(DistributionController))); + } + else + { + a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(CacheController), typeof(RequestController))); + } + }); + + // authentication and authorization services.AddOptions(JwtBearerDefaults.AuthenticationScheme) .Configure>((o, s) => { @@ -97,115 +202,21 @@ public class Startup ValidateLifetime = false, ValidateAudience = false, ValidateIssuerSigningKey = true, - IssuerSigningKey = new SymmetricSecurityKey(Encoding.ASCII.GetBytes(s.GetValue(nameof(MareConfigurationAuthBase.Jwt)))), + IssuerSigningKey = new SymmetricSecurityKey(Encoding.ASCII.GetBytes(s.GetValue(nameof(MareConfigurationAuthBase.Jwt)))) }; }); - services.AddAuthentication(o => { o.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme; o.DefaultChallengeScheme = JwtBearerDefaults.AuthenticationScheme; o.DefaultScheme = JwtBearerDefaults.AuthenticationScheme; }).AddJwtBearer(); - services.AddAuthorization(options => { options.FallbackPolicy = new AuthorizationPolicyBuilder().RequireAuthenticatedUser().Build(); options.AddPolicy("Internal", new AuthorizationPolicyBuilder().RequireClaim(MareClaimTypes.Internal, "true").Build()); }); - - if (_isMain) - { - services.AddSingleton, MareConfigurationServiceServer>(); - } - else - { - services.AddSingleton, MareConfigurationServiceClient>(); - services.AddHostedService(p => (MareConfigurationServiceClient)p.GetService>()); - } - - services.AddSingleton, MareConfigurationServiceClient>(); - - services.AddSingleton(); - services.AddSingleton(); - services.AddHostedService(p => p.GetService()); - services.AddControllers().ConfigureApplicationPartManager(a => - { - a.FeatureProviders.Remove(a.FeatureProviders.OfType().First()); - if (_isMain) - { - a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(MareStaticFilesServerConfigurationController), typeof(CacheController), typeof(RequestController), typeof(ServerFilesController))); - } - else - { - a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(CacheController), typeof(RequestController))); - } - }); - - services.AddHostedService(p => (MareConfigurationServiceClient)p.GetService>()); - services.AddSingleton(); - var signalRServiceBuilder = services.AddSignalR(hubOptions => - { - hubOptions.MaximumReceiveMessageSize = long.MaxValue; - hubOptions.EnableDetailedErrors = true; - hubOptions.MaximumParallelInvocationsPerClient = 10; - hubOptions.StreamBufferCapacity = 200; - }).AddMessagePackProtocol(opt => - { - var resolver = CompositeResolver.Create(StandardResolverAllowPrivate.Instance, - BuiltinResolver.Instance, - AttributeFormatterResolver.Instance, - // replace enum resolver - DynamicEnumAsStringResolver.Instance, - DynamicGenericResolver.Instance, - DynamicUnionResolver.Instance, - DynamicObjectResolver.Instance, - PrimitiveObjectResolver.Instance, - // final fallback(last priority) - StandardResolver.Instance); - - opt.SerializerOptions = MessagePackSerializerOptions.Standard - .WithCompression(MessagePackCompression.Lz4Block) - .WithResolver(resolver); - }); - - // configure redis for SignalR - var redisConnection = mareConfig.GetValue(nameof(ServerConfiguration.RedisConnectionString), string.Empty); - signalRServiceBuilder.AddStackExchangeRedis(redisConnection, options => { }); - - var options = ConfigurationOptions.Parse(redisConnection); - - var endpoint = options.EndPoints[0]; - string address = ""; - int port = 0; - if (endpoint is DnsEndPoint dnsEndPoint) { address = dnsEndPoint.Host; port = dnsEndPoint.Port; } - if (endpoint is IPEndPoint ipEndPoint) { address = ipEndPoint.Address.ToString(); port = ipEndPoint.Port; } - var redisConfiguration = new RedisConfiguration() - { - AbortOnConnectFail = true, - KeyPrefix = "", - Hosts = new RedisHost[] - { - new RedisHost(){ Host = address, Port = port }, - }, - AllowAdmin = true, - ConnectTimeout = options.ConnectTimeout, - Database = 0, - Ssl = false, - Password = options.Password, - ServerEnumerationStrategy = new ServerEnumerationStrategy() - { - Mode = ServerEnumerationStrategy.ModeOptions.All, - TargetRole = ServerEnumerationStrategy.TargetRoleOptions.Any, - UnreachableServerAction = ServerEnumerationStrategy.UnreachableServerActionOptions.Throw, - }, - MaxValueLength = 1024, - PoolSize = mareConfig.GetValue(nameof(ServerConfiguration.RedisPool), 50), - SyncTimeout = options.SyncTimeout, - }; - - services.AddStackExchangeRedisExtensions(redisConfiguration); services.AddHealthChecks(); services.AddHttpLogging(e => e = new Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions()); @@ -229,7 +240,11 @@ public class Startup app.UseEndpoints(e => { - e.MapHub("/dummyhub"); + if (_isMain) + { + e.MapHub("/dummyhub"); + } + e.MapControllers(); e.MapHealthChecks("/health").WithMetadata(new AllowAnonymousAttribute()); });