diff --git a/MareAPI b/MareAPI index cd8934a..b529a10 160000 --- a/MareAPI +++ b/MareAPI @@ -1 +1 @@ -Subproject commit cd8934a4ab37a3549bacf7e7108f83a34403da96 +Subproject commit b529a101ae75bf9f20ae3b47d964e0a69b063754 diff --git a/MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthReply.cs b/MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthReply.cs index 5ca2a01..7e76ef4 100644 --- a/MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthReply.cs +++ b/MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthReply.cs @@ -1,3 +1,3 @@ namespace MareSynchronosServer.Authentication; -public record SecretKeyAuthReply(bool Success, string Uid, bool TempBan, bool Permaban); +public record SecretKeyAuthReply(bool Success, string Uid, string Alias, bool TempBan, bool Permaban); diff --git a/MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthenticatorService.cs b/MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthenticatorService.cs index b9a0144..830c78a 100644 --- a/MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthenticatorService.cs +++ b/MareSynchronosServer/MareSynchronosServer/Authentication/SecretKeyAuthenticatorService.cs @@ -10,30 +10,24 @@ namespace MareSynchronosServer.Authentication; public class SecretKeyAuthenticatorService { private readonly MareMetrics _metrics; - private readonly IServiceScopeFactory _serviceScopeFactory; + private readonly MareDbContext _mareDbContext; private readonly IConfigurationService _configurationService; private readonly ILogger _logger; - private readonly ConcurrentDictionary _cachedPositiveResponses = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _failedAuthorizations = new(StringComparer.Ordinal); - public SecretKeyAuthenticatorService(MareMetrics metrics, IServiceScopeFactory serviceScopeFactory, IConfigurationService configuration, ILogger logger) + public SecretKeyAuthenticatorService(MareMetrics metrics, MareDbContext mareDbContext, + IConfigurationService configuration, ILogger logger) { _logger = logger; _configurationService = configuration; _metrics = metrics; - _serviceScopeFactory = serviceScopeFactory; + _mareDbContext = mareDbContext; } public async Task AuthorizeAsync(string ip, string hashedSecretKey) { _metrics.IncCounter(MetricsAPI.CounterAuthenticationRequests); - if (_cachedPositiveResponses.TryGetValue(hashedSecretKey, out var cachedPositiveResponse)) - { - _metrics.IncCounter(MetricsAPI.CounterAuthenticationCacheHits); - return cachedPositiveResponse; - } - if (_failedAuthorizations.TryGetValue(ip, out var existingFailedAuthorization) && existingFailedAuthorization.FailedAttempts > _configurationService.GetValueOrDefault(nameof(MareConfigurationAuthBase.FailedAuthForTempBan), 5)) { @@ -50,26 +44,17 @@ public class SecretKeyAuthenticatorService _failedAuthorizations.Remove(ip, out _); }); } - return new(Success: false, Uid: null, TempBan: true, Permaban: false); + return new(Success: false, Uid: null, TempBan: true, Alias: null, Permaban: false); } - using var scope = _serviceScopeFactory.CreateScope(); - using var context = scope.ServiceProvider.GetService(); - var authReply = await context.Auth.AsNoTracking().SingleOrDefaultAsync(u => u.HashedKey == hashedSecretKey).ConfigureAwait(false); + var authReply = await _mareDbContext.Auth.Include(a => a.User).AsNoTracking() + .SingleOrDefaultAsync(u => u.HashedKey == hashedSecretKey).ConfigureAwait(false); - SecretKeyAuthReply reply = new(authReply != null, authReply?.UserUID, false, authReply?.IsBanned ?? false); + SecretKeyAuthReply reply = new(authReply != null, authReply?.UserUID, authReply?.User?.Alias ?? string.Empty, TempBan: false, authReply?.IsBanned ?? false); if (reply.Success) { _metrics.IncCounter(MetricsAPI.CounterAuthenticationSuccesses); - - _cachedPositiveResponses[hashedSecretKey] = reply; - _ = Task.Run(async () => - { - await Task.Delay(TimeSpan.FromMinutes(5)).ConfigureAwait(false); - _cachedPositiveResponses.TryRemove(hashedSecretKey, out _); - }); - } else { @@ -97,6 +82,6 @@ public class SecretKeyAuthenticatorService } } - return new(Success: false, Uid: null, TempBan: false, Permaban: false); + return new(Success: false, Uid: null, Alias: null, TempBan: false, Permaban: false); } } diff --git a/MareSynchronosServer/MareSynchronosServer/Controllers/JwtController.cs b/MareSynchronosServer/MareSynchronosServer/Controllers/JwtController.cs index 242d547..65d7b17 100644 --- a/MareSynchronosServer/MareSynchronosServer/Controllers/JwtController.cs +++ b/MareSynchronosServer/MareSynchronosServer/Controllers/JwtController.cs @@ -1,5 +1,6 @@ using MareSynchronos.API.Routes; using MareSynchronosServer.Authentication; +using MareSynchronosServer.Services; using MareSynchronosShared; using MareSynchronosShared.Data; using MareSynchronosShared.Models; @@ -23,6 +24,7 @@ public class JwtController : Controller private readonly IHttpContextAccessor _accessor; private readonly IRedisDatabase _redis; private readonly MareDbContext _mareDbContext; + private readonly GeoIPService _geoIPProvider; private readonly SecretKeyAuthenticatorService _secretKeyAuthenticatorService; private readonly AccountRegistrationService _accountRegistrationService; private readonly IConfigurationService _configuration; @@ -31,10 +33,11 @@ public class JwtController : Controller SecretKeyAuthenticatorService secretKeyAuthenticatorService, AccountRegistrationService accountRegistrationService, IConfigurationService configuration, - IRedisDatabase redisDb) + IRedisDatabase redisDb, GeoIPService geoIPProvider) { _accessor = accessor; _redis = redisDb; + _geoIPProvider = geoIPProvider; _mareDbContext = mareDbContext; _secretKeyAuthenticatorService = secretKeyAuthenticatorService; _accountRegistrationService = accountRegistrationService; @@ -66,7 +69,7 @@ public class JwtController : Controller } if (!authResult.Success && !authResult.TempBan) return Unauthorized("The provided secret key is invalid. Verify your accounts existence and/or recover the secret key."); - if (!authResult.Success && authResult.TempBan) return Unauthorized("You are temporarily banned. Try connecting again in 5 minutes."); + if (!authResult.Success && authResult.TempBan) return Unauthorized("Due to an excessive amount of failed authentication attempts you are temporarily banned. Check your Secret Key configuration and try connecting again in 5 minutes."); if (authResult.Permaban) { if (!_mareDbContext.BannedUsers.Any(c => c.CharacterIdentification == charaIdent)) @@ -112,6 +115,8 @@ public class JwtController : Controller { new Claim(MareClaimTypes.Uid, authResult.Uid), new Claim(MareClaimTypes.CharaIdent, charaIdent), + new Claim(MareClaimTypes.Alias, authResult.Alias), + new Claim(MareClaimTypes.Continent, await _geoIPProvider.GetCountryFromIP(_accessor)), }); return Content(token.RawData); @@ -140,3 +145,4 @@ public class JwtController : Controller return handler.CreateJwtSecurityToken(token); } } + diff --git a/MareSynchronosServer/MareSynchronosServer/Controllers/MainController.cs b/MareSynchronosServer/MareSynchronosServer/Controllers/MainController.cs new file mode 100644 index 0000000..add64a4 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosServer/Controllers/MainController.cs @@ -0,0 +1,31 @@ +using MareSynchronos.API.Routes; +using MareSynchronos.API.SignalR; +using MareSynchronosServer.Hubs; +using MareSynchronosServer.Services; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.SignalR; + +namespace MareSynchronosServer.Controllers; + +[Route(MareFiles.Main)] +public class MainController : Controller +{ + private IHubContext _hubContext; + + public MainController(ILogger logger, IHubContext hubContext) + { + _hubContext = hubContext; + } + + [HttpGet(MareFiles.Main_SendReady)] + [Authorize(Policy = "Internal")] + public IActionResult SendReadyToClients(string uid, Guid requestId) + { + _ = Task.Run(async () => + { + await _hubContext.Clients.User(uid).Client_DownloadReady(requestId); + }); + return Ok(); + } +} \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Functions.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Functions.cs index bb2326b..dd2fa89 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Functions.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Functions.cs @@ -16,6 +16,8 @@ public partial class MareHub public string UserUID => Context.User?.Claims?.SingleOrDefault(c => string.Equals(c.Type, MareClaimTypes.Uid, StringComparison.Ordinal))?.Value ?? throw new Exception("No UID in Claims"); + public string Continent => Context.User?.Claims?.SingleOrDefault(c => string.Equals(c.Type, MareClaimTypes.Continent, StringComparison.Ordinal))?.Value ?? "UNK"; + private async Task DeleteUser(User user) { var ownPairData = await _dbContext.ClientPairs.Where(u => u.User.UID == user.UID).ToListAsync().ConfigureAwait(false); @@ -35,11 +37,6 @@ public partial class MareHub _dbContext.Remove(userProfileData); } - while (_dbContext.Files.Any(f => f.Uploader == user)) - { - await Task.Delay(1000).ConfigureAwait(false); - } - _dbContext.ClientPairs.RemoveRange(ownPairData); await _dbContext.SaveChangesAsync().ConfigureAwait(false); var otherPairData = await _dbContext.ClientPairs.Include(u => u.User) diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs index 3db19a2..6b5b005 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs @@ -93,7 +93,7 @@ public partial class MareHub : Hub, IMareHub [Authorize(Policy = "Authenticated")] public override async Task OnConnectedAsync() { - _mareMetrics.IncGauge(MetricsAPI.GaugeConnections); + _mareMetrics.IncGaugeWithLabels(MetricsAPI.GaugeConnections, labels: Continent); try { @@ -109,7 +109,7 @@ public partial class MareHub : Hub, IMareHub [Authorize(Policy = "Authenticated")] public override async Task OnDisconnectedAsync(Exception exception) { - _mareMetrics.DecGauge(MetricsAPI.GaugeConnections); + _mareMetrics.DecGaugeWithLabels(MetricsAPI.GaugeConnections, labels: Continent); try { @@ -120,9 +120,6 @@ public partial class MareHub : Hub, IMareHub await RemoveUserFromRedis().ConfigureAwait(false); await SendOfflineToAllPairedUsers().ConfigureAwait(false); - - _dbContext.RemoveRange(_dbContext.Files.Where(f => !f.Uploaded && f.UploaderUID == UserUID)); - await _dbContext.SaveChangesAsync().ConfigureAwait(false); } catch { } diff --git a/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj b/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj index b662f74..160d820 100644 --- a/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj +++ b/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj @@ -29,6 +29,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/MareSynchronosServer/MareSynchronosServer/Program.cs b/MareSynchronosServer/MareSynchronosServer/Program.cs index 69fe275..8beb97f 100644 --- a/MareSynchronosServer/MareSynchronosServer/Program.cs +++ b/MareSynchronosServer/MareSynchronosServer/Program.cs @@ -25,10 +25,8 @@ public class Program context.SaveChanges(); // clean up residuals - var looseFiles = context.Files.Where(f => f.Uploaded == false); var unfinishedRegistrations = context.LodeStoneAuth.Where(c => c.StartedAt != null); context.RemoveRange(unfinishedRegistrations); - context.RemoveRange(looseFiles); context.SaveChanges(); logger.LogInformation(options.ToString()); diff --git a/MareSynchronosServer/MareSynchronosServer/Services/GeoIPService.cs b/MareSynchronosServer/MareSynchronosServer/Services/GeoIPService.cs new file mode 100644 index 0000000..dbbbf97 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosServer/Services/GeoIPService.cs @@ -0,0 +1,137 @@ +using MareSynchronosShared; +using MareSynchronosShared.Services; +using MareSynchronosShared.Utils; +using MaxMind.GeoIP2; + +namespace MareSynchronosServer.Services; + +public class GeoIPService : IHostedService +{ + private readonly ILogger _logger; + private readonly IConfigurationService _mareConfiguration; + private bool _useGeoIP = false; + private string _cityFile = string.Empty; + private DatabaseReader? _dbReader; + private DateTime _dbLastWriteTime = DateTime.Now; + private CancellationTokenSource _fileWriteTimeCheckCts = new(); + private bool _processingReload = false; + + public GeoIPService(ILogger logger, + IConfigurationService mareConfiguration) + { + _logger = logger; + _mareConfiguration = mareConfiguration; + } + + public async Task GetCountryFromIP(IHttpContextAccessor httpContextAccessor) + { + if (!_useGeoIP) + { + return "*"; + } + + try + { + var ip = httpContextAccessor.GetIpAddress(); + + using CancellationTokenSource waitCts = new(); + waitCts.CancelAfter(TimeSpan.FromSeconds(5)); + while (_processingReload) await Task.Delay(100, waitCts.Token).ConfigureAwait(false); + + if (_dbReader.TryCity(ip, out var response)) + { + var continent = response.Continent.Code; + if (string.Equals(continent, "NA", StringComparison.Ordinal) + && response.Location.Longitude != null) + { + if (response.Location.Longitude < -102) + { + continent = "NA-W"; + } + else + { + continent = "NA-E"; + } + } + + return continent ?? "*"; + } + + return "*"; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error handling Geo IP country in request"); + return "*"; + } + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("GeoIP module starting update task"); + + var token = _fileWriteTimeCheckCts.Token; + _ = PeriodicReloadTask(token); + + return Task.CompletedTask; + } + + private async Task PeriodicReloadTask(CancellationToken token) + { + while (!token.IsCancellationRequested) + { + try + { + _processingReload = true; + + var useGeoIP = _mareConfiguration.GetValueOrDefault(nameof(ServerConfiguration.UseGeoIP), false); + var cityFile = _mareConfiguration.GetValueOrDefault(nameof(ServerConfiguration.GeoIPDbCityFile), string.Empty); + var lastWriteTime = new FileInfo(cityFile).LastWriteTimeUtc; + if (useGeoIP && (!string.Equals(cityFile, _cityFile, StringComparison.OrdinalIgnoreCase) || lastWriteTime != _dbLastWriteTime)) + { + _cityFile = cityFile; + if (!File.Exists(_cityFile)) throw new FileNotFoundException($"Could not open GeoIP City Database, path does not exist: {_cityFile}"); + _dbReader?.Dispose(); + _dbReader = null; + _dbReader = new DatabaseReader(_cityFile); + _dbLastWriteTime = lastWriteTime; + + _ = _dbReader.City("8.8.8.8").Continent; + + _logger.LogInformation($"Loaded GeoIP city file from {_cityFile}"); + + if (_useGeoIP != useGeoIP) + { + _logger.LogInformation("GeoIP module is now enabled"); + _useGeoIP = useGeoIP; + } + } + + if (_useGeoIP != useGeoIP && !useGeoIP) + { + _logger.LogInformation("GeoIP module is now disabled"); + _useGeoIP = useGeoIP; + } + } + catch (Exception e) + { + _logger.LogWarning(e, "Error during periodic GeoIP module reload task, disabling GeoIP"); + _useGeoIP = false; + } + finally + { + _processingReload = false; + } + + await Task.Delay(TimeSpan.FromMinutes(1)).ConfigureAwait(false); + } + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _fileWriteTimeCheckCts.Cancel(); + _fileWriteTimeCheckCts.Dispose(); + _dbReader.Dispose(); + return Task.CompletedTask; + } +} diff --git a/MareSynchronosServer/MareSynchronosServer/Services/UserCleanupService.cs b/MareSynchronosServer/MareSynchronosServer/Services/UserCleanupService.cs index 11b033d..b33c4bd 100644 --- a/MareSynchronosServer/MareSynchronosServer/Services/UserCleanupService.cs +++ b/MareSynchronosServer/MareSynchronosServer/Services/UserCleanupService.cs @@ -144,9 +144,6 @@ public class UserCleanupService : IHostedService var auth = dbContext.Auth.Single(a => a.UserUID == user.UID); - var userFiles = dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == user.UID).ToList(); - dbContext.Files.RemoveRange(userFiles); - var ownPairData = dbContext.ClientPairs.Where(u => u.User.UID == user.UID).ToList(); dbContext.ClientPairs.RemoveRange(ownPairData); var otherPairData = dbContext.ClientPairs.Include(u => u.User) diff --git a/MareSynchronosServer/MareSynchronosServer/Startup.cs b/MareSynchronosServer/MareSynchronosServer/Startup.cs index 8f3d499..efa3372 100644 --- a/MareSynchronosServer/MareSynchronosServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosServer/Startup.cs @@ -96,8 +96,10 @@ public class Startup if (isMainServer) { + services.AddSingleton(); services.AddSingleton(); services.AddHostedService(provider => provider.GetService()); + services.AddHostedService(provider => provider.GetService()); } } diff --git a/MareSynchronosServer/MareSynchronosShared/Metrics/MareMetrics.cs b/MareSynchronosServer/MareSynchronosShared/Metrics/MareMetrics.cs index 3add03f..624e907 100644 --- a/MareSynchronosServer/MareSynchronosShared/Metrics/MareMetrics.cs +++ b/MareSynchronosServer/MareSynchronosShared/Metrics/MareMetrics.cs @@ -17,7 +17,10 @@ public class MareMetrics foreach (var gauge in gaugesToServe) { logger.LogInformation($"Creating Metric for Counter {gauge}"); - gauges.Add(gauge, Prometheus.Metrics.CreateGauge(gauge, gauge)); + if (!string.Equals(gauge, MetricsAPI.GaugeConnections, StringComparison.OrdinalIgnoreCase)) + gauges.Add(gauge, Prometheus.Metrics.CreateGauge(gauge, gauge)); + else + gauges.Add(gauge, Prometheus.Metrics.CreateGauge(gauge, gauge, new[] { "continent" })); } } @@ -25,35 +28,57 @@ public class MareMetrics private readonly Dictionary gauges = new(StringComparer.Ordinal); + public void IncGaugeWithLabels(string gaugeName, double value = 1.0, params string[] labels) + { + if (gauges.TryGetValue(gaugeName, out Gauge gauge)) + { + lock (gauge) + gauge.WithLabels(labels).Inc(value); + } + } + + public void DecGaugeWithLabels(string gaugeName, double value = 1.0, params string[] labels) + { + if (gauges.TryGetValue(gaugeName, out Gauge gauge)) + { + lock (gauge) + gauge.WithLabels(labels).Dec(value); + } + } + public void SetGaugeTo(string gaugeName, double value) { - if (gauges.ContainsKey(gaugeName)) + if (gauges.TryGetValue(gaugeName, out Gauge gauge)) { - gauges[gaugeName].Set(value); + lock (gauge) + gauge.Set(value); } } public void IncGauge(string gaugeName, double value = 1.0) { - if (gauges.ContainsKey(gaugeName)) + if (gauges.TryGetValue(gaugeName, out Gauge gauge)) { - gauges[gaugeName].Inc(value); + lock (gauge) + gauge.Inc(value); } } public void DecGauge(string gaugeName, double value = 1.0) { - if (gauges.ContainsKey(gaugeName)) + if (gauges.TryGetValue(gaugeName, out Gauge gauge)) { - gauges[gaugeName].Dec(value); + lock (gauge) + gauge.Dec(value); } } public void IncCounter(string counterName, double value = 1.0) { - if (counters.ContainsKey(counterName)) + if (counters.TryGetValue(counterName, out Counter counter)) { - counters[counterName].Inc(value); + lock (counter) + counter.Inc(value); } } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs b/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs index 3ca9592..d821592 100644 --- a/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs +++ b/MareSynchronosServer/MareSynchronosShared/Metrics/MetricsAPI.cs @@ -3,7 +3,7 @@ public class MetricsAPI { public const string CounterInitializedConnections = "mare_initialized_connections"; - public const string GaugeConnections = "mare_unauthorized_connections"; + public const string GaugeConnections = "mare_connections"; public const string GaugeAuthorizedConnections = "mare_authorized_connections"; public const string GaugeAvailableWorkerThreads = "mare_available_threadpool"; public const string GaugeAvailableIOWorkerThreads = "mare_available_threadpool_io"; @@ -12,7 +12,11 @@ public class MetricsAPI public const string GaugePairs = "mare_pairs"; public const string GaugePairsPaused = "mare_pairs_paused"; public const string GaugeFilesTotal = "mare_files"; + public const string GaugeFilesTotalColdStorage = "mare_files_cold"; public const string GaugeFilesTotalSize = "mare_files_size"; + public const string GaugeFilesTotalSizeColdStorage = "mare_files_size_cold"; + public const string GaugeFilesDownloadingFromCache = "mare_files_downloading_from_cache"; + public const string GaugeFilesTasksWaitingForDownloadFromCache = "mare_files_waiting_for_dl"; public const string CounterUserPushData = "mare_user_push"; public const string CounterUserPushDataTo = "mare_user_push_to"; public const string CounterAuthenticationRequests = "mare_auth_requests"; @@ -31,6 +35,9 @@ public class MetricsAPI public const string GaugeQueueActive = "mare_download_queue_active"; public const string GaugeQueueInactive = "mare_download_queue_inactive"; public const string GaugeDownloadQueue = "mare_download_queue"; + public const string GaugeDownloadQueueCancelled = "mare_download_queue_cancelled"; + public const string GaugeDownloadPriorityQueue = "mare_download_priority_queue"; + public const string GaugeDownloadPriorityQueueCancelled = "mare_download_priority_queue_cancelled"; public const string CounterFileRequests = "mare_files_requests"; public const string CounterFileRequestSize = "mare_files_request_size"; public const string CounterAccountsCreated = "mare_accounts_created"; diff --git a/MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceServer.cs b/MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceServer.cs index 24c9522..579c4fe 100644 --- a/MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceServer.cs +++ b/MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceServer.cs @@ -1,5 +1,6 @@ using MareSynchronosShared.Utils; using Microsoft.Extensions.Options; +using System.Collections; using System.Text; namespace MareSynchronosShared.Services; @@ -30,11 +31,20 @@ public class MareConfigurationServiceServer : IConfigurationService where StringBuilder sb = new(); foreach (var prop in props) { - sb.AppendLine($"{prop.Name} (IsRemote: {prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any()}) => {prop.GetValue(_config.CurrentValue)}"); + var isRemote = prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any(); + var getValueMethod = GetType().GetMethod(nameof(GetValue)).MakeGenericMethod(prop.PropertyType); + var value = isRemote ? getValueMethod.Invoke(this, new[] { prop.Name }) : prop.GetValue(_config.CurrentValue); + if (typeof(IEnumerable).IsAssignableFrom(prop.PropertyType) && !typeof(string).IsAssignableFrom(prop.PropertyType)) + { + var enumVal = (IEnumerable)value; + value = string.Empty; + foreach (var listVal in enumVal) + { + value += listVal.ToString() + ", "; + } + } + sb.AppendLine($"{prop.Name} (IsRemote: {isRemote}) => {value}"); } - - sb.AppendLine(_config.ToString()); - return sb.ToString(); } } diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs b/MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs index 4b9d694..4ffe680 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs @@ -2,11 +2,12 @@ public class CdnShardConfiguration { + public List Continents { get; set; } public string FileMatch { get; set; } public Uri CdnFullUrl { get; set; } public override string ToString() { - return CdnFullUrl.ToString() + " == " + FileMatch; + return CdnFullUrl.ToString() + "[" + string.Join(',', Continents) + "] == " + FileMatch; } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/MareClaimTypes.cs b/MareSynchronosServer/MareSynchronosShared/Utils/MareClaimTypes.cs index 08539f4..7641e9e 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/MareClaimTypes.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/MareClaimTypes.cs @@ -3,6 +3,8 @@ public static class MareClaimTypes { public const string Uid = "uid"; + public const string Alias = "alias"; public const string CharaIdent = "character_identification"; public const string Internal = "internal"; + public const string Continent = "continent"; } diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/MareConfigurationAuthBase.cs b/MareSynchronosServer/MareSynchronosShared/Utils/MareConfigurationAuthBase.cs index b9683d0..a91c24e 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/MareConfigurationAuthBase.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/MareConfigurationAuthBase.cs @@ -14,6 +14,8 @@ public class MareConfigurationAuthBase : MareConfigurationBase public int RegisterIpDurationInMinutes { get; set; } = 10; [RemoteConfiguration] public List WhitelistedIps { get; set; } = new(); + [RemoteConfiguration] + public bool UseGeoIP { get; set; } = false; public override string ToString() { @@ -25,6 +27,7 @@ public class MareConfigurationAuthBase : MareConfigurationBase sb.AppendLine($"{nameof(RegisterIpDurationInMinutes)} => {RegisterIpDurationInMinutes}"); sb.AppendLine($"{nameof(Jwt)} => {Jwt}"); sb.AppendLine($"{nameof(WhitelistedIps)} => {string.Join(", ", WhitelistedIps)}"); + sb.AppendLine($"{nameof(UseGeoIP)} => {UseGeoIP}"); return sb.ToString(); } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs b/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs index 2b8d668..abb648a 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs @@ -24,6 +24,7 @@ public class ServerConfiguration : MareConfigurationAuthBase [RemoteConfiguration] public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14; + public string GeoIPDbCityFile { get; set; } = string.Empty; public int RedisPool { get; set; } = 50; @@ -40,6 +41,7 @@ public class ServerConfiguration : MareConfigurationAuthBase sb.AppendLine($"{nameof(MaxGroupUserCount)} => {MaxGroupUserCount}"); sb.AppendLine($"{nameof(PurgeUnusedAccounts)} => {PurgeUnusedAccounts}"); sb.AppendLine($"{nameof(PurgeUnusedAccountsPeriodInDays)} => {PurgeUnusedAccountsPeriodInDays}"); + sb.AppendLine($"{nameof(GeoIPDbCityFile)} => {GeoIPDbCityFile}"); return sb.ToString(); } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/SharedDbFunctions.cs b/MareSynchronosServer/MareSynchronosShared/Utils/SharedDbFunctions.cs index ac69f42..587ce06 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/SharedDbFunctions.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/SharedDbFunctions.cs @@ -65,9 +65,6 @@ public static class SharedDbFunctions var auth = dbContext.Auth.Single(a => a.UserUID == user.UID); - var userFiles = dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == user.UID).ToList(); - dbContext.Files.RemoveRange(userFiles); - var ownPairData = dbContext.ClientPairs.Where(u => u.User.UID == user.UID).ToList(); dbContext.ClientPairs.RemoveRange(ownPairData); var otherPairData = dbContext.ClientPairs.Include(u => u.User) diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs b/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs index 7404e8a..169889f 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs @@ -5,15 +5,28 @@ namespace MareSynchronosStaticFilesServer; public class StaticFilesServerConfiguration : MareConfigurationBase { + public bool IsDistributionNode { get; set; } = false; + public bool NotifyMainServerDirectly { get; set; } = false; public Uri? MainFileServerAddress { get; set; } = null; + public Uri? DistributionFileServerAddress { get; set; } = null; + public bool DistributionFileServerForceHTTP2 { get; set; } = false; public int ForcedDeletionOfFilesAfterHours { get; set; } = -1; public double CacheSizeHardLimitInGiB { get; set; } = -1; + public int MinimumFileRetentionPeriodInDays { get; set; } = 7; public int UnusedFileRetentionPeriodInDays { get; set; } = 14; public string CacheDirectory { get; set; } public int DownloadQueueSize { get; set; } = 50; public int DownloadTimeoutSeconds { get; set; } = 5; public int DownloadQueueReleaseSeconds { get; set; } = 15; public int DownloadQueueClearLimit { get; set; } = 15000; + public int CleanupCheckInMinutes { get; set; } = 15; + public bool UseColdStorage { get; set; } = false; + public string? ColdStorageDirectory { get; set; } = null; + public double ColdStorageSizeHardLimitInGiB { get; set; } = -1; + public int ColdStorageMinimumFileRetentionPeriodInDays { get; set; } = 30; + public int ColdStorageUnusedFileRetentionPeriodInDays { get; set; } = 30; + public double CacheSmallSizeThresholdKiB { get; set; } = 64; + public double CacheLargeSizeThresholdKiB { get; set; } = 1024; [RemoteConfiguration] public Uri CdnFullUrl { get; set; } = null; [RemoteConfiguration] @@ -22,13 +35,26 @@ public class StaticFilesServerConfiguration : MareConfigurationBase { StringBuilder sb = new(); sb.AppendLine(base.ToString()); + sb.AppendLine($"{nameof(IsDistributionNode)} => {IsDistributionNode}"); + sb.AppendLine($"{nameof(NotifyMainServerDirectly)} => {NotifyMainServerDirectly}"); sb.AppendLine($"{nameof(MainFileServerAddress)} => {MainFileServerAddress}"); + sb.AppendLine($"{nameof(DistributionFileServerAddress)} => {DistributionFileServerAddress}"); + sb.AppendLine($"{nameof(DistributionFileServerForceHTTP2)} => {DistributionFileServerForceHTTP2}"); sb.AppendLine($"{nameof(ForcedDeletionOfFilesAfterHours)} => {ForcedDeletionOfFilesAfterHours}"); sb.AppendLine($"{nameof(CacheSizeHardLimitInGiB)} => {CacheSizeHardLimitInGiB}"); + sb.AppendLine($"{nameof(UseColdStorage)} => {UseColdStorage}"); + sb.AppendLine($"{nameof(ColdStorageDirectory)} => {ColdStorageDirectory}"); + sb.AppendLine($"{nameof(ColdStorageSizeHardLimitInGiB)} => {ColdStorageSizeHardLimitInGiB}"); + sb.AppendLine($"{nameof(ColdStorageMinimumFileRetentionPeriodInDays)} => {ColdStorageMinimumFileRetentionPeriodInDays}"); + sb.AppendLine($"{nameof(ColdStorageUnusedFileRetentionPeriodInDays)} => {ColdStorageUnusedFileRetentionPeriodInDays}"); + sb.AppendLine($"{nameof(MinimumFileRetentionPeriodInDays)} => {MinimumFileRetentionPeriodInDays}"); sb.AppendLine($"{nameof(UnusedFileRetentionPeriodInDays)} => {UnusedFileRetentionPeriodInDays}"); + sb.AppendLine($"{nameof(CacheSmallSizeThresholdKiB)} => {CacheSmallSizeThresholdKiB}"); + sb.AppendLine($"{nameof(CacheLargeSizeThresholdKiB)} => {CacheLargeSizeThresholdKiB}"); sb.AppendLine($"{nameof(CacheDirectory)} => {CacheDirectory}"); sb.AppendLine($"{nameof(DownloadQueueSize)} => {DownloadQueueSize}"); sb.AppendLine($"{nameof(DownloadQueueReleaseSeconds)} => {DownloadQueueReleaseSeconds}"); + sb.AppendLine($"{nameof(CdnShardConfiguration)} => {string.Join(", ", CdnShardConfiguration)}"); return sb.ToString(); } } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs index 4ccbd64..018ab85 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs @@ -13,4 +13,6 @@ public class ControllerBase : Controller } protected string MareUser => HttpContext.User.Claims.First(f => string.Equals(f.Type, MareClaimTypes.Uid, StringComparison.Ordinal)).Value; + protected string Continent => HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, MareClaimTypes.Continent, StringComparison.Ordinal))?.Value ?? "*"; + protected bool IsPriority => false; } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/DistributionController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/DistributionController.cs new file mode 100644 index 0000000..dbaaa8f --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/DistributionController.cs @@ -0,0 +1,46 @@ +using MareSynchronos.API.Routes; +using MareSynchronosStaticFilesServer.Services; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace MareSynchronosStaticFilesServer.Controllers; + +[Route(MareFiles.Distribution)] +public class DistributionController : ControllerBase +{ + private readonly CachedFileProvider _cachedFileProvider; + + public DistributionController(ILogger logger, CachedFileProvider cachedFileProvider) : base(logger) + { + _cachedFileProvider = cachedFileProvider; + } + + [HttpGet(MareFiles.Distribution_Get)] + [Authorize(Policy = "Internal")] + public async Task GetFile(string file) + { + _logger.LogInformation($"GetFile:{MareUser}:{file}"); + + var fs = await _cachedFileProvider.GetAndDownloadFileStream(file); + if (fs == null) return NotFound(); + + return File(fs, "application/octet-stream"); + } + + [HttpPost("touch")] + [Authorize(Policy = "Internal")] + public IActionResult TouchFiles([FromBody] string[] files) + { + _logger.LogInformation($"TouchFiles:{MareUser}:{files.Length}"); + + if (files.Length == 0) + return Ok(); + + Task.Run(() => { + foreach (var file in files) + _cachedFileProvider.TouchColdHash(file); + }).ConfigureAwait(false); + + return Ok(); + } +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/MainController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/MainController.cs new file mode 100644 index 0000000..7e43ba0 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/MainController.cs @@ -0,0 +1,25 @@ +using MareSynchronos.API.Routes; +using MareSynchronosStaticFilesServer.Services; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace MareSynchronosStaticFilesServer.Controllers; + +[Route(MareFiles.Main)] +public class MainController : ControllerBase +{ + private readonly IClientReadyMessageService _messageService; + + public MainController(ILogger logger, IClientReadyMessageService mareHub) : base(logger) + { + _messageService = mareHub; + } + + [HttpGet(MareFiles.Main_SendReady)] + [Authorize(Policy = "Internal")] + public IActionResult SendReadyToClients(string uid, Guid requestId) + { + _messageService.SendDownloadReady(uid, requestId); + return Ok(); + } +} \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs index 966ebf5..833d4a6 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs @@ -1,5 +1,4 @@ using MareSynchronos.API.Routes; -using MareSynchronosShared.Data; using MareSynchronosStaticFilesServer.Services; using Microsoft.AspNetCore.Mvc; @@ -10,14 +9,11 @@ public class RequestController : ControllerBase { private readonly CachedFileProvider _cachedFileProvider; private readonly RequestQueueService _requestQueue; - private readonly MareDbContext _mareDbContext; - private static readonly SemaphoreSlim _parallelRequestSemaphore = new(500); - public RequestController(ILogger logger, CachedFileProvider cachedFileProvider, RequestQueueService requestQueue, MareDbContext mareDbContext) : base(logger) + public RequestController(ILogger logger, CachedFileProvider cachedFileProvider, RequestQueueService requestQueue) : base(logger) { _cachedFileProvider = cachedFileProvider; _requestQueue = requestQueue; - _mareDbContext = mareDbContext; } [HttpGet] @@ -26,15 +22,10 @@ public class RequestController : ControllerBase { try { - await _parallelRequestSemaphore.WaitAsync(HttpContext.RequestAborted); - _requestQueue.RemoveFromQueue(requestId, MareUser); + _requestQueue.RemoveFromQueue(requestId, MareUser, IsPriority); return Ok(); } catch (OperationCanceledException) { return BadRequest(); } - finally - { - _parallelRequestSemaphore.Release(); - } } [HttpPost] @@ -43,23 +34,18 @@ public class RequestController : ControllerBase { try { - await _parallelRequestSemaphore.WaitAsync(HttpContext.RequestAborted); foreach (var file in files) { _logger.LogDebug("Prerequested file: " + file); - _cachedFileProvider.DownloadFileWhenRequired(file); + await _cachedFileProvider.DownloadFileWhenRequired(file).ConfigureAwait(false); } Guid g = Guid.NewGuid(); - await _requestQueue.EnqueueUser(new(g, MareUser, files.ToList()), _mareDbContext); + await _requestQueue.EnqueueUser(new(g, MareUser, files.ToList()), IsPriority, HttpContext.RequestAborted); return Ok(g); } catch (OperationCanceledException) { return BadRequest(); } - finally - { - _parallelRequestSemaphore.Release(); - } } [HttpGet] @@ -68,8 +54,8 @@ public class RequestController : ControllerBase { try { - if (!await _requestQueue.StillEnqueued(requestId, MareUser, _mareDbContext)) - await _requestQueue.EnqueueUser(new(requestId, MareUser, files.ToList()), _mareDbContext); + if (!_requestQueue.StillEnqueued(requestId, MareUser, IsPriority)) + await _requestQueue.EnqueueUser(new(requestId, MareUser, files.ToList()), IsPriority, HttpContext.RequestAborted); return Ok(); } catch (OperationCanceledException) { return BadRequest(); } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs index 03c78c9..0fa1201 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs @@ -10,10 +10,10 @@ using MareSynchronosShared.Services; using MareSynchronosShared.Utils; using MareSynchronosStaticFilesServer.Services; using MareSynchronosStaticFilesServer.Utils; -using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; using System.Collections.Concurrent; using System.Security.Cryptography; using System.Security.Policy; @@ -28,6 +28,7 @@ public class ServerFilesController : ControllerBase private static readonly SemaphoreSlim _fileLockDictLock = new(1); private static readonly ConcurrentDictionary _fileUploadLocks = new(StringComparer.Ordinal); private readonly string _basePath; + private readonly string _coldBasePath; private readonly CachedFileProvider _cachedFileProvider; private readonly IConfigurationService _configuration; private readonly IHubContext _hubContext; @@ -36,12 +37,14 @@ public class ServerFilesController : ControllerBase public ServerFilesController(ILogger logger, CachedFileProvider cachedFileProvider, IConfigurationService configuration, - IHubContext hubContext, + IHubContext hubContext, MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger) { - _basePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); - _cachedFileProvider = cachedFileProvider; _configuration = configuration; + _basePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); + if (_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false)) + _basePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)); + _cachedFileProvider = cachedFileProvider; _hubContext = hubContext; _mareDbContext = mareDbContext; _metricsClient = metricsClient; @@ -50,44 +53,47 @@ public class ServerFilesController : ControllerBase [HttpPost(MareFiles.ServerFiles_DeleteAll)] public async Task FilesDeleteAll() { - var ownFiles = await _mareDbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == MareUser).ToListAsync().ConfigureAwait(false); - - foreach (var dbFile in ownFiles) - { - var fi = FilePathUtil.GetFileInfoForHash(_basePath, dbFile.Hash); - if (fi != null) - { - _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1); - _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0); - - fi?.Delete(); - } - } - - _mareDbContext.Files.RemoveRange(ownFiles); - await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); - return Ok(); } [HttpGet(MareFiles.ServerFiles_GetSizes)] public async Task FilesGetSizes([FromBody] List hashes) { - var allFiles = await _mareDbContext.Files.Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false); var forbiddenFiles = await _mareDbContext.ForbiddenUploadEntries. Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false); List response = new(); var cacheFile = await _mareDbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false); - var shardConfig = new List(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List())); + var allFileShards = new List(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List())); foreach (var file in cacheFile) { var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase)); + Uri? baseUrl = null; - var matchedShardConfig = shardConfig.OrderBy(g => Guid.NewGuid()).FirstOrDefault(f => new Regex(f.FileMatch).IsMatch(file.Hash)); - var baseUrl = matchedShardConfig?.CdnFullUrl ?? _configuration.GetValue(nameof(StaticFilesServerConfiguration.CdnFullUrl)); + if (forbiddenFile == null) + { + List selectedShards = new(); + var matchingShards = allFileShards.Where(f => new Regex(f.FileMatch).IsMatch(file.Hash)).ToList(); + + if (string.Equals(Continent, "*", StringComparison.Ordinal)) + { + selectedShards = matchingShards; + } + else + { + selectedShards = matchingShards.Where(c => c.Continents.Contains(Continent, StringComparer.OrdinalIgnoreCase)).ToList(); + if (!selectedShards.Any()) selectedShards = matchingShards; + } + + var shard = selectedShards + .OrderBy(s => !s.Continents.Any() ? 0 : 1) + .ThenBy(s => s.Continents.Contains("*", StringComparer.Ordinal) ? 0 : 1) + .ThenBy(g => Guid.NewGuid()).FirstOrDefault(); + + baseUrl = shard?.CdnFullUrl ?? _configuration.GetValue(nameof(StaticFilesServerConfiguration.CdnFullUrl)); + } response.Add(new DownloadFileDto { @@ -96,7 +102,7 @@ public class ServerFilesController : ControllerBase IsForbidden = forbiddenFile != null, Hash = file.Hash, Size = file.Size, - Url = baseUrl.ToString(), + Url = baseUrl?.ToString() ?? string.Empty, }); } @@ -145,18 +151,6 @@ public class ServerFilesController : ControllerBase return Ok(JsonSerializer.Serialize(notCoveredFiles.Values.ToList())); } - [HttpGet(MareFiles.ServerFiles_Get + "/{fileId}")] - [Authorize(Policy = "Internal")] - public IActionResult GetFile(string fileId) - { - _logger.LogInformation($"GetFile:{MareUser}:{fileId}"); - - var fs = _cachedFileProvider.GetLocalFileStream(fileId); - if (fs == null) return NotFound(); - - return File(fs, "application/octet-stream"); - } - [HttpPost(MareFiles.ServerFiles_Upload + "/{hash}")] [RequestSizeLimit(200 * 1024 * 1024)] public async Task UploadFile(string hash, CancellationToken requestAborted) diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs index d3ba1d0..28f5059 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs @@ -8,83 +8,182 @@ using MareSynchronos.API.Routes; namespace MareSynchronosStaticFilesServer.Services; -public class CachedFileProvider +public sealed class CachedFileProvider : IDisposable { + private readonly IConfigurationService _configuration; private readonly ILogger _logger; private readonly FileStatisticsService _fileStatisticsService; private readonly MareMetrics _metrics; private readonly ServerTokenGenerator _generator; + private readonly ITouchHashService _touchService; private readonly Uri _remoteCacheSourceUri; - private readonly string _basePath; + private readonly bool _useColdStorage; + private readonly string _hotStoragePath; + private readonly string _coldStoragePath; private readonly ConcurrentDictionary _currentTransfers = new(StringComparer.Ordinal); private readonly HttpClient _httpClient; - private bool IsMainServer => _remoteCacheSourceUri == null; + private readonly SemaphoreSlim _downloadSemaphore = new(1, 1); + private bool _disposed; - public CachedFileProvider(IConfigurationService configuration, ILogger logger, FileStatisticsService fileStatisticsService, MareMetrics metrics, ServerTokenGenerator generator) + private bool IsMainServer => _remoteCacheSourceUri == null && _isDistributionServer; + private bool _isDistributionServer; + + public CachedFileProvider(IConfigurationService configuration, ILogger logger, + FileStatisticsService fileStatisticsService, MareMetrics metrics, ServerTokenGenerator generator, ITouchHashService touchService) { + AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true); + _configuration = configuration; _logger = logger; _fileStatisticsService = fileStatisticsService; _metrics = metrics; _generator = generator; - _remoteCacheSourceUri = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.MainFileServerAddress), null); - _basePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); - _httpClient = new HttpClient(); + _touchService = touchService; + _remoteCacheSourceUri = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DistributionFileServerAddress), null); + _isDistributionServer = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false); + _useColdStorage = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); + _hotStoragePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); + _coldStoragePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)); + _httpClient = new(); + _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer", "1.0.0.0")); + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + _httpClient?.Dispose(); } private async Task DownloadTask(string hash) { - // download file from remote - var downloadUrl = MareFiles.ServerFilesGetFullPath(_remoteCacheSourceUri, hash); + var destinationFilePath = FilePathUtil.GetFilePath(_useColdStorage ? _coldStoragePath : _hotStoragePath, hash); + + // if cold storage is not configured or file not found or error is present try to download file from remote + var downloadUrl = MareFiles.DistributionGetFullPath(_remoteCacheSourceUri, hash); _logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl); using var requestMessage = new HttpRequestMessage(HttpMethod.Get, downloadUrl); requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _generator.Token); - var response = await _httpClient.SendAsync(requestMessage).ConfigureAwait(false); + if (_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DistributionFileServerForceHTTP2), false)) + { + requestMessage.Version = new Version(2, 0); + requestMessage.VersionPolicy = HttpVersionPolicy.RequestVersionExact; + } + HttpResponseMessage? response = null; try { + response = await _httpClient.SendAsync(requestMessage).ConfigureAwait(false); response.EnsureSuccessStatusCode(); } catch (Exception ex) { _logger.LogWarning(ex, "Failed to download {url}", downloadUrl); + response?.Dispose(); return; } - var fileName = FilePathUtil.GetFilePath(_basePath, hash); - using var fileStream = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite); - var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024; + var tempFileName = destinationFilePath + ".dl"; + var fileStream = new FileStream(tempFileName, FileMode.Create, FileAccess.ReadWrite); + var bufferSize = 4096; var buffer = new byte[bufferSize]; var bytesRead = 0; - while ((bytesRead = await (await response.Content.ReadAsStreamAsync().ConfigureAwait(false)).ReadAsync(buffer).ConfigureAwait(false)) > 0) + using var content = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); + while ((bytesRead = await content.ReadAsync(buffer).ConfigureAwait(false)) > 0) { await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead)).ConfigureAwait(false); } + await fileStream.FlushAsync().ConfigureAwait(false); + await fileStream.DisposeAsync().ConfigureAwait(false); + File.Move(tempFileName, destinationFilePath, true); - _metrics.IncGauge(MetricsAPI.GaugeFilesTotal); - _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_basePath, hash).Length); + _metrics.IncGauge(_useColdStorage ? MetricsAPI.GaugeFilesTotalColdStorage : MetricsAPI.GaugeFilesTotal); + _metrics.IncGauge(_useColdStorage ? MetricsAPI.GaugeFilesTotalSizeColdStorage : MetricsAPI.GaugeFilesTotalSize, new FileInfo(destinationFilePath).Length); + response.Dispose(); } - public void DownloadFileWhenRequired(string hash) + private bool TryCopyFromColdStorage(string hash, string destinationFilePath) { - var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash); - if (fi == null && IsMainServer) return; + if (!_useColdStorage) return false; - if (fi == null && !_currentTransfers.ContainsKey(hash)) + if (string.IsNullOrEmpty(_coldStoragePath)) return false; + + var coldStorageFilePath = FilePathUtil.GetFilePath(_coldStoragePath, hash); + if (coldStorageFilePath == null) return false; + + try + { + _logger.LogDebug("Copying {hash} from cold storage: {path}", hash, coldStorageFilePath); + var tempFileName = destinationFilePath + ".dl"; + File.Copy(coldStorageFilePath, tempFileName, true); + File.Move(tempFileName, destinationFilePath, true); + var destinationFile = new FileInfo(destinationFilePath); + destinationFile.LastAccessTimeUtc = DateTime.UtcNow; + destinationFile.CreationTimeUtc = DateTime.UtcNow; + destinationFile.LastWriteTimeUtc = DateTime.UtcNow; + _metrics.IncGauge(MetricsAPI.GaugeFilesTotal); + _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, new FileInfo(destinationFilePath).Length); + return true; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not copy {coldStoragePath} from cold storage", coldStorageFilePath); + } + + return false; + } + + public async Task DownloadFileWhenRequired(string hash) + { + var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash); + + if (fi != null && fi.Length != 0) + return; + + // first check cold storage + if (TryCopyFromColdStorage(hash, FilePathUtil.GetFilePath(_hotStoragePath, hash))) + return; + + // no distribution server configured to download from + if (_remoteCacheSourceUri == null) + return; + + await _downloadSemaphore.WaitAsync().ConfigureAwait(false); + if (!_currentTransfers.TryGetValue(hash, out var downloadTask) || (downloadTask?.IsCompleted ?? true)) { _currentTransfers[hash] = Task.Run(async () => { - await DownloadTask(hash).ConfigureAwait(false); - _currentTransfers.Remove(hash, out _); + try + { + _metrics.IncGauge(MetricsAPI.GaugeFilesDownloadingFromCache); + await DownloadTask(hash).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error during Download Task for {hash}", hash); + } + finally + { + _metrics.DecGauge(MetricsAPI.GaugeFilesDownloadingFromCache); + _currentTransfers.Remove(hash, out _); + } }); } + _downloadSemaphore.Release(); } public FileStream? GetLocalFileStream(string hash) { - var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash); + var fi = FilePathUtil.GetFileInfoForHash(_hotStoragePath, hash); if (fi == null) return null; + fi.LastAccessTimeUtc = DateTime.UtcNow; + + _touchService.TouchColdHash(hash); _fileStatisticsService.LogFile(hash, fi.Length); @@ -93,13 +192,38 @@ public class CachedFileProvider public async Task GetAndDownloadFileStream(string hash) { - DownloadFileWhenRequired(hash); + await DownloadFileWhenRequired(hash).ConfigureAwait(false); if (_currentTransfers.TryGetValue(hash, out var downloadTask)) { - await downloadTask.ConfigureAwait(false); + try + { + using CancellationTokenSource cts = new(); + cts.CancelAfter(TimeSpan.FromSeconds(120)); + _metrics.IncGauge(MetricsAPI.GaugeFilesTasksWaitingForDownloadFromCache); + await downloadTask.WaitAsync(cts.Token).ConfigureAwait(false); + } + catch (Exception e) + { + _logger.LogWarning(e, "Failed while waiting for download task for {hash}", hash); + return null; + } + finally + { + _metrics.DecGauge(MetricsAPI.GaugeFilesTasksWaitingForDownloadFromCache); + } } return GetLocalFileStream(hash); } + + public void TouchColdHash(string hash) + { + _touchService.TouchColdHash(hash); + } + + public bool AnyFilesDownloading(List hashes) + { + return hashes.Exists(_currentTransfers.Keys.Contains); + } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ColdTouchHashService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ColdTouchHashService.cs new file mode 100644 index 0000000..0ebffa6 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ColdTouchHashService.cs @@ -0,0 +1,68 @@ +using MareSynchronosShared.Services; +using MareSynchronosStaticFilesServer.Utils; + +namespace MareSynchronosStaticFilesServer.Services; + +// Perform access time updates for cold cache files accessed via hot cache or shard servers +public class ColdTouchHashService : ITouchHashService +{ + private readonly ILogger _logger; + private readonly IConfigurationService _configuration; + + private readonly bool _useColdStorage; + private readonly string _coldStoragePath; + + // Debounce multiple updates towards the same file + private readonly Dictionary _lastUpdateTimesUtc = new(1009, StringComparer.Ordinal); + private int _cleanupCounter = 0; + private const double _debounceTimeSecs = 90.0; + + public ColdTouchHashService(ILogger logger, IConfigurationService configuration) + { + _logger = logger; + _configuration = configuration; + _useColdStorage = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); + _coldStoragePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)); + } + + public Task StartAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public void TouchColdHash(string hash) + { + if (!_useColdStorage) + return; + + var nowUtc = DateTime.UtcNow; + + // Clean up debounce dictionary regularly + if (_cleanupCounter++ >= 1000) + { + foreach (var entry in _lastUpdateTimesUtc.Where(entry => (nowUtc - entry.Value).TotalSeconds >= _debounceTimeSecs).ToList()) + _lastUpdateTimesUtc.Remove(entry.Key); + _cleanupCounter = 0; + } + + // Ignore multiple updates within a 90 second window of the first + if (_lastUpdateTimesUtc.TryGetValue(hash, out var lastUpdateTimeUtc) && (nowUtc - lastUpdateTimeUtc).TotalSeconds < _debounceTimeSecs) + { + _logger.LogDebug($"Debounced touch for {hash}"); + return; + } + + var fileInfo = FilePathUtil.GetFileInfoForHash(_coldStoragePath, hash); + if (fileInfo != null) + { + _logger.LogDebug($"Touching {fileInfo.Name}"); + fileInfo.LastAccessTimeUtc = nowUtc; + _lastUpdateTimesUtc.TryAdd(hash, nowUtc); + } + } +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs index c15f216..5ede841 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs @@ -1,79 +1,77 @@ using ByteSizeLib; +using MareSynchronos.API.Dto.Files; using MareSynchronosShared.Data; using MareSynchronosShared.Metrics; using MareSynchronosShared.Models; using MareSynchronosShared.Services; using MareSynchronosStaticFilesServer.Utils; +using MessagePack.Formatters; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Hosting.Systemd; namespace MareSynchronosStaticFilesServer.Services; public class FileCleanupService : IHostedService { - private readonly string _cacheDir; private readonly IConfigurationService _configuration; - private readonly bool _isMainServer; private readonly ILogger _logger; private readonly MareMetrics _metrics; private readonly IServiceProvider _services; + + private readonly string _hotStoragePath; + private readonly string _coldStoragePath; + private readonly bool _isMain = false; + private readonly bool _isDistributionNode = false; + private readonly bool _useColdStorage = false; + private HashSet _orphanedFiles = new(StringComparer.Ordinal); + private CancellationTokenSource _cleanupCts; - public FileCleanupService(MareMetrics metrics, ILogger logger, IServiceProvider services, IConfigurationService configuration) + private int HotStorageMinimumRetention => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.MinimumFileRetentionPeriodInDays), 7); + private int HotStorageRetention => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14); + private double HotStorageSize => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1.0); + + private int ColdStorageMinimumRetention => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageMinimumFileRetentionPeriodInDays), 60); + private int ColdStorageRetention => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60); + private double ColdStorageSize => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1.0); + + private double SmallSizeKiB => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheSmallSizeThresholdKiB), 64.0); + private double LargeSizeKiB => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheLargeSizeThresholdKiB), 1024.0); + + private int ForcedDeletionAfterHours => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1); + private int CleanupCheckMinutes => _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CleanupCheckInMinutes), 15); + + private List GetAllHotFiles() => new DirectoryInfo(_hotStoragePath).GetFiles("*", SearchOption.AllDirectories) + .Where(f => f != null && f.Name.Length == 40) + .OrderBy(f => f.LastAccessTimeUtc).ToList(); + + private List GetAllColdFiles() => new DirectoryInfo(_coldStoragePath).GetFiles("*", SearchOption.AllDirectories) + .Where(f => f != null && f.Name.Length == 40) + .OrderBy(f => f.LastAccessTimeUtc).ToList(); + + private List GetTempFiles() => new DirectoryInfo(_useColdStorage ? _coldStoragePath : _hotStoragePath).GetFiles("*", SearchOption.AllDirectories) + .Where(f => f != null && (f.Name.EndsWith(".dl", StringComparison.InvariantCultureIgnoreCase) || f.Name.EndsWith(".tmp", StringComparison.InvariantCultureIgnoreCase))).ToList(); + + public FileCleanupService(MareMetrics metrics, ILogger logger, + IServiceProvider services, IConfigurationService configuration) { _metrics = metrics; _logger = logger; _services = services; _configuration = configuration; - _isMainServer = configuration.IsMain; - _cacheDir = _configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); - } - - public async Task CleanUpTask(CancellationToken ct) - { - _logger.LogInformation("Starting periodic cleanup task"); - - while (!ct.IsCancellationRequested) - { - try - { - DirectoryInfo dir = new(_cacheDir); - var allFiles = dir.GetFiles("*", SearchOption.AllDirectories); - _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFiles.Sum(f => f.Length)); - _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFiles.Length); - - using var scope = _services.CreateScope(); - using var dbContext = scope.ServiceProvider.GetService()!; - - await CleanUpOutdatedFiles(dbContext, ct).ConfigureAwait(false); - - CleanUpFilesBeyondSizeLimit(dbContext, ct); - - if (_isMainServer) - { - CleanUpStuckUploads(dbContext); - - await dbContext.SaveChangesAsync(ct).ConfigureAwait(false); - } - } - catch (Exception e) - { - _logger.LogError(e, "Error during cleanup task"); - } - - var now = DateTime.Now; - TimeOnly currentTime = new(now.Hour, now.Minute, now.Second); - TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % 10, 0); - var span = futureTime.AddMinutes(10) - currentTime; - - _logger.LogInformation("File Cleanup Complete, next run at {date}", now.Add(span)); - await Task.Delay(span, ct).ConfigureAwait(false); - } + _useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); + _hotStoragePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); + _coldStoragePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)); + _isDistributionNode = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.IsDistributionNode), false); + _isMain = configuration.GetValue(nameof(StaticFilesServerConfiguration.MainFileServerAddress)) == null && _isDistributionNode; } public Task StartAsync(CancellationToken cancellationToken) { _logger.LogInformation("Cleanup Service started"); + InitializeGauges(); + _cleanupCts = new(); _ = CleanUpTask(_cleanupCts.Token); @@ -88,145 +86,273 @@ public class FileCleanupService : IHostedService return Task.CompletedTask; } - private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct) + private List CleanUpFilesBeyondSizeLimit(List files, double sizeLimit, double minTTL, double maxTTL, CancellationToken ct) { - var sizeLimit = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1); + var removedFiles = new List(); if (sizeLimit <= 0) { - return; + return removedFiles; + } + + var smallSize = SmallSizeKiB * 1024.0; + var largeSize = LargeSizeKiB * 1024.0; + var now = DateTime.Now; + + // Avoid nonsense in future calculations + if (smallSize < 0.0) + smallSize = 0.0; + + if (largeSize < smallSize) + largeSize = smallSize; + + if (minTTL < 0.0) + minTTL = 0.0; + + if (maxTTL < minTTL) + maxTTL = minTTL; + + // Calculates a deletion priority to prioritize deletion of larger files over a configured TTL range based on a file's size. + // This is intended to be applied to the hot cache, as the cost of recovering many small files is greater than a single large file. + // Example (minTTL=7, maxTTL=30): + // - A 10MB file was last accessed 5 days ago. Its calculated optimum TTL is 7 days. result = 0.7143 + // - A 50kB file was last accessed 10 days ago. Its calculated optimum TTL is 30 days. result = 0.3333 + // The larger file will be deleted with a higher priority than the smaller file. + double CalculateTTLProgression(FileInfo file) + { + var fileLength = (double)file.Length; + var fileAgeDays = (now - file.LastAccessTime).TotalDays; + var sizeNorm = Math.Clamp((fileLength - smallSize) / (largeSize - smallSize), 0.0, 1.0); + // Using Math.Sqrt(sizeNorm) would create a more logical scaling curve, but it barely matters + var ttlDayRange = (maxTTL - minTTL) * (1.0 - sizeNorm); + var daysPastMinTTL = Math.Max(fileAgeDays - minTTL, 0.0); + // There is some creativity in choosing an upper bound here: + // - With no upper bound, any file larger than `largeSize` is always the highest priority for deletion once it passes its calculated TTL + // - With 1.0 as an upper bound, all files older than `maxTTL` will have the same priority regardless of size + // - Using maxTTL/minTTL chooses a logical cut-off point where any files old enough to be affected would have been cleaned up already + var ttlProg = Math.Clamp(daysPastMinTTL / ttlDayRange, 0.0, maxTTL / minTTL); + return ttlProg; } try { - _logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", sizeLimit); - var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories) - .Select(f => new FileInfo(f)).ToList() - .OrderBy(f => f.LastAccessTimeUtc).ToList(); - var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length); - long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes; - while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested) + // Since we already have the file list sorted by access time, the list index is incorporated in to + // the dictionary key to preserve it as a secondary ordering + var sortedFiles = new PriorityQueue(); + + foreach (var (file, i) in files.Select((file, i) => ( file, i ))) { - var oldestFile = allLocalFiles[0]; - allLocalFiles.Remove(oldestFile); - totalCacheSizeInBytes -= oldestFile.Length; - _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, oldestFile.Length); - _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); - _logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes); - oldestFile.Delete(); - if (_isMainServer) - { - FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() }; - dbContext.Entry(f).State = EntityState.Deleted; - } + double ttlProg = CalculateTTLProgression(file); + sortedFiles.Enqueue(file, (-ttlProg, i)); } + + _logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", sizeLimit); + var totalCacheSizeInBytes = files.Sum(s => s.Length); + long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes; + while (totalCacheSizeInBytes > cacheSizeLimitInBytes && sortedFiles.Count != 0 && !ct.IsCancellationRequested) + { + var file = sortedFiles.Dequeue(); + totalCacheSizeInBytes -= file.Length; + _logger.LogInformation("Deleting {file} with size {size:N2}MiB", file.FullName, ByteSize.FromBytes(file.Length).MebiBytes); + file.Delete(); + removedFiles.Add(file.Name); + } + files.RemoveAll(f => removedFiles.Contains(f.Name, StringComparer.InvariantCultureIgnoreCase)); } catch (Exception ex) { _logger.LogWarning(ex, "Error during cache size limit cleanup"); } + + return removedFiles; } - private void CleanUpOrphanedFiles(List allFiles, FileInfo[] allPhysicalFiles, CancellationToken ct) + private void CleanUpOrphanedFiles(HashSet allDbFileHashes, List allPhysicalFiles, CancellationToken ct) { - if (_isMainServer) + // To avoid race conditions with file uploads, only delete files on a second pass + var newOrphanedFiles = new HashSet(StringComparer.Ordinal); + + foreach (var file in allPhysicalFiles.ToList()) { - var allFilesHashes = new HashSet(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal); - foreach (var file in allPhysicalFiles) + if (!allDbFileHashes.Contains(file.Name.ToUpperInvariant())) { - if (!allFilesHashes.Contains(file.Name.ToUpperInvariant())) - { - _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); - _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); - file.Delete(); - _logger.LogInformation("File not in DB, deleting: {fileName}", file.Name); - } - - ct.ThrowIfCancellationRequested(); + _logger.LogInformation("File not in DB, marking: {fileName}", file.Name); + newOrphanedFiles.Add(file.FullName); } + + ct.ThrowIfCancellationRequested(); } + + foreach (var fullName in _orphanedFiles.Where(f => newOrphanedFiles.Contains(f))) + { + var name = Path.GetFileName(fullName); + File.Delete(fullName); + _logger.LogInformation("File still not in DB, deleting: {fileName}", name); + allPhysicalFiles.RemoveAll(f => f.FullName.Equals(fullName, StringComparison.InvariantCultureIgnoreCase)); + } + + _orphanedFiles = newOrphanedFiles; } - private async Task CleanUpOutdatedFiles(MareDbContext dbContext, CancellationToken ct) + private List CleanUpOutdatedFiles(List files, int unusedRetention, int forcedDeletionAfterHours, CancellationToken ct) { + var removedFiles = new List(); try { - var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14); - var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1); - _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", unusedRetention); if (forcedDeletionAfterHours > 0) { _logger.LogInformation("Cleaning up files written to longer than {hours}h ago", forcedDeletionAfterHours); } - // clean up files in DB but not on disk or last access is expired - var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention)); - var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours)); - DirectoryInfo dir = new(_cacheDir); - var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories); - var allFiles = await dbContext.Files.ToListAsync().ConfigureAwait(false); - int fileCounter = 0; - foreach (var fileCache in allFiles.Where(f => f.Uploaded)) + var lastAccessCutoffTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention)); + var forcedDeletionCutoffTime = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours)); + + foreach (var file in files) { - var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash); - bool fileDeleted = false; - if (file == null && _isMainServer) + if (file.LastAccessTime < lastAccessCutoffTime) { - _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash); - dbContext.Files.Remove(fileCache); - fileDeleted = true; - } - else if (file != null && file.LastAccessTime < prevTime) - { - _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); - _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); - _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); + _logger.LogInformation("File outdated: {fileName}, {fileSize:N2}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); file.Delete(); - if (_isMainServer) - { - fileDeleted = true; - dbContext.Files.Remove(fileCache); - } + removedFiles.Add(file.Name); } - else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion) + else if (forcedDeletionAfterHours > 0 && file.LastWriteTime < forcedDeletionCutoffTime) { - _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); - _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); - _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); + _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize:N2}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); file.Delete(); - if (_isMainServer) - { - fileDeleted = true; - dbContext.Files.Remove(fileCache); - } + removedFiles.Add(file.Name); } - if (_isMainServer && !fileDeleted && file != null && fileCache.Size == 0) - { - _logger.LogInformation("Setting File Size of " + fileCache.Hash + " to " + file.Length); - fileCache.Size = file.Length; - // commit every 1000 files to db - if (fileCounter % 1000 == 0) await dbContext.SaveChangesAsync().ConfigureAwait(false); - } - - fileCounter++; - ct.ThrowIfCancellationRequested(); } - - // clean up files that are on disk but not in DB for some reason - CleanUpOrphanedFiles(allFiles, allFilesInDir, ct); + files.RemoveAll(f => removedFiles.Contains(f.Name, StringComparer.InvariantCultureIgnoreCase)); } catch (Exception ex) { _logger.LogWarning(ex, "Error during file cleanup of old files"); } + + return removedFiles; } - private void CleanUpStuckUploads(MareDbContext dbContext) + private void CleanUpTempFiles() { - var pastTime = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10)); - var stuckUploads = dbContext.Files.Where(f => !f.Uploaded && f.UploadDate < pastTime); - dbContext.Files.RemoveRange(stuckUploads); + var pastTime = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(20)); + var tempFiles = GetTempFiles(); + foreach (var tempFile in tempFiles.Where(f => f.LastWriteTimeUtc < pastTime)) + tempFile.Delete(); + } + + private async Task CleanUpTask(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + try + { + using var scope = _services.CreateScope(); + using var dbContext = _isMain ? scope.ServiceProvider.GetService()! : null; + + HashSet allDbFileHashes = null; + + // Database operations only performed on main server + if (_isMain) + { + var allDbFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false); + allDbFileHashes = new HashSet(allDbFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal); + } + + if (_useColdStorage) + { + var coldFiles = GetAllColdFiles(); + var removedColdFiles = new List(); + + removedColdFiles.AddRange( + CleanUpOutdatedFiles(coldFiles, ColdStorageRetention, ForcedDeletionAfterHours, ct) + ); + removedColdFiles.AddRange( + CleanUpFilesBeyondSizeLimit(coldFiles, ColdStorageSize, ColdStorageMinimumRetention, ColdStorageRetention, ct) + ); + + // Remove cold storage files are deleted from the database, if we are the main file server + if (_isMain) + { + dbContext.Files.RemoveRange( + dbContext.Files.Where(f => removedColdFiles.Contains(f.Hash)) + ); + allDbFileHashes.ExceptWith(removedColdFiles); + CleanUpOrphanedFiles(allDbFileHashes, coldFiles, ct); + } + + // Remove hot copies of files now that the authoritative copy is gone + foreach (var removedFile in removedColdFiles) + { + var hotFile = FilePathUtil.GetFileInfoForHash(_hotStoragePath, removedFile); + hotFile?.Delete(); + } + + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, coldFiles.Sum(f => { try { return f.Length; } catch { return 0; } })); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, coldFiles.Count); + } + + var hotFiles = GetAllHotFiles(); + var removedHotFiles = new List(); + + removedHotFiles.AddRange( + CleanUpOutdatedFiles(hotFiles, HotStorageRetention, forcedDeletionAfterHours: _useColdStorage ? ForcedDeletionAfterHours : -1, ct) + ); + removedHotFiles.AddRange( + CleanUpFilesBeyondSizeLimit(hotFiles, HotStorageSize, HotStorageMinimumRetention, HotStorageRetention, ct) + ); + + if (_isMain) + { + // If cold storage is not active, then "hot" files are deleted from the database instead + if (!_useColdStorage) + { + dbContext.Files.RemoveRange( + dbContext.Files.Where(f => removedHotFiles.Contains(f.Hash)) + ); + allDbFileHashes.ExceptWith(removedHotFiles); + } + + CleanUpOrphanedFiles(allDbFileHashes, hotFiles, ct); + + await dbContext.SaveChangesAsync(ct).ConfigureAwait(false); + } + + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, hotFiles.Sum(f => { try { return f.Length; } catch { return 0; } })); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, hotFiles.Count); + + CleanUpTempFiles(); + } + catch (Exception e) + { + _logger.LogError(e, "Error during cleanup task"); + } + + var cleanupCheckMinutes = CleanupCheckMinutes; + var now = DateTime.Now; + TimeOnly currentTime = new(now.Hour, now.Minute, now.Second); + TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % cleanupCheckMinutes, 0); + var span = futureTime.AddMinutes(cleanupCheckMinutes) - currentTime; + + _logger.LogInformation("File Cleanup Complete, next run at {date}", now.Add(span)); + await Task.Delay(span, ct).ConfigureAwait(false); + } + } + + private void InitializeGauges() + { + if (_useColdStorage) + { + var allFilesInColdStorageDir = GetAllColdFiles(); + + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSizeColdStorage, allFilesInColdStorageDir.Sum(f => f.Length)); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalColdStorage, allFilesInColdStorageDir.Count); + } + + var allFilesInHotStorage = GetAllHotFiles(); + + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFilesInHotStorage.Sum(f => { try { return f.Length; } catch { return 0; } })); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFilesInHotStorage.Count); } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/IClientReadyMessageService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/IClientReadyMessageService.cs new file mode 100644 index 0000000..932cf51 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/IClientReadyMessageService.cs @@ -0,0 +1,6 @@ +namespace MareSynchronosStaticFilesServer.Services; + +public interface IClientReadyMessageService +{ + void SendDownloadReady(string uid, Guid requestId); +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ITouchHashService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ITouchHashService.cs new file mode 100644 index 0000000..679c19e --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ITouchHashService.cs @@ -0,0 +1,6 @@ +namespace MareSynchronosStaticFilesServer.Services; + +public interface ITouchHashService : IHostedService +{ + void TouchColdHash(string hash); +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainClientReadyMessageService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainClientReadyMessageService.cs new file mode 100644 index 0000000..17149c9 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainClientReadyMessageService.cs @@ -0,0 +1,26 @@ +using Microsoft.AspNetCore.SignalR; +using MareSynchronos.API.SignalR; +using MareSynchronosServer.Hubs; + +namespace MareSynchronosStaticFilesServer.Services; + +public class MainClientReadyMessageService : IClientReadyMessageService +{ + private readonly ILogger _logger; + private readonly IHubContext _mareHub; + + public MainClientReadyMessageService(ILogger logger, IHubContext mareHub) + { + _logger = logger; + _mareHub = mareHub; + } + + public void SendDownloadReady(string uid, Guid requestId) + { + _ = Task.Run(async () => + { + _logger.LogDebug("Sending Client Ready for {uid}:{requestId} to SignalR", uid, requestId); + await _mareHub.Clients.User(uid).SendAsync(nameof(IMareHub.Client_DownloadReady), requestId).ConfigureAwait(false); + }); + } +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs index f271591..0458e57 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs @@ -1,37 +1,29 @@ using MareSynchronosShared.Metrics; using MareSynchronosShared.Services; using MareSynchronosStaticFilesServer.Utils; -using Microsoft.AspNetCore.SignalR; using System.Collections.Concurrent; -using System.Timers; -using MareSynchronos.API.SignalR; -using MareSynchronosShared.Data; -using Microsoft.EntityFrameworkCore; using System.Linq; +using System.Timers; namespace MareSynchronosStaticFilesServer.Services; public class RequestQueueService : IHostedService { - private record PriorityEntry(bool IsHighPriority, DateTime LastChecked); - - private readonly IHubContext _hubContext; + private readonly IClientReadyMessageService _clientReadyMessageService; + private readonly CachedFileProvider _cachedFileProvider; private readonly ILogger _logger; private readonly MareMetrics _metrics; private readonly ConcurrentQueue _queue = new(); private readonly ConcurrentQueue _priorityQueue = new(); private readonly int _queueExpirationSeconds; private readonly SemaphoreSlim _queueProcessingSemaphore = new(1); - private readonly ConcurrentDictionary _queueRemoval = new(); - private readonly SemaphoreSlim _queueSemaphore = new(1); private readonly UserQueueEntry[] _userQueueRequests; - private readonly ConcurrentDictionary _priorityCache = new(StringComparer.Ordinal); private int _queueLimitForReset; private readonly int _queueReleaseSeconds; private System.Timers.Timer _queueTimer; public RequestQueueService(MareMetrics metrics, IConfigurationService configurationService, - ILogger logger, IHubContext hubContext) + ILogger logger, IClientReadyMessageService hubContext, CachedFileProvider cachedFileProvider) { _userQueueRequests = new UserQueueEntry[configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueSize), 50)]; _queueExpirationSeconds = configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadTimeoutSeconds), 5); @@ -39,7 +31,8 @@ public class RequestQueueService : IHostedService _queueReleaseSeconds = configurationService.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DownloadQueueReleaseSeconds), 15); _metrics = metrics; _logger = logger; - _hubContext = hubContext; + _clientReadyMessageService = hubContext; + _cachedFileProvider = cachedFileProvider; } public void ActivateRequest(Guid request) @@ -49,50 +42,16 @@ public class RequestQueueService : IHostedService req.MarkActive(); } - private async Task IsHighPriority(string uid, MareDbContext mareDbContext) + public async Task EnqueueUser(UserRequest request, bool isPriority, CancellationToken token) { - return false; - if (!_priorityCache.TryGetValue(uid, out PriorityEntry entry) || entry.LastChecked.Add(TimeSpan.FromHours(6)) < DateTime.UtcNow) + while (_queueProcessingSemaphore.CurrentCount == 0) { - var user = await mareDbContext.Users.FirstOrDefaultAsync(u => u.UID == uid).ConfigureAwait(false); - entry = new(user != null && !string.IsNullOrEmpty(user.Alias), DateTime.UtcNow); - _priorityCache[uid] = entry; + await Task.Delay(50, token).ConfigureAwait(false); } - return entry.IsHighPriority; - } - - public async Task EnqueueUser(UserRequest request, MareDbContext mareDbContext) - { _logger.LogDebug("Enqueueing req {guid} from {user} for {file}", request.RequestId, request.User, string.Join(", ", request.FileIds)); - bool isPriorityQueue = await IsHighPriority(request.User, mareDbContext).ConfigureAwait(false); - - if (_queueProcessingSemaphore.CurrentCount == 0) - { - if (isPriorityQueue) _priorityQueue.Enqueue(request); - else _queue.Enqueue(request); - return; - } - - try - { - await _queueSemaphore.WaitAsync().ConfigureAwait(false); - if (isPriorityQueue) _priorityQueue.Enqueue(request); - else _queue.Enqueue(request); - - return; - } - catch (Exception ex) - { - _logger.LogError(ex, "Error during EnqueueUser"); - } - finally - { - _queueSemaphore.Release(); - } - - throw new Exception("Error during EnqueueUser"); + GetQueue(isPriority).Enqueue(request); } public void FinishRequest(Guid request) @@ -117,9 +76,10 @@ public class RequestQueueService : IHostedService return userQueueRequest != null && userRequest != null && userQueueRequest.ExpirationDate > DateTime.UtcNow; } - public void RemoveFromQueue(Guid requestId, string user) + public void RemoveFromQueue(Guid requestId, string user, bool isPriority) { - if (!_queue.Any(f => f.RequestId == requestId && string.Equals(f.User, user, StringComparison.Ordinal))) + var existingRequest = GetQueue(isPriority).FirstOrDefault(f => f.RequestId == requestId && string.Equals(f.User, user, StringComparison.Ordinal)); + if (existingRequest == null) { var activeSlot = _userQueueRequests.FirstOrDefault(r => r != null && string.Equals(r.UserRequest.User, user, StringComparison.Ordinal) && r.UserRequest.RequestId == requestId); if (activeSlot != null) @@ -130,29 +90,27 @@ public class RequestQueueService : IHostedService _userQueueRequests[idx] = null; } } - - return; } - _queueRemoval[requestId] = user; + else + { + existingRequest.IsCancelled = true; + } } public Task StartAsync(CancellationToken cancellationToken) { - _queueTimer = new System.Timers.Timer(250); + _queueTimer = new System.Timers.Timer(500); _queueTimer.Elapsed += ProcessQueue; _queueTimer.AutoReset = true; _queueTimer.Start(); return Task.CompletedTask; } - public async Task StillEnqueued(Guid request, string user, MareDbContext mareDbContext) + private ConcurrentQueue GetQueue(bool isPriority) => isPriority ? _priorityQueue : _queue; + + public bool StillEnqueued(Guid request, string user, bool isPriority) { - bool isPriorityQueue = await IsHighPriority(user, mareDbContext).ConfigureAwait(false); - if (isPriorityQueue) - { - return _priorityQueue.Any(c => c.RequestId == request && string.Equals(c.User, user, StringComparison.Ordinal)); - } - return _queue.Any(c => c.RequestId == request && string.Equals(c.User, user, StringComparison.Ordinal)); + return GetQueue(isPriority).Any(c => c.RequestId == request && string.Equals(c.User, user, StringComparison.Ordinal)); } public Task StopAsync(CancellationToken cancellationToken) @@ -161,92 +119,88 @@ public class RequestQueueService : IHostedService return Task.CompletedTask; } - private async Task DequeueIntoSlotAsync(UserRequest userRequest, int slot) + private void DequeueIntoSlot(UserRequest userRequest, int slot) { _logger.LogDebug("Dequeueing {req} into {i}: {user} with {file}", userRequest.RequestId, slot, userRequest.User, string.Join(", ", userRequest.FileIds)); _userQueueRequests[slot] = new(userRequest, DateTime.UtcNow.AddSeconds(_queueExpirationSeconds)); - await _hubContext.Clients.User(userRequest.User).SendAsync(nameof(IMareHub.Client_DownloadReady), userRequest.RequestId).ConfigureAwait(false); + _clientReadyMessageService.SendDownloadReady(userRequest.User, userRequest.RequestId); } - private async void ProcessQueue(object src, ElapsedEventArgs e) + private void ProcessQueue(object src, ElapsedEventArgs e) { if (_queueProcessingSemaphore.CurrentCount == 0) return; - await _queueProcessingSemaphore.WaitAsync().ConfigureAwait(false); + _queueProcessingSemaphore.Wait(); try { - if (_queue.Count > _queueLimitForReset) + if (_queue.Count(c => !c.IsCancelled) > _queueLimitForReset) { _queue.Clear(); return; } - Parallel.For(0, _userQueueRequests.Length, new ParallelOptions() - { - MaxDegreeOfParallelism = 10, - }, - async (i) => + for (int i = 0; i < _userQueueRequests.Length; i++) { try { - if (_userQueueRequests[i] != null && ((!_userQueueRequests[i].IsActive && _userQueueRequests[i].ExpirationDate < DateTime.UtcNow))) + if (_userQueueRequests[i] != null + && (((!_userQueueRequests[i].IsActive && _userQueueRequests[i].ExpirationDate < DateTime.UtcNow)) + || (_userQueueRequests[i].IsActive && _userQueueRequests[i].ActivationDate < DateTime.UtcNow.Subtract(TimeSpan.FromSeconds(_queueReleaseSeconds)))) + ) { - _logger.LogDebug("Expiring inactive request {guid} slot {slot}", _userQueueRequests[i].UserRequest.RequestId, i); + _logger.LogDebug("Expiring request {guid} slot {slot}", _userQueueRequests[i].UserRequest.RequestId, i); _userQueueRequests[i] = null; } - if (_userQueueRequests[i] != null && (_userQueueRequests[i].IsActive && _userQueueRequests[i].ActivationDate < DateTime.UtcNow.Subtract(TimeSpan.FromSeconds(_queueReleaseSeconds)))) - { - _logger.LogDebug("Expiring active request {guid} slot {slot}", _userQueueRequests[i].UserRequest.RequestId, i); - _userQueueRequests[i] = null; - } + if (_userQueueRequests[i] != null) continue; - if (!_queue.Any()) return; - - if (_userQueueRequests[i] == null) + while (true) { - bool enqueued = false; - while (!enqueued) + if (!_priorityQueue.All(u => _cachedFileProvider.AnyFilesDownloading(u.FileIds)) + && _priorityQueue.TryDequeue(out var prioRequest)) { - if (_priorityQueue.TryDequeue(out var prioRequest)) + if (prioRequest.IsCancelled) { - if (_queueRemoval.TryGetValue(prioRequest.RequestId, out string user) && string.Equals(user, prioRequest.User, StringComparison.Ordinal)) - { - _logger.LogDebug("Request cancelled: {requestId} by {user}", prioRequest.RequestId, user); - _queueRemoval.Remove(prioRequest.RequestId, out _); - continue; - } - - await DequeueIntoSlotAsync(prioRequest, i).ConfigureAwait(false); - enqueued = true; - break; + continue; } - if (_queue.TryDequeue(out var request)) + if (_cachedFileProvider.AnyFilesDownloading(prioRequest.FileIds)) { - if (_queueRemoval.TryGetValue(request.RequestId, out string user) && string.Equals(user, request.User, StringComparison.Ordinal)) - { - _logger.LogDebug("Request cancelled: {requestId} by {user}", request.RequestId, user); - _queueRemoval.Remove(request.RequestId, out _); - continue; - } + _priorityQueue.Enqueue(prioRequest); + continue; + } - await DequeueIntoSlotAsync(request, i).ConfigureAwait(false); - enqueued = true; - } - else - { - enqueued = true; - } + DequeueIntoSlot(prioRequest, i); + break; } + + if (!_queue.All(u => _cachedFileProvider.AnyFilesDownloading(u.FileIds)) + && _queue.TryDequeue(out var request)) + { + if (request.IsCancelled) + { + continue; + } + + if (_cachedFileProvider.AnyFilesDownloading(request.FileIds)) + { + _queue.Enqueue(request); + continue; + } + + DequeueIntoSlot(request, i); + break; + } + + break; } } catch (Exception ex) { _logger.LogWarning(ex, "Error during inside queue processing"); } - }); + } } catch (Exception ex) { @@ -260,6 +214,9 @@ public class RequestQueueService : IHostedService _metrics.SetGaugeTo(MetricsAPI.GaugeQueueFree, _userQueueRequests.Count(c => c == null)); _metrics.SetGaugeTo(MetricsAPI.GaugeQueueActive, _userQueueRequests.Count(c => c != null && c.IsActive)); _metrics.SetGaugeTo(MetricsAPI.GaugeQueueInactive, _userQueueRequests.Count(c => c != null && !c.IsActive)); - _metrics.SetGaugeTo(MetricsAPI.GaugeDownloadQueue, _queue.Count); + _metrics.SetGaugeTo(MetricsAPI.GaugeDownloadQueue, _queue.Count(q => !q.IsCancelled)); + _metrics.SetGaugeTo(MetricsAPI.GaugeDownloadQueueCancelled, _queue.Count(q => q.IsCancelled)); + _metrics.SetGaugeTo(MetricsAPI.GaugeDownloadPriorityQueue, _priorityQueue.Count(q => !q.IsCancelled)); + _metrics.SetGaugeTo(MetricsAPI.GaugeDownloadPriorityQueueCancelled, _priorityQueue.Count(q => q.IsCancelled)); } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardClientReadyMessageService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardClientReadyMessageService.cs new file mode 100644 index 0000000..3e4a1c8 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardClientReadyMessageService.cs @@ -0,0 +1,50 @@ +using MareSynchronos.API.Routes; +using MareSynchronosShared.Services; +using MareSynchronosShared.Utils; +using System.Net.Http.Headers; + +namespace MareSynchronosStaticFilesServer.Services; + +public class ShardClientReadyMessageService : IClientReadyMessageService +{ + private readonly ILogger _logger; + private readonly ServerTokenGenerator _tokenGenerator; + private readonly IConfigurationService _configurationService; + private readonly HttpClient _httpClient; + + public ShardClientReadyMessageService(ILogger logger, ServerTokenGenerator tokenGenerator, IConfigurationService configurationService) + { + _logger = logger; + _tokenGenerator = tokenGenerator; + _configurationService = configurationService; + _httpClient = new(); + _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer", "1.0.0.0")); + } + + public void SendDownloadReady(string uid, Guid requestId) + { + _ = Task.Run(async () => + { + var mainUrlConfigKey = _configurationService.GetValue(nameof(StaticFilesServerConfiguration.NotifyMainServerDirectly)) + ? nameof(StaticFilesServerConfiguration.MainServerAddress) + : nameof(StaticFilesServerConfiguration.MainFileServerAddress); + var mainUrl = _configurationService.GetValue(mainUrlConfigKey); + var path = MareFiles.MainSendReadyFullPath(mainUrl, uid, requestId); + using HttpRequestMessage msg = new() + { + RequestUri = path + }; + msg.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _tokenGenerator.Token); + + _logger.LogDebug("Sending Client Ready for {uid}:{requestId} to {path}", uid, requestId, path); + try + { + using var result = await _httpClient.SendAsync(msg).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failure to send for {uid}:{requestId}", uid, requestId); + } + }); + } +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardTouchMessageService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardTouchMessageService.cs new file mode 100644 index 0000000..1c22e3e --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardTouchMessageService.cs @@ -0,0 +1,131 @@ +using MareSynchronos.API.Routes; +using MareSynchronosShared.Services; +using MareSynchronosShared.Utils; +using System.Net.Http.Headers; + +namespace MareSynchronosStaticFilesServer.Services; + +// Notify distribution server of file hashes downloaded via shards, so they are not prematurely purged from its cold cache +public class ShardTouchMessageService : ITouchHashService +{ + private readonly ILogger _logger; + private readonly ServerTokenGenerator _tokenGenerator; + private readonly IConfigurationService _configuration; + private readonly HttpClient _httpClient; + private readonly Uri _remoteCacheSourceUri; + private readonly HashSet _touchHashSet = new(); + private readonly ColdTouchHashService _nestedService = null; + + private CancellationTokenSource _touchmsgCts; + + public ShardTouchMessageService(ILogger logger, ILogger nestedLogger, + ServerTokenGenerator tokenGenerator, IConfigurationService configuration) + { + _logger = logger; + _tokenGenerator = tokenGenerator; + _configuration = configuration; + _remoteCacheSourceUri = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DistributionFileServerAddress), null); + _httpClient = new(); + _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer", "1.0.0.0")); + + if (configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false)) + { + _nestedService = new ColdTouchHashService(nestedLogger, configuration); + } + } + + public Task StartAsync(CancellationToken cancellationToken) + { + if (_remoteCacheSourceUri == null) + return Task.CompletedTask; + + _logger.LogInformation("Touch Message Service started"); + + _touchmsgCts = new(); + + _ = TouchMessageTask(_touchmsgCts.Token); + + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + if (_remoteCacheSourceUri == null) + return Task.CompletedTask; + + _touchmsgCts.Cancel(); + + return Task.CompletedTask; + } + + private async Task SendTouches(IEnumerable hashes) + { + var mainUrl = _remoteCacheSourceUri; + var path = new Uri(mainUrl, MareFiles.Distribution + "/touch"); + using HttpRequestMessage msg = new() + { + RequestUri = path + }; + msg.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _tokenGenerator.Token); + msg.Method = HttpMethod.Post; + msg.Content = JsonContent.Create(hashes); + if (_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.DistributionFileServerForceHTTP2), false)) + { + msg.Version = new Version(2, 0); + msg.VersionPolicy = HttpVersionPolicy.RequestVersionExact; + } + + _logger.LogDebug("Sending remote touch to {path}", path); + try + { + using var result = await _httpClient.SendAsync(msg).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failure to send touches for {hashChunk}", hashes); + } + } + + private async Task TouchMessageTask(CancellationToken ct) + { + List hashes; + + while (!ct.IsCancellationRequested) + { + try + { + lock (_touchHashSet) + { + hashes = _touchHashSet.ToList(); + _touchHashSet.Clear(); + } + if (hashes.Count > 0) + await SendTouches(hashes); + await Task.Delay(TimeSpan.FromSeconds(30), ct).ConfigureAwait(false); + } + catch (Exception e) + { + _logger.LogError(e, "Error during touch message task"); + } + } + + lock (_touchHashSet) + { + hashes = _touchHashSet.ToList(); + _touchHashSet.Clear(); + } + if (hashes.Count > 0) + await SendTouches(hashes); + } + + public void TouchColdHash(string hash) + { + if (_nestedService != null) + _nestedService.TouchColdHash(hash); + + lock (_touchHashSet) + { + _touchHashSet.Add(hash); + } + } +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs index 089ee24..904a3d9 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs @@ -26,6 +26,7 @@ namespace MareSynchronosStaticFilesServer; public class Startup { private bool _isMain; + private bool _isDistributionNode; private readonly ILogger _logger; public Startup(IConfiguration configuration, ILogger logger) @@ -33,7 +34,8 @@ public class Startup Configuration = configuration; _logger = logger; var mareSettings = Configuration.GetRequiredSection("MareSynchronos"); - _isMain = string.IsNullOrEmpty(mareSettings.GetValue(nameof(StaticFilesServerConfiguration.MainFileServerAddress), string.Empty)); + _isDistributionNode = mareSettings.GetValue(nameof(StaticFilesServerConfiguration.IsDistributionNode), false); + _isMain = string.IsNullOrEmpty(mareSettings.GetValue(nameof(StaticFilesServerConfiguration.MainFileServerAddress), string.Empty)) && _isDistributionNode; } public IConfiguration Configuration { get; } @@ -51,12 +53,15 @@ public class Startup var mareConfig = Configuration.GetRequiredSection("MareSynchronos"); + // metrics configuration services.AddSingleton(m => new MareMetrics(m.GetService>(), new List { MetricsAPI.CounterFileRequests, MetricsAPI.CounterFileRequestSize }, new List { + MetricsAPI.GaugeFilesTotalColdStorage, + MetricsAPI.GaugeFilesTotalSizeColdStorage, MetricsAPI.GaugeFilesTotalSize, MetricsAPI.GaugeFilesTotal, MetricsAPI.GaugeFilesUniquePastDay, @@ -65,26 +70,143 @@ public class Startup MetricsAPI.GaugeFilesUniquePastHourSize, MetricsAPI.GaugeCurrentDownloads, MetricsAPI.GaugeDownloadQueue, + MetricsAPI.GaugeDownloadQueueCancelled, + MetricsAPI.GaugeDownloadPriorityQueue, + MetricsAPI.GaugeDownloadPriorityQueueCancelled, MetricsAPI.GaugeQueueFree, MetricsAPI.GaugeQueueInactive, MetricsAPI.GaugeQueueActive, + MetricsAPI.GaugeFilesDownloadingFromCache, + MetricsAPI.GaugeFilesTasksWaitingForDownloadFromCache })); + + // generic services services.AddSingleton(); + services.AddHostedService(); services.AddSingleton(); services.AddSingleton(); - + services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(p => p.GetService()); services.AddHostedService(m => m.GetService()); - services.AddHostedService(); + services.AddSingleton, MareConfigurationServiceClient>(); + services.AddHostedService(p => (MareConfigurationServiceClient)p.GetService>()); - services.AddDbContextPool(options => + // specific services + if (_isMain) { - options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder => + services.AddSingleton(); + services.AddSingleton, MareConfigurationServiceServer>(); + services.AddDbContextPool(options => { - builder.MigrationsHistoryTable("_efmigrationshistory", "public"); - }).UseSnakeCaseNamingConvention(); - options.EnableThreadSafetyChecks(false); - }, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024)); + options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder => + { + builder.MigrationsHistoryTable("_efmigrationshistory", "public"); + }).UseSnakeCaseNamingConvention(); + options.EnableThreadSafetyChecks(false); + }, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024)); + var signalRServiceBuilder = services.AddSignalR(hubOptions => + { + hubOptions.MaximumReceiveMessageSize = long.MaxValue; + hubOptions.EnableDetailedErrors = true; + hubOptions.MaximumParallelInvocationsPerClient = 10; + hubOptions.StreamBufferCapacity = 200; + }).AddMessagePackProtocol(opt => + { + var resolver = CompositeResolver.Create(StandardResolverAllowPrivate.Instance, + BuiltinResolver.Instance, + AttributeFormatterResolver.Instance, + // replace enum resolver + DynamicEnumAsStringResolver.Instance, + DynamicGenericResolver.Instance, + DynamicUnionResolver.Instance, + DynamicObjectResolver.Instance, + PrimitiveObjectResolver.Instance, + // final fallback(last priority) + StandardResolver.Instance); + + opt.SerializerOptions = MessagePackSerializerOptions.Standard + .WithCompression(MessagePackCompression.Lz4Block) + .WithResolver(resolver); + }); + + // configure redis for SignalR + var redisConnection = mareConfig.GetValue(nameof(ServerConfiguration.RedisConnectionString), string.Empty); + signalRServiceBuilder.AddStackExchangeRedis(redisConnection, options => { }); + + var options = ConfigurationOptions.Parse(redisConnection); + + var endpoint = options.EndPoints[0]; + string address = ""; + int port = 0; + if (endpoint is DnsEndPoint dnsEndPoint) { address = dnsEndPoint.Host; port = dnsEndPoint.Port; } + if (endpoint is IPEndPoint ipEndPoint) { address = ipEndPoint.Address.ToString(); port = ipEndPoint.Port; } + var redisConfiguration = new RedisConfiguration() + { + AbortOnConnectFail = true, + KeyPrefix = "", + Hosts = new RedisHost[] + { + new RedisHost(){ Host = address, Port = port }, + }, + AllowAdmin = true, + ConnectTimeout = options.ConnectTimeout, + Database = 0, + Ssl = false, + Password = options.Password, + ServerEnumerationStrategy = new ServerEnumerationStrategy() + { + Mode = ServerEnumerationStrategy.ModeOptions.All, + TargetRole = ServerEnumerationStrategy.TargetRoleOptions.Any, + UnreachableServerAction = ServerEnumerationStrategy.UnreachableServerActionOptions.Throw, + }, + MaxValueLength = 1024, + PoolSize = mareConfig.GetValue(nameof(ServerConfiguration.RedisPool), 50), + SyncTimeout = options.SyncTimeout, + }; + + services.AddStackExchangeRedisExtensions(redisConfiguration); + } + else + { + services.AddSingleton(); + services.AddSingleton, MareConfigurationServiceClient>(); + services.AddHostedService(p => (MareConfigurationServiceClient)p.GetService>()); + } + + if (_isDistributionNode) + { + services.AddSingleton(); + services.AddHostedService(p => p.GetService()); + } + else + { + services.AddSingleton(); + services.AddHostedService(p => p.GetService()); + } + + // controller setup + services.AddControllers().ConfigureApplicationPartManager(a => + { + a.FeatureProviders.Remove(a.FeatureProviders.OfType().First()); + if (_isMain) + { + a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(MareStaticFilesServerConfigurationController), + typeof(CacheController), typeof(RequestController), typeof(ServerFilesController), + typeof(DistributionController), typeof(MainController))); + } + else if (_isDistributionNode) + { + a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(CacheController), typeof(RequestController), typeof(DistributionController))); + } + else + { + a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(CacheController), typeof(RequestController))); + } + }); + + // authentication and authorization services.AddOptions(JwtBearerDefaults.AuthenticationScheme) .Configure>((o, s) => { @@ -94,115 +216,21 @@ public class Startup ValidateLifetime = false, ValidateAudience = false, ValidateIssuerSigningKey = true, - IssuerSigningKey = new SymmetricSecurityKey(Encoding.ASCII.GetBytes(s.GetValue(nameof(MareConfigurationAuthBase.Jwt)))), + IssuerSigningKey = new SymmetricSecurityKey(Encoding.ASCII.GetBytes(s.GetValue(nameof(MareConfigurationAuthBase.Jwt)))) }; }); - services.AddAuthentication(o => { o.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme; o.DefaultChallengeScheme = JwtBearerDefaults.AuthenticationScheme; o.DefaultScheme = JwtBearerDefaults.AuthenticationScheme; }).AddJwtBearer(); - services.AddAuthorization(options => { options.FallbackPolicy = new AuthorizationPolicyBuilder().RequireAuthenticatedUser().Build(); options.AddPolicy("Internal", new AuthorizationPolicyBuilder().RequireClaim(MareClaimTypes.Internal, "true").Build()); }); - - if (_isMain) - { - services.AddSingleton, MareConfigurationServiceServer>(); - } - else - { - services.AddSingleton, MareConfigurationServiceClient>(); - services.AddHostedService(p => (MareConfigurationServiceClient)p.GetService>()); - } - - services.AddSingleton, MareConfigurationServiceClient>(); - - services.AddSingleton(); - services.AddSingleton(); - services.AddHostedService(p => p.GetService()); - services.AddControllers().ConfigureApplicationPartManager(a => - { - a.FeatureProviders.Remove(a.FeatureProviders.OfType().First()); - if (_isMain) - { - a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(MareStaticFilesServerConfigurationController), typeof(CacheController), typeof(RequestController), typeof(ServerFilesController))); - } - else - { - a.FeatureProviders.Add(new AllowedControllersFeatureProvider(typeof(CacheController), typeof(RequestController))); - } - }); - - services.AddHostedService(p => (MareConfigurationServiceClient)p.GetService>()); - services.AddSingleton(); - var signalRServiceBuilder = services.AddSignalR(hubOptions => - { - hubOptions.MaximumReceiveMessageSize = long.MaxValue; - hubOptions.EnableDetailedErrors = true; - hubOptions.MaximumParallelInvocationsPerClient = 10; - hubOptions.StreamBufferCapacity = 200; - }).AddMessagePackProtocol(opt => - { - var resolver = CompositeResolver.Create(StandardResolverAllowPrivate.Instance, - BuiltinResolver.Instance, - AttributeFormatterResolver.Instance, - // replace enum resolver - DynamicEnumAsStringResolver.Instance, - DynamicGenericResolver.Instance, - DynamicUnionResolver.Instance, - DynamicObjectResolver.Instance, - PrimitiveObjectResolver.Instance, - // final fallback(last priority) - StandardResolver.Instance); - - opt.SerializerOptions = MessagePackSerializerOptions.Standard - .WithCompression(MessagePackCompression.Lz4Block) - .WithResolver(resolver); - }); - - // configure redis for SignalR - var redisConnection = mareConfig.GetValue(nameof(ServerConfiguration.RedisConnectionString), string.Empty); - signalRServiceBuilder.AddStackExchangeRedis(redisConnection, options => { }); - - var options = ConfigurationOptions.Parse(redisConnection); - - var endpoint = options.EndPoints[0]; - string address = ""; - int port = 0; - if (endpoint is DnsEndPoint dnsEndPoint) { address = dnsEndPoint.Host; port = dnsEndPoint.Port; } - if (endpoint is IPEndPoint ipEndPoint) { address = ipEndPoint.Address.ToString(); port = ipEndPoint.Port; } - var redisConfiguration = new RedisConfiguration() - { - AbortOnConnectFail = true, - KeyPrefix = "", - Hosts = new RedisHost[] - { - new RedisHost(){ Host = address, Port = port }, - }, - AllowAdmin = true, - ConnectTimeout = options.ConnectTimeout, - Database = 0, - Ssl = false, - Password = options.Password, - ServerEnumerationStrategy = new ServerEnumerationStrategy() - { - Mode = ServerEnumerationStrategy.ModeOptions.All, - TargetRole = ServerEnumerationStrategy.TargetRoleOptions.Any, - UnreachableServerAction = ServerEnumerationStrategy.UnreachableServerActionOptions.Throw, - }, - MaxValueLength = 1024, - PoolSize = mareConfig.GetValue(nameof(ServerConfiguration.RedisPool), 50), - SyncTimeout = options.SyncTimeout, - }; - - services.AddStackExchangeRedisExtensions(redisConfiguration); services.AddHealthChecks(); services.AddHttpLogging(e => e = new Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions()); @@ -226,7 +254,11 @@ public class Startup app.UseEndpoints(e => { - e.MapHub("/dummyhub"); + if (_isMain) + { + e.MapHub("/dummyhub"); + } + e.MapControllers(); e.MapHealthChecks("/health").WithMetadata(new AllowAnonymousAttribute()); }); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/FilePathUtil.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/FilePathUtil.cs index 14a4a95..2dcc40a 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/FilePathUtil.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/FilePathUtil.cs @@ -1,13 +1,17 @@ -namespace MareSynchronosStaticFilesServer.Utils; +using System.Text.RegularExpressions; -public static class FilePathUtil +namespace MareSynchronosStaticFilesServer.Utils; + +public static partial class FilePathUtil { public static FileInfo GetFileInfoForHash(string basePath, string hash) { - FileInfo fi = new(Path.Combine(basePath, hash[0].ToString(), hash)); + if (hash.Length != 40 || !hash.All(char.IsAsciiLetterOrDigit)) throw new InvalidOperationException(); + + FileInfo fi = new(Path.Join(basePath, hash[0].ToString(), hash)); if (!fi.Exists) { - fi = new FileInfo(Path.Combine(basePath, hash)); + fi = new FileInfo(Path.Join(basePath, hash)); if (!fi.Exists) { return null; @@ -19,8 +23,10 @@ public static class FilePathUtil public static string GetFilePath(string basePath, string hash) { - var dirPath = Path.Combine(basePath, hash[0].ToString()); - var path = Path.Combine(dirPath, hash); + if (hash.Length != 40 || !hash.All(char.IsAsciiLetterOrDigit)) throw new InvalidOperationException(); + + var dirPath = Path.Join(basePath, hash[0].ToString()); + var path = Path.Join(dirPath, hash); if (!Directory.Exists(dirPath)) Directory.CreateDirectory(dirPath); return path; } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs index 207524d..31e7b0a 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/RequestFileStreamResult.cs @@ -34,6 +34,7 @@ public class RequestFileStreamResult : FileStreamResult _requestQueueService.FinishRequest(_requestId); _mareMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads); + FileStream?.Dispose(); } } @@ -50,8 +51,8 @@ public class RequestFileStreamResult : FileStreamResult finally { _requestQueueService.FinishRequest(_requestId); - _mareMetrics.DecGauge(MetricsAPI.GaugeCurrentDownloads); + FileStream?.Dispose(); } } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserRequest.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserRequest.cs index 4640c09..f64def1 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserRequest.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/UserRequest.cs @@ -1,3 +1,6 @@ namespace MareSynchronosStaticFilesServer.Utils; -public record UserRequest(Guid RequestId, string User, List FileIds); +public record UserRequest(Guid RequestId, string User, List FileIds) +{ + public bool IsCancelled { get; set; } = false; +} \ No newline at end of file