From f6af114d44bd3996487755a85cec44c733cad5f0 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Sat, 7 Jan 2023 11:36:20 +0100 Subject: [PATCH] add FileDbService to handle file requests --- .../Hubs/MareHub.Files.cs | 25 ++++--- .../MareSynchronosServer/Hubs/MareHub.cs | 4 +- .../Identity/IdentityHandler.cs | 2 +- .../Services/FileDbService.cs | 71 +++++++++++++++++++ .../MareSynchronosServer/Startup.cs | 2 + 5 files changed, 89 insertions(+), 15 deletions(-) create mode 100644 MareSynchronosServer/MareSynchronosServer/Services/FileDbService.cs diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs index 3966c9d..e7f127b 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs @@ -46,30 +46,29 @@ public partial class MareHub { _logger.LogCallInfo(MareHubLogger.Args(hashes.Count.ToString())); - var allFiles = await _dbContext.Files.Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false); - var forbiddenFiles = await _dbContext.ForbiddenUploadEntries. - Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false); + hashes = hashes.Distinct(StringComparer.Ordinal).ToList(); + var allFiles = hashes.Select(_fileDbService.GetFileCache).Select(t => t.Result).Where(t => t != null).ToList(); + var forbiddenFiles = await _dbContext.ForbiddenUploadEntries.Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false); List response = new(); - var cacheFile = await _dbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false); - var shardConfig = new List(_configurationService.GetValueOrDefault(nameof(ServerConfiguration.CdnShardConfiguration), new List())); - foreach (var file in cacheFile) + foreach (var file in hashes) { - var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase)); + var cacheFile = allFiles.SingleOrDefault(f => string.Equals(f.Hash, file, StringComparison.Ordinal)); + var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file, StringComparison.OrdinalIgnoreCase)); - var matchedShardConfig = shardConfig.OrderBy(g => Guid.NewGuid()).FirstOrDefault(f => new Regex(f.FileMatch).IsMatch(file.Hash)); + var matchedShardConfig = shardConfig.OrderBy(g => Guid.NewGuid()).FirstOrDefault(f => new Regex(f.FileMatch).IsMatch(file)); var baseUrl = matchedShardConfig?.CdnFullUrl ?? _mainCdnFullUrl; response.Add(new DownloadFileDto { - FileExists = file.Size > 0, + FileExists = cacheFile.Size > 0, ForbiddenBy = forbiddenFile?.ForbiddenBy ?? string.Empty, IsForbidden = forbiddenFile != null, - Hash = file.Hash, - Size = file.Size, - Url = new Uri(baseUrl, file.Hash.ToUpperInvariant()).ToString() + Hash = cacheFile.Hash, + Size = cacheFile.Size, + Url = new Uri(baseUrl, cacheFile.Hash.ToUpperInvariant()).ToString() }); } @@ -91,7 +90,7 @@ public partial class MareHub _logger.LogCallInfo(MareHubLogger.Args(userSentHashes.Count.ToString())); var notCoveredFiles = new Dictionary(StringComparer.Ordinal); var forbiddenFiles = await _dbContext.ForbiddenUploadEntries.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false); - var existingFiles = await _dbContext.Files.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false); + var existingFiles = fileListHashes.Select(_fileDbService.GetFileCache).Select(t => t.Result).Where(t => t != null).ToDictionary(k => k.Hash, k => k, StringComparer.Ordinal); var uploader = await _dbContext.Users.SingleAsync(u => u.UID == UserUID).ConfigureAwait(false); List fileCachesToUpload = new(); diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs index 9900ea8..1b93fa6 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs @@ -20,6 +20,7 @@ public partial class MareHub : Hub, IMareHub private readonly SystemInfoService _systemInfoService; private readonly IHttpContextAccessor _contextAccessor; private readonly IClientIdentificationService _clientIdentService; + private readonly FileDbService _fileDbService; private readonly MareHubLogger _logger; private readonly MareDbContext _dbContext; private readonly Uri _mainCdnFullUrl; @@ -32,7 +33,7 @@ public partial class MareHub : Hub, IMareHub public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient, MareDbContext mareDbContext, ILogger logger, SystemInfoService systemInfoService, IConfigurationService configuration, IHttpContextAccessor contextAccessor, - IClientIdentificationService clientIdentService) + IClientIdentificationService clientIdentService, FileDbService fileDbService) { _mareMetrics = mareMetrics; _fileServiceClient = fileServiceClient; @@ -45,6 +46,7 @@ public partial class MareHub : Hub, IMareHub _maxGroupUserCount = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxGroupUserCount), 100); _contextAccessor = contextAccessor; _clientIdentService = clientIdentService; + _fileDbService = fileDbService; _logger = new MareHubLogger(this, logger); _dbContext = mareDbContext; } diff --git a/MareSynchronosServer/MareSynchronosServer/Identity/IdentityHandler.cs b/MareSynchronosServer/MareSynchronosServer/Identity/IdentityHandler.cs index 2b52d58..9103fdb 100644 --- a/MareSynchronosServer/MareSynchronosServer/Identity/IdentityHandler.cs +++ b/MareSynchronosServer/MareSynchronosServer/Identity/IdentityHandler.cs @@ -65,7 +65,7 @@ public class IdentityHandler internal void EnqueueIdentChange(IdentChange identchange) { - _logger.LogInformation("Enqueued " + identchange.UidWithIdent.Uid.Uid + ":" + identchange.IsOnline + " from " + identchange.UidWithIdent.Ident.ServerId); + _logger.LogDebug("Enqueued " + identchange.UidWithIdent.Uid.Uid + ":" + identchange.IsOnline + " from " + identchange.UidWithIdent.Ident.ServerId); foreach (var k in _identChanges.Keys) { diff --git a/MareSynchronosServer/MareSynchronosServer/Services/FileDbService.cs b/MareSynchronosServer/MareSynchronosServer/Services/FileDbService.cs new file mode 100644 index 0000000..40bc77b --- /dev/null +++ b/MareSynchronosServer/MareSynchronosServer/Services/FileDbService.cs @@ -0,0 +1,71 @@ +using MareSynchronosShared.Data; +using MareSynchronosShared.Models; +using Microsoft.EntityFrameworkCore; + +namespace MareSynchronosServer.Services; + +public class FileDbService : IHostedService +{ + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + private Dictionary _fileCaches = new(StringComparer.Ordinal); + private readonly SemaphoreSlim _semaphore = new(5); + private readonly CancellationTokenSource _shutdownCancellationToken = new(); + + public FileDbService(IServiceProvider serviceProvider, ILogger logger) + { + _serviceProvider = serviceProvider; + _logger = logger; + } + + public async Task GetFileCache(string hash) + { + await _semaphore.WaitAsync().ConfigureAwait(false); + if (!_fileCaches.TryGetValue(hash, out var cache)) + { + using var db = _serviceProvider.GetService(); + cache = db.Files.AsNoTracking().SingleOrDefault(f => f.Hash == hash && f.Uploaded); + if (cache != null) _fileCaches[hash] = cache; + } + _semaphore.Release(); + + return cache; + } + + private async Task UpdateDatabasePeriodically(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + var now = DateTime.Now; + TimeOnly currentTime = new(now.Hour, now.Minute, now.Second); + TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % 10 + 5, 0); + var span = futureTime.AddMinutes(10) - currentTime; + + await Task.Delay(span, ct).ConfigureAwait(false); + + await UpdateDatabase().ConfigureAwait(false); + } + } + + private async Task UpdateDatabase() + { + using var scope = _serviceProvider.CreateScope(); + using var db = scope.ServiceProvider.GetRequiredService(); + _fileCaches = new(await db.Files.AsNoTracking().Where(f => f.Uploaded).AsNoTracking().ToDictionaryAsync(k => k.Hash, k => k).ConfigureAwait(false), StringComparer.Ordinal); + _logger.LogInformation("Updated FileCaches, now at {count}", _fileCaches.Count); + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + await UpdateDatabase().ConfigureAwait(false); + _ = UpdateDatabasePeriodically(_shutdownCancellationToken.Token); + _logger.LogInformation("Started FileDb Service, initially at {count} files from DB", _fileCaches.Count); + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _shutdownCancellationToken.Cancel(); + + return Task.CompletedTask; + } +} diff --git a/MareSynchronosServer/MareSynchronosServer/Startup.cs b/MareSynchronosServer/MareSynchronosServer/Startup.cs index f50d0ff..b38de8f 100644 --- a/MareSynchronosServer/MareSynchronosServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosServer/Startup.cs @@ -78,6 +78,8 @@ public class Startup services.AddSingleton(); services.AddSingleton(); services.AddHostedService(provider => provider.GetService()); + services.AddSingleton(); + services.AddHostedService(p => p.GetService()); // configure services based on main server status ConfigureServicesBasedOnShardType(services, mareConfig, isMainServer);