diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs index e7f127b..3966c9d 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs @@ -46,29 +46,30 @@ public partial class MareHub { _logger.LogCallInfo(MareHubLogger.Args(hashes.Count.ToString())); - hashes = hashes.Distinct(StringComparer.Ordinal).ToList(); - var allFiles = hashes.Select(_fileDbService.GetFileCache).Select(t => t.Result).Where(t => t != null).ToList(); - var forbiddenFiles = await _dbContext.ForbiddenUploadEntries.Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false); + var allFiles = await _dbContext.Files.Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false); + var forbiddenFiles = await _dbContext.ForbiddenUploadEntries. + Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false); List response = new(); + var cacheFile = await _dbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false); + var shardConfig = new List(_configurationService.GetValueOrDefault(nameof(ServerConfiguration.CdnShardConfiguration), new List())); - foreach (var file in hashes) + foreach (var file in cacheFile) { - var cacheFile = allFiles.SingleOrDefault(f => string.Equals(f.Hash, file, StringComparison.Ordinal)); - var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file, StringComparison.OrdinalIgnoreCase)); + var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase)); - var matchedShardConfig = shardConfig.OrderBy(g => Guid.NewGuid()).FirstOrDefault(f => new Regex(f.FileMatch).IsMatch(file)); + var matchedShardConfig = shardConfig.OrderBy(g => Guid.NewGuid()).FirstOrDefault(f => new Regex(f.FileMatch).IsMatch(file.Hash)); var baseUrl = matchedShardConfig?.CdnFullUrl ?? _mainCdnFullUrl; response.Add(new DownloadFileDto { - FileExists = cacheFile.Size > 0, + FileExists = file.Size > 0, ForbiddenBy = forbiddenFile?.ForbiddenBy ?? string.Empty, IsForbidden = forbiddenFile != null, - Hash = cacheFile.Hash, - Size = cacheFile.Size, - Url = new Uri(baseUrl, cacheFile.Hash.ToUpperInvariant()).ToString() + Hash = file.Hash, + Size = file.Size, + Url = new Uri(baseUrl, file.Hash.ToUpperInvariant()).ToString() }); } @@ -90,7 +91,7 @@ public partial class MareHub _logger.LogCallInfo(MareHubLogger.Args(userSentHashes.Count.ToString())); var notCoveredFiles = new Dictionary(StringComparer.Ordinal); var forbiddenFiles = await _dbContext.ForbiddenUploadEntries.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false); - var existingFiles = fileListHashes.Select(_fileDbService.GetFileCache).Select(t => t.Result).Where(t => t != null).ToDictionary(k => k.Hash, k => k, StringComparer.Ordinal); + var existingFiles = await _dbContext.Files.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false); var uploader = await _dbContext.Users.SingleAsync(u => u.UID == UserUID).ConfigureAwait(false); List fileCachesToUpload = new(); diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs index 1b93fa6..9900ea8 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs @@ -20,7 +20,6 @@ public partial class MareHub : Hub, IMareHub private readonly SystemInfoService _systemInfoService; private readonly IHttpContextAccessor _contextAccessor; private readonly IClientIdentificationService _clientIdentService; - private readonly FileDbService _fileDbService; private readonly MareHubLogger _logger; private readonly MareDbContext _dbContext; private readonly Uri _mainCdnFullUrl; @@ -33,7 +32,7 @@ public partial class MareHub : Hub, IMareHub public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient, MareDbContext mareDbContext, ILogger logger, SystemInfoService systemInfoService, IConfigurationService configuration, IHttpContextAccessor contextAccessor, - IClientIdentificationService clientIdentService, FileDbService fileDbService) + IClientIdentificationService clientIdentService) { _mareMetrics = mareMetrics; _fileServiceClient = fileServiceClient; @@ -46,7 +45,6 @@ public partial class MareHub : Hub, IMareHub _maxGroupUserCount = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxGroupUserCount), 100); _contextAccessor = contextAccessor; _clientIdentService = clientIdentService; - _fileDbService = fileDbService; _logger = new MareHubLogger(this, logger); _dbContext = mareDbContext; } diff --git a/MareSynchronosServer/MareSynchronosServer/Identity/IdentityHandler.cs b/MareSynchronosServer/MareSynchronosServer/Identity/IdentityHandler.cs index 9103fdb..2b52d58 100644 --- a/MareSynchronosServer/MareSynchronosServer/Identity/IdentityHandler.cs +++ b/MareSynchronosServer/MareSynchronosServer/Identity/IdentityHandler.cs @@ -65,7 +65,7 @@ public class IdentityHandler internal void EnqueueIdentChange(IdentChange identchange) { - _logger.LogDebug("Enqueued " + identchange.UidWithIdent.Uid.Uid + ":" + identchange.IsOnline + " from " + identchange.UidWithIdent.Ident.ServerId); + _logger.LogInformation("Enqueued " + identchange.UidWithIdent.Uid.Uid + ":" + identchange.IsOnline + " from " + identchange.UidWithIdent.Ident.ServerId); foreach (var k in _identChanges.Keys) { diff --git a/MareSynchronosServer/MareSynchronosServer/Services/FileDbService.cs b/MareSynchronosServer/MareSynchronosServer/Services/FileDbService.cs deleted file mode 100644 index 40bc77b..0000000 --- a/MareSynchronosServer/MareSynchronosServer/Services/FileDbService.cs +++ /dev/null @@ -1,71 +0,0 @@ -using MareSynchronosShared.Data; -using MareSynchronosShared.Models; -using Microsoft.EntityFrameworkCore; - -namespace MareSynchronosServer.Services; - -public class FileDbService : IHostedService -{ - private readonly IServiceProvider _serviceProvider; - private readonly ILogger _logger; - private Dictionary _fileCaches = new(StringComparer.Ordinal); - private readonly SemaphoreSlim _semaphore = new(5); - private readonly CancellationTokenSource _shutdownCancellationToken = new(); - - public FileDbService(IServiceProvider serviceProvider, ILogger logger) - { - _serviceProvider = serviceProvider; - _logger = logger; - } - - public async Task GetFileCache(string hash) - { - await _semaphore.WaitAsync().ConfigureAwait(false); - if (!_fileCaches.TryGetValue(hash, out var cache)) - { - using var db = _serviceProvider.GetService(); - cache = db.Files.AsNoTracking().SingleOrDefault(f => f.Hash == hash && f.Uploaded); - if (cache != null) _fileCaches[hash] = cache; - } - _semaphore.Release(); - - return cache; - } - - private async Task UpdateDatabasePeriodically(CancellationToken ct) - { - while (!ct.IsCancellationRequested) - { - var now = DateTime.Now; - TimeOnly currentTime = new(now.Hour, now.Minute, now.Second); - TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % 10 + 5, 0); - var span = futureTime.AddMinutes(10) - currentTime; - - await Task.Delay(span, ct).ConfigureAwait(false); - - await UpdateDatabase().ConfigureAwait(false); - } - } - - private async Task UpdateDatabase() - { - using var scope = _serviceProvider.CreateScope(); - using var db = scope.ServiceProvider.GetRequiredService(); - _fileCaches = new(await db.Files.AsNoTracking().Where(f => f.Uploaded).AsNoTracking().ToDictionaryAsync(k => k.Hash, k => k).ConfigureAwait(false), StringComparer.Ordinal); - _logger.LogInformation("Updated FileCaches, now at {count}", _fileCaches.Count); - } - - public async Task StartAsync(CancellationToken cancellationToken) - { - await UpdateDatabase().ConfigureAwait(false); - _ = UpdateDatabasePeriodically(_shutdownCancellationToken.Token); - _logger.LogInformation("Started FileDb Service, initially at {count} files from DB", _fileCaches.Count); - } - - public Task StopAsync(CancellationToken cancellationToken) - { - _shutdownCancellationToken.Cancel(); - - return Task.CompletedTask; - } -} diff --git a/MareSynchronosServer/MareSynchronosServer/Startup.cs b/MareSynchronosServer/MareSynchronosServer/Startup.cs index b38de8f..f50d0ff 100644 --- a/MareSynchronosServer/MareSynchronosServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosServer/Startup.cs @@ -78,8 +78,6 @@ public class Startup services.AddSingleton(); services.AddSingleton(); services.AddHostedService(provider => provider.GetService()); - services.AddSingleton(); - services.AddHostedService(p => p.GetService()); // configure services based on main server status ConfigureServicesBasedOnShardType(services, mareConfig, isMainServer);