From 48323d68b0b2b938057e037f7a8f359f247b7878 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Tue, 14 Mar 2023 19:46:59 +0100 Subject: [PATCH] [Draft] Update 0.8 (#25) * get rid of file handling through grpc and signalr * fix upload on controller * adapt usersetpairpermissions * send user perms * server-side fixes * rework file upload * adjust log level to debug in docker standalone json * update dependencies --------- Co-authored-by: rootdarkarchon --- Docker/run/compose/mare-sharded.yml | 17 +- Docker/run/compose/mare-standalone.yml | 1 + .../config/standalone/files-standalone.json | 12 +- .../config/standalone/server-standalone.json | 6 +- .../standalone/services-standalone.json | 1 + MareAPI | 2 +- .../Hubs/MareHub.Files.cs | 253 -------- .../Hubs/MareHub.Functions.cs | 2 +- .../MareSynchronosServer/Hubs/MareHub.User.cs | 39 +- .../MareSynchronosServer/Hubs/MareHub.cs | 19 +- .../MareSynchronosServer.csproj | 21 +- .../MareSynchronosServer/Startup.cs | 29 +- .../Discord/MareModule.cs | 2 +- .../MareSynchronosServices.csproj | 13 +- .../MareSynchronosShared.csproj | 22 +- .../20230228001033_UserPerms.Designer.cs | 552 ++++++++++++++++++ .../Migrations/20230228001033_UserPerms.cs | 40 ++ .../Migrations/MareDbContextModelSnapshot.cs | 10 +- .../MareSynchronosShared/Models/ClientPair.cs | 2 + .../Protos/mareservices.proto | 82 +-- .../RequirementHandlers/UserRequirement.cs | 2 +- .../UserRequirementHandler.cs | 3 +- .../RequirementHandlers/UserRequirements.cs | 2 +- .../Utils/ServerConfiguration.cs | 10 +- .../Utils/StaticFilesServerConfiguration.cs | 4 + .../Controllers/RequestController.cs | 5 +- .../Controllers/ServerFilesController.cs | 223 ++++++- .../MareSynchronosStaticFilesServer.csproj | 12 +- .../Services/GrpcFileService.cs | 105 ---- .../Services/RequestQueueService.cs | 100 ++-- .../Startup.cs | 53 +- 31 files changed, 1010 insertions(+), 634 deletions(-) delete mode 100644 MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs create mode 100644 MareSynchronosServer/MareSynchronosShared/Migrations/20230228001033_UserPerms.Designer.cs create mode 100644 MareSynchronosServer/MareSynchronosShared/Migrations/20230228001033_UserPerms.cs rename MareSynchronosServer/{MareSynchronosServer => MareSynchronosShared}/RequirementHandlers/UserRequirement.cs (83%) rename MareSynchronosServer/{MareSynchronosServer => MareSynchronosShared}/RequirementHandlers/UserRequirementHandler.cs (96%) rename MareSynchronosServer/{MareSynchronosServer => MareSynchronosShared}/RequirementHandlers/UserRequirements.cs (69%) delete mode 100644 MareSynchronosServer/MareSynchronosStaticFilesServer/Services/GrpcFileService.cs diff --git a/Docker/run/compose/mare-sharded.yml b/Docker/run/compose/mare-sharded.yml index 12f4252..a683216 100644 --- a/Docker/run/compose/mare-sharded.yml +++ b/Docker/run/compose/mare-sharded.yml @@ -37,11 +37,7 @@ services: image: darkarchon/mare-synchronos-server:latest restart: on-failure environment: - MareSynchronos__CdnFullUrl: "http://darkarchon.internet-box.ch:9999" - MareSynchronos__CdnShardConfiguration__0__CdnFullUrl: "${DEV_MARE_CDNURL}" - MareSynchronos__CdnShardConfiguration__0__FileMatch: "^[012345678]" - MareSynchronos__CdnShardConfiguration__1__CdnFullUrl: "${DEV_MARE_CDNURL2}" - MareSynchronos__CdnShardConfiguration__1__FileMatch: "^[789ABCDEF]" + MareSynchronos__CdnFullUrl: "${DEV_MARE_CDNURL}" volumes: - ../config/sharded/server-shard-main.json:/opt/MareSynchronosServer/appsettings.json - ../log/server-shard-main/:/opt/MareSynchronosServer/logs/:rw @@ -94,6 +90,13 @@ services: mare-files: image: darkarchon/mare-synchronos-staticfilesserver:latest restart: on-failure + ports: + - 6200:6200/tcp + environment: + MareSynchronos__CdnShardConfiguration__0__CdnFullUrl: "${DEV_MARE_FILES1}" + MareSynchronos__CdnShardConfiguration__0__FileMatch: "^[012345678]" + MareSynchronos__CdnShardConfiguration__1__CdnFullUrl: "${DEV_MARE_FILES2}" + MareSynchronos__CdnShardConfiguration__1__FileMatch: "^[789ABCDEF]" volumes: - ../config/sharded/files-shard-main.json:/opt/MareSynchronosStaticFilesServer/appsettings.json - ../log/files-standalone/:/opt/MareSynchronosStaticFilesServer/logs/:rw @@ -117,7 +120,7 @@ services: - postgres_socket:/var/run/postgresql/:rw - ../data/files-shard-1/:/marecache/:rw ports: - - 6200:6200/tcp + - 6201:6200/tcp depends_on: mare-files: condition: service_healthy @@ -131,7 +134,7 @@ services: - postgres_socket:/var/run/postgresql/:rw - ../data/files-shard-2/:/marecache/:rw ports: - - 6201:6200/tcp + - 6202:6200/tcp depends_on: mare-files: condition: service_healthy diff --git a/Docker/run/compose/mare-standalone.yml b/Docker/run/compose/mare-standalone.yml index 1843387..4c89bb0 100644 --- a/Docker/run/compose/mare-standalone.yml +++ b/Docker/run/compose/mare-standalone.yml @@ -65,6 +65,7 @@ services: - 6200:6200/tcp restart: on-failure environment: + MareSynchronos__CdnFullUrl: "${DEV_MARE_CDNURL}" DOTNET_USE_POLLING_FILE_WATCHER: 1 volumes: - ../config/standalone/files-standalone.json:/opt/MareSynchronosStaticFilesServer/appsettings.json diff --git a/Docker/run/config/standalone/files-standalone.json b/Docker/run/config/standalone/files-standalone.json index c61baf0..891358b 100644 --- a/Docker/run/config/standalone/files-standalone.json +++ b/Docker/run/config/standalone/files-standalone.json @@ -4,11 +4,11 @@ }, "Logging": { "LogLevel": { - "Default": "Warning", - "Microsoft": "Warning", + "Default": "Debug", + "Microsoft": "Debug", "Microsoft.Hosting.Lifetime": "Information", - "MareSynchronosStaticFilesServer": "Information", - "MareSynchronosShared": "Information", + "MareSynchronosStaticFilesServer": "Debug", + "MareSynchronosShared": "Debug", "System.IO": "Information" }, "File": { @@ -44,10 +44,6 @@ "Endpoints": { "Http": { "Url": "http://+:6200" - }, - "Grpc": { - "Protocols": "Http2", - "Url": "http://+:6205" } } }, diff --git a/Docker/run/config/standalone/server-standalone.json b/Docker/run/config/standalone/server-standalone.json index ddbc1cc..42812fd 100644 --- a/Docker/run/config/standalone/server-standalone.json +++ b/Docker/run/config/standalone/server-standalone.json @@ -36,13 +36,13 @@ "" ], "RedisConnectionString": "redis,password=secretredispassword", - "CdnFullUrl": "http://localhost:6200/", - "StaticFileServiceAddress": "http://mare-files:6205", + "CdnFullUrl": "http://localhost:6200", "MaxExistingGroupsByUser": 3, "MaxJoinedGroupsByUser": 6, "MaxGroupUserCount": 100, "PurgeUnusedAccounts": false, - "PurgeUnusedAccountsPeriodInDays": 14 + "PurgeUnusedAccountsPeriodInDays": 14, + "ExpectedClientVersion": "0.8.0" }, "AllowedHosts": "*", "Kestrel": { diff --git a/Docker/run/config/standalone/services-standalone.json b/Docker/run/config/standalone/services-standalone.json index 8a07ba9..56365a4 100644 --- a/Docker/run/config/standalone/services-standalone.json +++ b/Docker/run/config/standalone/services-standalone.json @@ -28,6 +28,7 @@ "DbContextPoolSize": 512, "ShardName": "Services", "MetricsPort": 6150, + "CdnFullUrl": "http://localhost:6200/", "MainServerAddress": "http://mare-server:6000/", "MainServerGrpcAddress": "http://mare-server:6005/", "DiscordBotToken": "", diff --git a/MareAPI b/MareAPI index 381f9a4..f8e647a 160000 --- a/MareAPI +++ b/MareAPI @@ -1 +1 @@ -Subproject commit 381f9a48085c7feb4acb7d00658d06d555c1dd96 +Subproject commit f8e647af00be85404523e77920c03fc1a9eb894a diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs deleted file mode 100644 index e60ba57..0000000 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs +++ /dev/null @@ -1,253 +0,0 @@ -using System.Security.Cryptography; -using System.Text.RegularExpressions; -using Google.Protobuf; -using Grpc.Core; -using MareSynchronos.API.Dto.Files; -using MareSynchronosServer.Utils; -using MareSynchronosShared.Models; -using MareSynchronosShared.Protos; -using MareSynchronosShared.Utils; -using Microsoft.AspNetCore.Authorization; -using Microsoft.EntityFrameworkCore; - -namespace MareSynchronosServer.Hubs; - -public partial class MareHub -{ - private static readonly SemaphoreSlim _uploadSemaphore = new(20); - - [Authorize(Policy = "Identified")] - public async Task FilesAbortUpload() - { - _logger.LogCallInfo(); - var notUploadedFiles = await _dbContext.Files.Where(f => !f.Uploaded && f.Uploader.UID == UserUID).ToListAsync(); - if (notUploadedFiles.Any()) - { - _dbContext.RemoveRange(notUploadedFiles); - await _dbContext.SaveChangesAsync().ConfigureAwait(false); - } - } - - [Authorize(Policy = "Identified")] - public async Task FilesDeleteAll() - { - _logger.LogCallInfo(); - - var ownFiles = await _dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == UserUID).ToListAsync().ConfigureAwait(false); - var request = new DeleteFilesRequest(); - request.Hash.AddRange(ownFiles.Select(f => f.Hash)); - Metadata headers = new Metadata() - { - { "Authorization", "Bearer " + _generator.Token }, - }; - _ = await _fileServiceClient.DeleteFilesAsync(request, headers).ConfigureAwait(false); - } - - [Authorize(Policy = "Identified")] - public async Task> FilesGetSizes(List hashes) - { - _logger.LogCallInfo(MareHubLogger.Args(hashes.Count.ToString())); - - var allFiles = await _dbContext.Files.Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false); - var forbiddenFiles = await _dbContext.ForbiddenUploadEntries. - Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false); - List response = new(); - - var cacheFile = await _dbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false); - - var shardConfig = new List(_configurationService.GetValueOrDefault(nameof(ServerConfiguration.CdnShardConfiguration), new List())); - - foreach (var file in cacheFile) - { - var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase)); - - var matchedShardConfig = shardConfig.OrderBy(g => Guid.NewGuid()).FirstOrDefault(f => new Regex(f.FileMatch).IsMatch(file.Hash)); - var baseUrl = matchedShardConfig?.CdnFullUrl ?? _mainCdnFullUrl; - - response.Add(new DownloadFileDto - { - FileExists = file.Size > 0, - ForbiddenBy = forbiddenFile?.ForbiddenBy ?? string.Empty, - IsForbidden = forbiddenFile != null, - Hash = file.Hash, - Size = file.Size, - Url = baseUrl.ToString(), - }); - } - - return response; - } - - [Authorize(Policy = "Identified")] - public async Task FilesIsUploadFinished() - { - _logger.LogCallInfo(); - return await _dbContext.Files.AsNoTracking() - .AnyAsync(f => f.Uploader.UID == UserUID && !f.Uploaded).ConfigureAwait(false); - } - - [Authorize(Policy = "Identified")] - public async Task> FilesSend(List fileListHashes) - { - var userSentHashes = new HashSet(fileListHashes.Distinct(StringComparer.Ordinal).Select(s => string.Concat(s.Where(c => char.IsLetterOrDigit(c)))), StringComparer.Ordinal); - _logger.LogCallInfo(MareHubLogger.Args(userSentHashes.Count.ToString())); - var notCoveredFiles = new Dictionary(StringComparer.Ordinal); - var forbiddenFiles = await _dbContext.ForbiddenUploadEntries.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false); - var existingFiles = await _dbContext.Files.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false); - var uploader = await _dbContext.Users.SingleAsync(u => u.UID == UserUID).ConfigureAwait(false); - - List fileCachesToUpload = new(); - foreach (var hash in userSentHashes) - { - // Skip empty file hashes, duplicate file hashes, forbidden file hashes and existing file hashes - if (string.IsNullOrEmpty(hash)) { continue; } - if (notCoveredFiles.ContainsKey(hash)) { continue; } - if (forbiddenFiles.ContainsKey(hash)) - { - notCoveredFiles[hash] = new UploadFileDto() - { - ForbiddenBy = forbiddenFiles[hash].ForbiddenBy, - Hash = hash, - IsForbidden = true, - }; - - continue; - } - if (existingFiles.TryGetValue(hash, out var file) && file.Uploaded) { continue; } - - _logger.LogCallInfo(MareHubLogger.Args(hash, "Missing")); - - if (file == null) - { - fileCachesToUpload.Add(new FileCache() - { - Hash = hash, - Uploaded = false, - Uploader = uploader, - UploadDate = DateTime.UtcNow, - }); - } - - notCoveredFiles[hash] = new UploadFileDto() - { - Hash = hash, - }; - } - //Save bulk - await _dbContext.Files.AddRangeAsync(fileCachesToUpload).ConfigureAwait(false); - await _dbContext.SaveChangesAsync().ConfigureAwait(false); - return notCoveredFiles.Values.ToList(); - } - - [Authorize(Policy = "Identified")] - public async Task FilesUploadStreamAsync(string hash, IAsyncEnumerable fileContent) - { - _logger.LogCallInfo(MareHubLogger.Args(hash)); - - await _uploadSemaphore.WaitAsync(Context.ConnectionAborted).ConfigureAwait(false); - - var relatedFile = _dbContext.Files.SingleOrDefault(f => f.Hash == hash && f.Uploader.UID == UserUID && !f.Uploaded); - if (relatedFile == null) - { - _uploadSemaphore.Release(); - return; - } - var forbiddenFile = _dbContext.ForbiddenUploadEntries.SingleOrDefault(f => f.Hash == hash); - if (forbiddenFile != null) - { - _uploadSemaphore.Release(); - return; - } - - var tempFileName = Path.GetTempFileName(); - using var fileStream = new FileStream(tempFileName, FileMode.OpenOrCreate); - long length = 0; - try - { - await foreach (var chunk in fileContent.ConfigureAwait(false)) - { - length += chunk.Length; - await fileStream.WriteAsync(chunk).ConfigureAwait(false); - } - - await fileStream.FlushAsync().ConfigureAwait(false); - await fileStream.DisposeAsync().ConfigureAwait(false); - } - catch - { - try - { - await fileStream.FlushAsync().ConfigureAwait(false); - await fileStream.DisposeAsync().ConfigureAwait(false); - _dbContext.Files.Remove(relatedFile); - await _dbContext.SaveChangesAsync().ConfigureAwait(false); - } - catch - { - // already removed - } - finally - { - File.Delete(tempFileName); - } - - _uploadSemaphore.Release(); - return; - } - - _logger.LogCallInfo(MareHubLogger.Args(hash, "Uploaded")); - - try - { - var decodedFile = LZ4.LZ4Codec.Unwrap(await File.ReadAllBytesAsync(tempFileName).ConfigureAwait(false)); - using var sha1 = SHA1.Create(); - using var ms = new MemoryStream(decodedFile); - var computedHash = await sha1.ComputeHashAsync(ms).ConfigureAwait(false); - var computedHashString = BitConverter.ToString(computedHash).Replace("-", "", StringComparison.Ordinal); - if (!string.Equals(hash, computedHashString, StringComparison.Ordinal)) - { - _logger.LogCallWarning(MareHubLogger.Args(hash, "Invalid", computedHashString)); - _dbContext.Remove(relatedFile); - await _dbContext.SaveChangesAsync().ConfigureAwait(false); - - _uploadSemaphore.Release(); - return; - } - - Metadata headers = new Metadata() - { - { "Authorization", "Bearer " + _generator.Token }, - }; - var streamingCall = _fileServiceClient.UploadFile(headers); - using var tempFileStream = new FileStream(tempFileName, FileMode.Open, FileAccess.Read); - int size = 1024 * 1024; - byte[] data = new byte[size]; - int readBytes; - while ((readBytes = tempFileStream.Read(data, 0, size)) > 0) - { - await streamingCall.RequestStream.WriteAsync(new UploadFileRequest() - { - FileData = ByteString.CopyFrom(data, 0, readBytes), - Hash = computedHashString, - Uploader = UserUID, - }).ConfigureAwait(false); - } - await streamingCall.RequestStream.CompleteAsync().ConfigureAwait(false); - tempFileStream.Close(); - await tempFileStream.DisposeAsync().ConfigureAwait(false); - - _logger.LogCallInfo(MareHubLogger.Args(hash, "Pushed")); - } - catch (Exception ex) - { - _logger.LogCallWarning(MareHubLogger.Args("Failed", hash, ex.Message)); - _dbContext.Remove(relatedFile); - await _dbContext.SaveChangesAsync().ConfigureAwait(false); - } - finally - { - _uploadSemaphore.Release(); - File.Delete(tempFileName); - } - } -} diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Functions.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Functions.cs index 7fba578..cf75d67 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Functions.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Functions.cs @@ -30,7 +30,7 @@ public partial class MareHub private async Task> GetOnlineUsers(List uids) { - var result = await _redis.GetAllAsync(uids.Select(u => "UID:" + u).ToArray()).ConfigureAwait(false); + var result = await _redis.GetAllAsync(uids.Select(u => "UID:" + u).ToHashSet(StringComparer.Ordinal)).ConfigureAwait(false); return uids.Where(u => result.TryGetValue("UID:" + u, out var ident) && !string.IsNullOrEmpty(ident)).ToDictionary(u => u, u => result["UID:" + u], StringComparer.Ordinal); } diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.User.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.User.cs index e4cfcda..f1289f7 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.User.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.User.cs @@ -68,6 +68,10 @@ public partial class MareHub OtherIsPaused = otherEntry != null && otherEntry.IsPaused, userToOther.OtherUserUID, IsSynced = otherEntry != null, + DisableOwnAnimations = userToOther.DisableAnimations, + DisableOwnSounds = userToOther.DisableSounds, + DisableOtherAnimations = otherEntry == null ? false : otherEntry.DisableAnimations, + DisableOtherSounds = otherEntry == null ? false : otherEntry.DisableSounds }; var results = await query.AsNoTracking().ToListAsync().ConfigureAwait(false); @@ -76,9 +80,13 @@ public partial class MareHub { var ownPerm = UserPermissions.Paired; ownPerm.SetPaused(c.IsPaused); + ownPerm.SetDisableAnimations(c.DisableOwnAnimations); + ownPerm.SetDisableSounds(c.DisableOwnSounds); var otherPerm = UserPermissions.NoneSet; otherPerm.SetPaired(c.IsSynced); otherPerm.SetPaused(c.OtherIsPaused); + otherPerm.SetDisableAnimations(c.DisableOtherAnimations); + otherPerm.SetDisableSounds(c.DisableOtherSounds); return new UserPairDto(new(c.OtherUserUID, c.Alias), ownPerm, otherPerm); }).ToList(); } @@ -215,7 +223,11 @@ public partial class MareHub ClientPair pair = await _dbContext.ClientPairs.SingleOrDefaultAsync(w => w.UserUID == UserUID && w.OtherUserUID == dto.User.UID).ConfigureAwait(false); if (pair == null) return; + var pauseChange = pair.IsPaused != dto.Permissions.IsPaused(); + pair.IsPaused = dto.Permissions.IsPaused(); + pair.DisableAnimations = dto.Permissions.IsDisableAnimations(); + pair.DisableSounds = dto.Permissions.IsDisableSounds(); _dbContext.Update(pair); await _dbContext.SaveChangesAsync().ConfigureAwait(false); @@ -229,19 +241,22 @@ public partial class MareHub { await Clients.User(dto.User.UID).Client_UserUpdateOtherPairPermissions(new UserPermissionsDto(new UserData(UserUID), dto.Permissions)).ConfigureAwait(false); - var otherCharaIdent = await GetUserIdent(pair.OtherUserUID).ConfigureAwait(false); - - if (UserCharaIdent == null || otherCharaIdent == null || otherEntry.IsPaused) return; - - if (dto.Permissions.IsPaused()) + if (pauseChange) { - await Clients.User(UserUID).Client_UserSendOffline(dto).ConfigureAwait(false); - await Clients.User(dto.User.UID).Client_UserSendOffline(new(new(UserUID))).ConfigureAwait(false); - } - else - { - await Clients.User(UserUID).Client_UserSendOnline(new(dto.User, otherCharaIdent)).ConfigureAwait(false); - await Clients.User(dto.User.UID).Client_UserSendOnline(new(new(UserUID), UserCharaIdent)).ConfigureAwait(false); + var otherCharaIdent = await GetUserIdent(pair.OtherUserUID).ConfigureAwait(false); + + if (UserCharaIdent == null || otherCharaIdent == null || otherEntry.IsPaused) return; + + if (dto.Permissions.IsPaused()) + { + await Clients.User(UserUID).Client_UserSendOffline(dto).ConfigureAwait(false); + await Clients.User(dto.User.UID).Client_UserSendOffline(new(new(UserUID))).ConfigureAwait(false); + } + else + { + await Clients.User(UserUID).Client_UserSendOnline(new(dto.User, otherCharaIdent)).ConfigureAwait(false); + await Clients.User(dto.User.UID).Client_UserSendOnline(new(new(UserUID), UserCharaIdent)).ConfigureAwait(false); + } } } } diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs index 9e0d20a..19db0b7 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs @@ -7,7 +7,6 @@ using MareSynchronosServer.Utils; using MareSynchronosShared; using MareSynchronosShared.Data; using MareSynchronosShared.Metrics; -using MareSynchronosShared.Protos; using MareSynchronosShared.Services; using MareSynchronosShared.Utils; using Microsoft.AspNetCore.Authorization; @@ -20,37 +19,33 @@ namespace MareSynchronosServer.Hubs; public partial class MareHub : Hub, IMareHub { private readonly MareMetrics _mareMetrics; - private readonly FileService.FileServiceClient _fileServiceClient; private readonly SystemInfoService _systemInfoService; private readonly IHttpContextAccessor _contextAccessor; private readonly MareHubLogger _logger; private readonly MareDbContext _dbContext; - private readonly Uri _mainCdnFullUrl; private readonly string _shardName; private readonly int _maxExistingGroupsByUser; private readonly int _maxJoinedGroupsByUser; private readonly int _maxGroupUserCount; - private readonly IConfigurationService _configurationService; private readonly IRedisDatabase _redis; - private readonly ServerTokenGenerator _generator; + private readonly Uri _fileServerAddress; + private readonly Version _expectedClientVersion; - public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient, + public MareHub(MareMetrics mareMetrics, MareDbContext mareDbContext, ILogger logger, SystemInfoService systemInfoService, IConfigurationService configuration, IHttpContextAccessor contextAccessor, - IRedisDatabase redisDb, ServerTokenGenerator generator) + IRedisDatabase redisDb) { _mareMetrics = mareMetrics; - _fileServiceClient = fileServiceClient; _systemInfoService = systemInfoService; - _configurationService = configuration; - _mainCdnFullUrl = configuration.GetValue(nameof(ServerConfiguration.CdnFullUrl)); _shardName = configuration.GetValue(nameof(ServerConfiguration.ShardName)); _maxExistingGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxExistingGroupsByUser), 3); _maxJoinedGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxJoinedGroupsByUser), 6); _maxGroupUserCount = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxGroupUserCount), 100); + _fileServerAddress = configuration.GetValue(nameof(ServerConfiguration.CdnFullUrl)); + _expectedClientVersion = configuration.GetValueOrDefault(nameof(ServerConfiguration.ExpectedClientVersion), new Version(0, 0, 0)); _contextAccessor = contextAccessor; _redis = redisDb; - _generator = generator; _logger = new MareHubLogger(this, logger); _dbContext = mareDbContext; } @@ -73,6 +68,7 @@ public partial class MareHub : Hub, IMareHub return new ConnectionDto(new UserData(dbUser.UID, string.IsNullOrWhiteSpace(dbUser.Alias) ? null : dbUser.Alias)) { + CurrentClientVersion = _expectedClientVersion, ServerVersion = IMareHub.ApiVersion, IsAdmin = dbUser.IsAdmin, IsModerator = dbUser.IsModerator, @@ -82,6 +78,7 @@ public partial class MareHub : Hub, IMareHub ShardName = _shardName, MaxGroupsJoinedByUser = _maxJoinedGroupsByUser, MaxGroupUserCount = _maxGroupUserCount, + FileServerAddress = _fileServerAddress }, }; } diff --git a/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj b/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj index 18eec02..0c380a1 100644 --- a/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj +++ b/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj @@ -21,29 +21,18 @@ - - - - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - + + - - - - - - - + + diff --git a/MareSynchronosServer/MareSynchronosServer/Startup.cs b/MareSynchronosServer/MareSynchronosServer/Startup.cs index d408311..509357c 100644 --- a/MareSynchronosServer/MareSynchronosServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosServer/Startup.cs @@ -9,7 +9,6 @@ using MareSynchronosShared.Protos; using Grpc.Net.Client.Configuration; using MareSynchronosShared.Metrics; using MareSynchronosServer.Services; -using MareSynchronosServer.RequirementHandlers; using MareSynchronosShared.Utils; using MareSynchronosShared.Services; using Prometheus; @@ -26,6 +25,7 @@ using MessagePack; using MessagePack.Resolvers; using Microsoft.AspNetCore.Mvc.Controllers; using MareSynchronosServer.Controllers; +using MareSynchronosShared.RequirementHandlers; namespace MareSynchronosServer; @@ -52,9 +52,6 @@ public class Startup // configure metrics ConfigureMetrics(services); - // configure file service grpc connection - ConfigureFileServiceGrpcClient(services); - // configure database ConfigureDatabase(services, mareConfig); @@ -294,30 +291,6 @@ public class Startup } } - private static void ConfigureFileServiceGrpcClient(IServiceCollection services) - { - var defaultMethodConfig = new MethodConfig - { - Names = { MethodName.Default }, - RetryPolicy = new RetryPolicy - { - MaxAttempts = 1000, - InitialBackoff = TimeSpan.FromSeconds(1), - MaxBackoff = TimeSpan.FromSeconds(5), - BackoffMultiplier = 1.5, - RetryableStatusCodes = { Grpc.Core.StatusCode.Unavailable }, - }, - }; - services.AddGrpcClient((serviceProvider, c) => - { - c.Address = serviceProvider.GetRequiredService>() - .GetValue(nameof(ServerConfiguration.StaticFileServiceAddress)); - }).ConfigureChannel(c => - { - c.ServiceConfig = new ServiceConfig { MethodConfigs = { defaultMethodConfig } }; - }); - } - public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILogger logger) { logger.LogInformation("Running Configure"); diff --git a/MareSynchronosServer/MareSynchronosServices/Discord/MareModule.cs b/MareSynchronosServer/MareSynchronosServices/Discord/MareModule.cs index 89dd6db..5761cb1 100644 --- a/MareSynchronosServer/MareSynchronosServices/Discord/MareModule.cs +++ b/MareSynchronosServer/MareSynchronosServices/Discord/MareModule.cs @@ -461,7 +461,7 @@ public class MareModule : InteractionModuleBase }; await db.Users.AddAsync(newUser).ConfigureAwait(false); - await db.Auth.AddAsync(auth).ConfigureAwait(false); ; + await db.Auth.AddAsync(auth).ConfigureAwait(false); await db.SaveChangesAsync().ConfigureAwait(false); diff --git a/MareSynchronosServer/MareSynchronosServices/MareSynchronosServices.csproj b/MareSynchronosServer/MareSynchronosServices/MareSynchronosServices.csproj index 6c5dd4d..29a0e7e 100644 --- a/MareSynchronosServer/MareSynchronosServices/MareSynchronosServices.csproj +++ b/MareSynchronosServer/MareSynchronosServices/MareSynchronosServices.csproj @@ -22,14 +22,15 @@ - - - - + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + - - + diff --git a/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj b/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj index f02c534..e1b53af 100644 --- a/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj +++ b/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj @@ -21,7 +21,8 @@ - + + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -30,16 +31,21 @@ - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - + + + + + + + + + diff --git a/MareSynchronosServer/MareSynchronosShared/Migrations/20230228001033_UserPerms.Designer.cs b/MareSynchronosServer/MareSynchronosShared/Migrations/20230228001033_UserPerms.Designer.cs new file mode 100644 index 0000000..58f336e --- /dev/null +++ b/MareSynchronosServer/MareSynchronosShared/Migrations/20230228001033_UserPerms.Designer.cs @@ -0,0 +1,552 @@ +// +using System; +using MareSynchronosShared.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace MareSynchronosServer.Migrations +{ + [DbContext(typeof(MareDbContext))] + [Migration("20230228001033_UserPerms")] + partial class UserPerms + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "7.0.3") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("MareSynchronosShared.Models.Auth", b => + { + b.Property("HashedKey") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("hashed_key"); + + b.Property("IsBanned") + .HasColumnType("boolean") + .HasColumnName("is_banned"); + + b.Property("PrimaryUserUID") + .HasColumnType("character varying(10)") + .HasColumnName("primary_user_uid"); + + b.Property("UserUID") + .HasColumnType("character varying(10)") + .HasColumnName("user_uid"); + + b.HasKey("HashedKey") + .HasName("pk_auth"); + + b.HasIndex("PrimaryUserUID") + .HasDatabaseName("ix_auth_primary_user_uid"); + + b.HasIndex("UserUID") + .HasDatabaseName("ix_auth_user_uid"); + + b.ToTable("auth", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.Banned", b => + { + b.Property("CharacterIdentification") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("character_identification"); + + b.Property("Reason") + .HasColumnType("text") + .HasColumnName("reason"); + + b.Property("Timestamp") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("bytea") + .HasColumnName("timestamp"); + + b.HasKey("CharacterIdentification") + .HasName("pk_banned_users"); + + b.ToTable("banned_users", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.BannedRegistrations", b => + { + b.Property("DiscordIdOrLodestoneAuth") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("discord_id_or_lodestone_auth"); + + b.HasKey("DiscordIdOrLodestoneAuth") + .HasName("pk_banned_registrations"); + + b.ToTable("banned_registrations", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b => + { + b.Property("UserUID") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("user_uid"); + + b.Property("OtherUserUID") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("other_user_uid"); + + b.Property("AllowReceivingMessages") + .HasColumnType("boolean") + .HasColumnName("allow_receiving_messages"); + + b.Property("DisableAnimations") + .HasColumnType("boolean") + .HasColumnName("disable_animations"); + + b.Property("DisableSounds") + .HasColumnType("boolean") + .HasColumnName("disable_sounds"); + + b.Property("IsPaused") + .HasColumnType("boolean") + .HasColumnName("is_paused"); + + b.Property("Timestamp") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("bytea") + .HasColumnName("timestamp"); + + b.HasKey("UserUID", "OtherUserUID") + .HasName("pk_client_pairs"); + + b.HasIndex("OtherUserUID") + .HasDatabaseName("ix_client_pairs_other_user_uid"); + + b.HasIndex("UserUID") + .HasDatabaseName("ix_client_pairs_user_uid"); + + b.ToTable("client_pairs", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b => + { + b.Property("Hash") + .HasMaxLength(40) + .HasColumnType("character varying(40)") + .HasColumnName("hash"); + + b.Property("Size") + .HasColumnType("bigint") + .HasColumnName("size"); + + b.Property("Timestamp") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("bytea") + .HasColumnName("timestamp"); + + b.Property("UploadDate") + .HasColumnType("timestamp with time zone") + .HasColumnName("upload_date"); + + b.Property("Uploaded") + .HasColumnType("boolean") + .HasColumnName("uploaded"); + + b.Property("UploaderUID") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("uploader_uid"); + + b.HasKey("Hash") + .HasName("pk_file_caches"); + + b.HasIndex("UploaderUID") + .HasDatabaseName("ix_file_caches_uploader_uid"); + + b.ToTable("file_caches", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.ForbiddenUploadEntry", b => + { + b.Property("Hash") + .HasMaxLength(40) + .HasColumnType("character varying(40)") + .HasColumnName("hash"); + + b.Property("ForbiddenBy") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("forbidden_by"); + + b.Property("Timestamp") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("bytea") + .HasColumnName("timestamp"); + + b.HasKey("Hash") + .HasName("pk_forbidden_upload_entries"); + + b.ToTable("forbidden_upload_entries", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.Group", b => + { + b.Property("GID") + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("gid"); + + b.Property("Alias") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("alias"); + + b.Property("DisableAnimations") + .HasColumnType("boolean") + .HasColumnName("disable_animations"); + + b.Property("DisableSounds") + .HasColumnType("boolean") + .HasColumnName("disable_sounds"); + + b.Property("HashedPassword") + .HasColumnType("text") + .HasColumnName("hashed_password"); + + b.Property("InvitesEnabled") + .HasColumnType("boolean") + .HasColumnName("invites_enabled"); + + b.Property("OwnerUID") + .HasColumnType("character varying(10)") + .HasColumnName("owner_uid"); + + b.HasKey("GID") + .HasName("pk_groups"); + + b.HasIndex("OwnerUID") + .HasDatabaseName("ix_groups_owner_uid"); + + b.ToTable("groups", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b => + { + b.Property("GroupGID") + .HasColumnType("character varying(20)") + .HasColumnName("group_gid"); + + b.Property("BannedUserUID") + .HasColumnType("character varying(10)") + .HasColumnName("banned_user_uid"); + + b.Property("BannedByUID") + .HasColumnType("character varying(10)") + .HasColumnName("banned_by_uid"); + + b.Property("BannedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("banned_on"); + + b.Property("BannedReason") + .HasColumnType("text") + .HasColumnName("banned_reason"); + + b.HasKey("GroupGID", "BannedUserUID") + .HasName("pk_group_bans"); + + b.HasIndex("BannedByUID") + .HasDatabaseName("ix_group_bans_banned_by_uid"); + + b.HasIndex("BannedUserUID") + .HasDatabaseName("ix_group_bans_banned_user_uid"); + + b.HasIndex("GroupGID") + .HasDatabaseName("ix_group_bans_group_gid"); + + b.ToTable("group_bans", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b => + { + b.Property("GroupGID") + .HasColumnType("character varying(20)") + .HasColumnName("group_gid"); + + b.Property("GroupUserUID") + .HasColumnType("character varying(10)") + .HasColumnName("group_user_uid"); + + b.Property("DisableAnimations") + .HasColumnType("boolean") + .HasColumnName("disable_animations"); + + b.Property("DisableSounds") + .HasColumnType("boolean") + .HasColumnName("disable_sounds"); + + b.Property("IsModerator") + .HasColumnType("boolean") + .HasColumnName("is_moderator"); + + b.Property("IsPaused") + .HasColumnType("boolean") + .HasColumnName("is_paused"); + + b.Property("IsPinned") + .HasColumnType("boolean") + .HasColumnName("is_pinned"); + + b.HasKey("GroupGID", "GroupUserUID") + .HasName("pk_group_pairs"); + + b.HasIndex("GroupGID") + .HasDatabaseName("ix_group_pairs_group_gid"); + + b.HasIndex("GroupUserUID") + .HasDatabaseName("ix_group_pairs_group_user_uid"); + + b.ToTable("group_pairs", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b => + { + b.Property("GroupGID") + .HasColumnType("character varying(20)") + .HasColumnName("group_gid"); + + b.Property("Invite") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("invite"); + + b.Property("ExpirationDate") + .HasColumnType("timestamp with time zone") + .HasColumnName("expiration_date"); + + b.HasKey("GroupGID", "Invite") + .HasName("pk_group_temp_invites"); + + b.HasIndex("GroupGID") + .HasDatabaseName("ix_group_temp_invites_group_gid"); + + b.HasIndex("Invite") + .HasDatabaseName("ix_group_temp_invites_invite"); + + b.ToTable("group_temp_invites", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b => + { + b.Property("DiscordId") + .ValueGeneratedOnAdd() + .HasColumnType("numeric(20,0)") + .HasColumnName("discord_id"); + + b.Property("HashedLodestoneId") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("hashed_lodestone_id"); + + b.Property("LodestoneAuthString") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("lodestone_auth_string"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("UserUID") + .HasColumnType("character varying(10)") + .HasColumnName("user_uid"); + + b.HasKey("DiscordId") + .HasName("pk_lodestone_auth"); + + b.HasIndex("UserUID") + .HasDatabaseName("ix_lodestone_auth_user_uid"); + + b.ToTable("lodestone_auth", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.User", b => + { + b.Property("UID") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("uid"); + + b.Property("Alias") + .HasMaxLength(15) + .HasColumnType("character varying(15)") + .HasColumnName("alias"); + + b.Property("IsAdmin") + .HasColumnType("boolean") + .HasColumnName("is_admin"); + + b.Property("IsModerator") + .HasColumnType("boolean") + .HasColumnName("is_moderator"); + + b.Property("LastLoggedIn") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_logged_in"); + + b.Property("Timestamp") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("bytea") + .HasColumnName("timestamp"); + + b.HasKey("UID") + .HasName("pk_users"); + + b.ToTable("users", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.Auth", b => + { + b.HasOne("MareSynchronosShared.Models.User", "PrimaryUser") + .WithMany() + .HasForeignKey("PrimaryUserUID") + .HasConstraintName("fk_auth_users_primary_user_temp_id"); + + b.HasOne("MareSynchronosShared.Models.User", "User") + .WithMany() + .HasForeignKey("UserUID") + .HasConstraintName("fk_auth_users_user_temp_id1"); + + b.Navigation("PrimaryUser"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b => + { + b.HasOne("MareSynchronosShared.Models.User", "OtherUser") + .WithMany() + .HasForeignKey("OtherUserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_client_pairs_users_other_user_temp_id2"); + + b.HasOne("MareSynchronosShared.Models.User", "User") + .WithMany() + .HasForeignKey("UserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_client_pairs_users_user_temp_id3"); + + b.Navigation("OtherUser"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b => + { + b.HasOne("MareSynchronosShared.Models.User", "Uploader") + .WithMany() + .HasForeignKey("UploaderUID") + .HasConstraintName("fk_file_caches_users_uploader_uid"); + + b.Navigation("Uploader"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.Group", b => + { + b.HasOne("MareSynchronosShared.Models.User", "Owner") + .WithMany() + .HasForeignKey("OwnerUID") + .HasConstraintName("fk_groups_users_owner_temp_id8"); + + b.Navigation("Owner"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b => + { + b.HasOne("MareSynchronosShared.Models.User", "BannedBy") + .WithMany() + .HasForeignKey("BannedByUID") + .HasConstraintName("fk_group_bans_users_banned_by_temp_id5"); + + b.HasOne("MareSynchronosShared.Models.User", "BannedUser") + .WithMany() + .HasForeignKey("BannedUserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_bans_users_banned_user_temp_id6"); + + b.HasOne("MareSynchronosShared.Models.Group", "Group") + .WithMany() + .HasForeignKey("GroupGID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_bans_groups_group_temp_id"); + + b.Navigation("BannedBy"); + + b.Navigation("BannedUser"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b => + { + b.HasOne("MareSynchronosShared.Models.Group", "Group") + .WithMany() + .HasForeignKey("GroupGID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_pairs_groups_group_temp_id1"); + + b.HasOne("MareSynchronosShared.Models.User", "GroupUser") + .WithMany() + .HasForeignKey("GroupUserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_pairs_users_group_user_temp_id7"); + + b.Navigation("Group"); + + b.Navigation("GroupUser"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b => + { + b.HasOne("MareSynchronosShared.Models.Group", "Group") + .WithMany() + .HasForeignKey("GroupGID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_temp_invites_groups_group_gid"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b => + { + b.HasOne("MareSynchronosShared.Models.User", "User") + .WithMany() + .HasForeignKey("UserUID") + .HasConstraintName("fk_lodestone_auth_users_user_uid"); + + b.Navigation("User"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/MareSynchronosServer/MareSynchronosShared/Migrations/20230228001033_UserPerms.cs b/MareSynchronosServer/MareSynchronosShared/Migrations/20230228001033_UserPerms.cs new file mode 100644 index 0000000..18556ac --- /dev/null +++ b/MareSynchronosServer/MareSynchronosShared/Migrations/20230228001033_UserPerms.cs @@ -0,0 +1,40 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace MareSynchronosServer.Migrations +{ + /// + public partial class UserPerms : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "disable_animations", + table: "client_pairs", + type: "boolean", + nullable: false, + defaultValue: false); + + migrationBuilder.AddColumn( + name: "disable_sounds", + table: "client_pairs", + type: "boolean", + nullable: false, + defaultValue: false); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "disable_animations", + table: "client_pairs"); + + migrationBuilder.DropColumn( + name: "disable_sounds", + table: "client_pairs"); + } + } +} diff --git a/MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs b/MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs index 7b1c4b6..d5608df 100644 --- a/MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs +++ b/MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs @@ -17,7 +17,7 @@ namespace MareSynchronosServer.Migrations { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "7.0.0") + .HasAnnotation("ProductVersion", "7.0.3") .HasAnnotation("Relational:MaxIdentifierLength", 63); NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); @@ -105,6 +105,14 @@ namespace MareSynchronosServer.Migrations .HasColumnType("boolean") .HasColumnName("allow_receiving_messages"); + b.Property("DisableAnimations") + .HasColumnType("boolean") + .HasColumnName("disable_animations"); + + b.Property("DisableSounds") + .HasColumnType("boolean") + .HasColumnName("disable_sounds"); + b.Property("IsPaused") .HasColumnType("boolean") .HasColumnName("is_paused"); diff --git a/MareSynchronosServer/MareSynchronosShared/Models/ClientPair.cs b/MareSynchronosServer/MareSynchronosShared/Models/ClientPair.cs index 509af61..d149ed0 100644 --- a/MareSynchronosServer/MareSynchronosShared/Models/ClientPair.cs +++ b/MareSynchronosServer/MareSynchronosShared/Models/ClientPair.cs @@ -14,4 +14,6 @@ public class ClientPair public bool AllowReceivingMessages { get; set; } = false; [Timestamp] public byte[] Timestamp { get; set; } + public bool DisableSounds { get; set; } = false; + public bool DisableAnimations { get; set; } = false; } diff --git a/MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto b/MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto index 6b83f4b..8426269 100644 --- a/MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto +++ b/MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto @@ -4,11 +4,6 @@ option csharp_namespace = "MareSynchronosShared.Protos"; package mareservices; -service FileService { - rpc UploadFile (stream UploadFileRequest) returns (Empty); - rpc DeleteFiles (DeleteFilesRequest) returns (Empty); -} - service ClientMessageService { rpc SendClientMessage (ClientMessage) returns (Empty); } @@ -25,79 +20,4 @@ enum MessageType { ERROR = 2; } -message Empty { } - -message MultiUidMessage { - repeated UidMessage uids = 1; -} - -message ServerIdentMessage { - repeated SetIdentMessage idents = 1; -} - -message IdentChangeMessage { - oneof payload { - ServerMessage server = 1; - IdentChange identChange = 2; - } -} - -message IdentChange { - UidWithIdent uidWithIdent = 1; - bool isOnline = 2; -} - -message UidWithIdentMessage { - repeated UidWithIdent uidWithIdent = 1; -} - -message UidWithIdent { - UidMessage uid = 1; - CharacterIdentMessage ident = 2; -} - -message UidMessage { - string uid = 1; -} - -message ServerMessage { - string server_id = 1; -} - -message OnlineUserCountResponse { - int64 count = 1; -} - -message RemoveIdentMessage { - string uid = 1; - string server_id = 2; -} - -message SetIdentMessage { - UidWithIdent uidWithIdent = 1; -} - -message CharacterIdentMessage { - string server_id = 1; - string ident = 2; -} - -message UploadFileRequest { - string hash = 1; - string uploader = 2; - bytes fileData = 3; -} - -message DeleteFilesRequest { - repeated string hash = 1; -} - -message AuthRequest { - string ip = 1; - string secretKey = 2; -} - -message AuthReply { - bool success = 1; - UidMessage uid = 2; -} \ No newline at end of file +message Empty { } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosServer/RequirementHandlers/UserRequirement.cs b/MareSynchronosServer/MareSynchronosShared/RequirementHandlers/UserRequirement.cs similarity index 83% rename from MareSynchronosServer/MareSynchronosServer/RequirementHandlers/UserRequirement.cs rename to MareSynchronosServer/MareSynchronosShared/RequirementHandlers/UserRequirement.cs index bb7442b..317a30c 100644 --- a/MareSynchronosServer/MareSynchronosServer/RequirementHandlers/UserRequirement.cs +++ b/MareSynchronosServer/MareSynchronosShared/RequirementHandlers/UserRequirement.cs @@ -1,6 +1,6 @@ using Microsoft.AspNetCore.Authorization; -namespace MareSynchronosServer.RequirementHandlers; +namespace MareSynchronosShared.RequirementHandlers; public class UserRequirement : IAuthorizationRequirement { diff --git a/MareSynchronosServer/MareSynchronosServer/RequirementHandlers/UserRequirementHandler.cs b/MareSynchronosServer/MareSynchronosShared/RequirementHandlers/UserRequirementHandler.cs similarity index 96% rename from MareSynchronosServer/MareSynchronosServer/RequirementHandlers/UserRequirementHandler.cs rename to MareSynchronosServer/MareSynchronosShared/RequirementHandlers/UserRequirementHandler.cs index 00a64d1..1070739 100644 --- a/MareSynchronosServer/MareSynchronosServer/RequirementHandlers/UserRequirementHandler.cs +++ b/MareSynchronosServer/MareSynchronosShared/RequirementHandlers/UserRequirementHandler.cs @@ -5,8 +5,9 @@ using Microsoft.EntityFrameworkCore; using MareSynchronosShared.Utils; using StackExchange.Redis; using StackExchange.Redis.Extensions.Core.Abstractions; +using Microsoft.Extensions.Logging; -namespace MareSynchronosServer.RequirementHandlers; +namespace MareSynchronosShared.RequirementHandlers; public class UserRequirementHandler : AuthorizationHandler { diff --git a/MareSynchronosServer/MareSynchronosServer/RequirementHandlers/UserRequirements.cs b/MareSynchronosServer/MareSynchronosShared/RequirementHandlers/UserRequirements.cs similarity index 69% rename from MareSynchronosServer/MareSynchronosServer/RequirementHandlers/UserRequirements.cs rename to MareSynchronosServer/MareSynchronosShared/RequirementHandlers/UserRequirements.cs index b56a89b..95fed1f 100644 --- a/MareSynchronosServer/MareSynchronosServer/RequirementHandlers/UserRequirements.cs +++ b/MareSynchronosServer/MareSynchronosShared/RequirementHandlers/UserRequirements.cs @@ -1,4 +1,4 @@ -namespace MareSynchronosServer.RequirementHandlers; +namespace MareSynchronosShared.RequirementHandlers; public enum UserRequirements { diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs b/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs index 33e2832..ac584c9 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs @@ -6,12 +6,11 @@ public class ServerConfiguration : MareConfigurationAuthBase { public string RedisConnectionString { get; set; } = string.Empty; public int RedisPool { get; set; } = 50; - + [RemoteConfiguration] + public Version ExpectedClientVersion { get; set; } = new Version(0, 0, 0); [RemoteConfiguration] public Uri CdnFullUrl { get; set; } = null; [RemoteConfiguration] - public Uri StaticFileServiceAddress { get; set; } = null; - [RemoteConfiguration] public int MaxExistingGroupsByUser { get; set; } = 3; [RemoteConfiguration] public int MaxJoinedGroupsByUser { get; set; } = 6; @@ -21,18 +20,15 @@ public class ServerConfiguration : MareConfigurationAuthBase public bool PurgeUnusedAccounts { get; set; } = false; [RemoteConfiguration] public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14; - [RemoteConfiguration] - public List CdnShardConfiguration { get; set; } = new(); public override string ToString() { StringBuilder sb = new(); sb.AppendLine(base.ToString()); sb.AppendLine($"{nameof(CdnFullUrl)} => {CdnFullUrl}"); - sb.AppendLine($"{nameof(CdnShardConfiguration)} => {string.Join(", ", CdnShardConfiguration.Select(c => c.ToString()))}"); - sb.AppendLine($"{nameof(StaticFileServiceAddress)} => {StaticFileServiceAddress}"); sb.AppendLine($"{nameof(RedisConnectionString)} => {RedisConnectionString}"); sb.AppendLine($"{nameof(RedisPool)} => {RedisPool}"); + sb.AppendLine($"{nameof(ExpectedClientVersion)} => {ExpectedClientVersion}"); sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}"); sb.AppendLine($"{nameof(MaxJoinedGroupsByUser)} => {MaxJoinedGroupsByUser}"); sb.AppendLine($"{nameof(MaxGroupUserCount)} => {MaxGroupUserCount}"); diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs b/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs index f35e0fb..7404e8a 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs @@ -14,6 +14,10 @@ public class StaticFilesServerConfiguration : MareConfigurationBase public int DownloadTimeoutSeconds { get; set; } = 5; public int DownloadQueueReleaseSeconds { get; set; } = 15; public int DownloadQueueClearLimit { get; set; } = 15000; + [RemoteConfiguration] + public Uri CdnFullUrl { get; set; } = null; + [RemoteConfiguration] + public List CdnShardConfiguration { get; set; } = new(); public override string ToString() { StringBuilder sb = new(); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs index e4be17c..fd52936 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs @@ -60,18 +60,19 @@ public class RequestController : ControllerBase [Route(MareFiles.Request_RequestFile)] public async Task RequestFile(string file) { + Guid g = Guid.NewGuid(); + try { await _parallelRequestSemaphore.WaitAsync(HttpContext.RequestAborted); - Guid g = Guid.NewGuid(); _cachedFileProvider.DownloadFileWhenRequired(file); - await _requestQueue.EnqueueUser(new(g, MareUser, file)); return Ok(g); } catch (OperationCanceledException) { return BadRequest(); } finally { _parallelRequestSemaphore.Release(); + await _requestQueue.EnqueueUser(new(g, MareUser, file)); } } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs index 78e02e5..b23f44e 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs @@ -1,18 +1,146 @@ -using MareSynchronos.API.Routes; +using LZ4; +using MareSynchronos.API.Dto.Files; +using MareSynchronos.API.Routes; +using MareSynchronos.API.SignalR; +using MareSynchronosServer.Hubs; +using MareSynchronosShared.Data; +using MareSynchronosShared.Metrics; +using MareSynchronosShared.Models; +using MareSynchronosShared.Services; +using MareSynchronosShared.Utils; using MareSynchronosStaticFilesServer.Services; +using MareSynchronosStaticFilesServer.Utils; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.SignalR; +using Microsoft.EntityFrameworkCore; +using System.Collections.Concurrent; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.RegularExpressions; namespace MareSynchronosStaticFilesServer.Controllers; [Route(MareFiles.ServerFiles)] public class ServerFilesController : ControllerBase { + private static readonly ConcurrentDictionary _fileUploadLocks = new(StringComparer.Ordinal); + private readonly string _basePath; private readonly CachedFileProvider _cachedFileProvider; + private readonly IConfigurationService _configuration; + private readonly IHubContext _hubContext; + private readonly MareDbContext _mareDbContext; + private readonly MareMetrics _metricsClient; - public ServerFilesController(ILogger logger, CachedFileProvider cachedFileProvider) : base(logger) + public ServerFilesController(ILogger logger, CachedFileProvider cachedFileProvider, + IConfigurationService configuration, + IHubContext hubContext, + MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger) { + _basePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); _cachedFileProvider = cachedFileProvider; + _configuration = configuration; + _hubContext = hubContext; + _mareDbContext = mareDbContext; + _metricsClient = metricsClient; + } + + [HttpPost(MareFiles.ServerFiles_DeleteAll)] + public async Task FilesDeleteAll() + { + var ownFiles = await _mareDbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == MareUser).ToListAsync().ConfigureAwait(false); + + foreach (var dbFile in ownFiles) + { + var fi = FilePathUtil.GetFileInfoForHash(_basePath, dbFile.Hash); + if (fi != null) + { + _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1); + _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0); + + fi?.Delete(); + } + } + + _mareDbContext.Files.RemoveRange(ownFiles); + await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); + + return Ok(); + } + + [HttpGet(MareFiles.ServerFiles_GetSizes)] + public async Task FilesGetSizes([FromBody] List hashes) + { + var allFiles = await _mareDbContext.Files.Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false); + var forbiddenFiles = await _mareDbContext.ForbiddenUploadEntries. + Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false); + List response = new(); + + var cacheFile = await _mareDbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false); + + var shardConfig = new List(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List())); + + foreach (var file in cacheFile) + { + var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase)); + + var matchedShardConfig = shardConfig.OrderBy(g => Guid.NewGuid()).FirstOrDefault(f => new Regex(f.FileMatch).IsMatch(file.Hash)); + var baseUrl = matchedShardConfig?.CdnFullUrl ?? _configuration.GetValue(nameof(StaticFilesServerConfiguration.CdnFullUrl)); + + response.Add(new DownloadFileDto + { + FileExists = file.Size > 0, + ForbiddenBy = forbiddenFile?.ForbiddenBy ?? string.Empty, + IsForbidden = forbiddenFile != null, + Hash = file.Hash, + Size = file.Size, + Url = baseUrl.ToString(), + }); + } + + return Ok(JsonSerializer.Serialize(response)); + } + + [HttpPost(MareFiles.ServerFiles_FilesSend)] + public async Task FilesSend([FromBody] FilesSendDto filesSendDto) + { + var userSentHashes = new HashSet(filesSendDto.FileHashes.Distinct(StringComparer.Ordinal).Select(s => string.Concat(s.Where(c => char.IsLetterOrDigit(c)))), StringComparer.Ordinal); + var notCoveredFiles = new Dictionary(StringComparer.Ordinal); + var forbiddenFiles = await _mareDbContext.ForbiddenUploadEntries.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false); + var existingFiles = await _mareDbContext.Files.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false); + + List fileCachesToUpload = new(); + foreach (var hash in userSentHashes) + { + // Skip empty file hashes, duplicate file hashes, forbidden file hashes and existing file hashes + if (string.IsNullOrEmpty(hash)) { continue; } + if (notCoveredFiles.ContainsKey(hash)) { continue; } + if (forbiddenFiles.ContainsKey(hash)) + { + notCoveredFiles[hash] = new UploadFileDto() + { + ForbiddenBy = forbiddenFiles[hash].ForbiddenBy, + Hash = hash, + IsForbidden = true, + }; + + continue; + } + if (existingFiles.TryGetValue(hash, out var file) && file.Uploaded) { continue; } + + notCoveredFiles[hash] = new UploadFileDto() + { + Hash = hash, + }; + } + + if (notCoveredFiles.Any()) + { + await _hubContext.Clients.Users(filesSendDto.UIDs).SendAsync(nameof(IMareHub.Client_UserReceiveUploadStatus), new MareSynchronos.API.Dto.User.UserDto(new(MareUser))) + .ConfigureAwait(false); + } + + return Ok(JsonSerializer.Serialize(notCoveredFiles.Values.ToList())); } [HttpGet(MareFiles.ServerFiles_Get + "/{fileId}")] @@ -26,4 +154,93 @@ public class ServerFilesController : ControllerBase return File(fs, "application/octet-stream"); } -} + + [HttpPost(MareFiles.ServerFiles_Upload + "/{hash}")] + [RequestSizeLimit(200 * 1024 * 1024)] + public async Task UploadFile(string hash, CancellationToken requestAborted) + { + _logger.LogInformation("{user} uploading file {file}", MareUser, hash); + bool initiated = false; + hash = hash.ToUpperInvariant(); + var existingFile = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash); + if (existingFile != null) return Ok(); + + if (!_fileUploadLocks.TryGetValue(hash, out var fileLock)) + { + initiated = true; + _fileUploadLocks[hash] = fileLock = new SemaphoreSlim(1); + await fileLock.WaitAsync(requestAborted).ConfigureAwait(false); + } + + if (!initiated) + { + try + { + await fileLock.WaitAsync(requestAborted).ConfigureAwait(false); + var file = await _mareDbContext.Files.SingleOrDefaultAsync(c => c.Hash == hash).ConfigureAwait(false); + if (file != null) + { + fileLock.Release(); + return Ok(); + } + } + catch (OperationCanceledException) + { + fileLock.Release(); + return Ok(); + } + } + + try + { + // copy the request body to memory + using var compressedFileStream = new MemoryStream(); + await Request.Body.CopyToAsync(compressedFileStream, requestAborted).ConfigureAwait(false); + + // decompress and copy the decompressed stream to memory + var data = LZ4Codec.Unwrap(compressedFileStream.ToArray()); + + // reset streams + compressedFileStream.Seek(0, SeekOrigin.Begin); + + // compute hash to verify + var hashString = BitConverter.ToString(SHA1.HashData(data)) + .Replace("-", "", StringComparison.Ordinal).ToUpperInvariant(); + if (!string.Equals(hashString, hash, StringComparison.Ordinal)) + throw new InvalidOperationException($"Hash does not match file, computed: {hashString}, expected: {hash}"); + + // save file + var path = FilePathUtil.GetFilePath(_basePath, hash); + using var fileStream = new FileStream(path, FileMode.Create); + await compressedFileStream.CopyToAsync(fileStream).ConfigureAwait(false); + + // update on db + await _mareDbContext.Files.AddAsync(new FileCache() + { + Hash = hash, + UploadDate = DateTime.UtcNow, + UploaderUID = MareUser, + Size = compressedFileStream.Length, + Uploaded = true + }).ConfigureAwait(false); + await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); + + _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotal, 1); + _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotalSize, compressedFileStream.Length); + + _fileUploadLocks.Remove(hash, out _); + } + catch (Exception e) + { + _logger.LogError(e, "Error during file upload"); + return BadRequest(); + } + finally + { + fileLock.Release(); + _fileUploadLocks.TryRemove(hash, out _); + } + + return Ok(); + } +} \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj b/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj index 0466482..b470b07 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj @@ -18,17 +18,15 @@ - - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + - + diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/GrpcFileService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/GrpcFileService.cs deleted file mode 100644 index dc12453..0000000 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/GrpcFileService.cs +++ /dev/null @@ -1,105 +0,0 @@ -using Grpc.Core; -using MareSynchronosShared.Data; -using MareSynchronosShared.Metrics; -using MareSynchronosShared.Protos; -using MareSynchronosShared.Services; -using MareSynchronosStaticFilesServer.Utils; -using Microsoft.AspNetCore.Authorization; -using Microsoft.EntityFrameworkCore; - -namespace MareSynchronosStaticFilesServer.Services; - -[Authorize(Policy = "Internal")] -public class GrpcFileService : FileService.FileServiceBase -{ - private readonly string _basePath; - private readonly MareDbContext _mareDbContext; - private readonly ILogger _logger; - private readonly MareMetrics _metricsClient; - - public GrpcFileService(MareDbContext mareDbContext, IConfigurationService configuration, ILogger logger, MareMetrics metricsClient) - { - _basePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); - _mareDbContext = mareDbContext; - _logger = logger; - _metricsClient = metricsClient; - } - - [Authorize(Policy = "Internal")] - public override async Task UploadFile(IAsyncStreamReader requestStream, ServerCallContext context) - { - _ = await requestStream.MoveNext().ConfigureAwait(false); - var uploadMsg = requestStream.Current; - var filePath = FilePathUtil.GetFilePath(_basePath, uploadMsg.Hash); - using var fileWriter = File.OpenWrite(filePath); - var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == uploadMsg.Hash && f.UploaderUID == uploadMsg.Uploader).ConfigureAwait(false); - try - { - if (file != null) - { - await fileWriter.WriteAsync(uploadMsg.FileData.ToArray()).ConfigureAwait(false); - - while (await requestStream.MoveNext().ConfigureAwait(false)) - { - await fileWriter.WriteAsync(requestStream.Current.FileData.ToArray()).ConfigureAwait(false); - } - - await fileWriter.FlushAsync().ConfigureAwait(false); - fileWriter.Close(); - - var fileSize = new FileInfo(filePath).Length; - file.Uploaded = true; - file.Size = fileSize; - - await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); - - _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotal, 1); - _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotalSize, fileSize); - - _logger.LogInformation("User {user} uploaded file {hash}", uploadMsg.Uploader, uploadMsg.Hash); - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Error during UploadFile"); - var fileNew = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == uploadMsg.Hash && f.UploaderUID == uploadMsg.Uploader).ConfigureAwait(false); - if (fileNew != null) - { - _mareDbContext.Files.Remove(fileNew); - } - - await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); - } - - return new Empty(); - } - - [Authorize(Policy = "Internal")] - public override async Task DeleteFiles(DeleteFilesRequest request, ServerCallContext context) - { - foreach (var hash in request.Hash) - { - try - { - var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash); - var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash).ConfigureAwait(false); - if (file != null && fi != null) - { - _mareDbContext.Files.Remove(file); - await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); - - _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1); - _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0); - - fi?.Delete(); - } - } - catch (Exception ex) - { - _logger.LogError(ex, "Could not delete file for hash {hash}", hash); - } - } - - return new Empty(); - } -} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs index 76ea2ba..0dc1e01 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs @@ -10,17 +10,17 @@ namespace MareSynchronosStaticFilesServer.Services; public class RequestQueueService : IHostedService { - private readonly UserQueueEntry[] _userQueueRequests; - private readonly ConcurrentQueue _queue = new(); - private readonly MareMetrics _metrics; - private readonly ILogger _logger; private readonly IHubContext _hubContext; + private readonly ILogger _logger; + private readonly MareMetrics _metrics; + private readonly ConcurrentQueue _queue = new(); private readonly int _queueExpirationSeconds; - private readonly SemaphoreSlim _queueSemaphore = new(1); private readonly SemaphoreSlim _queueProcessingSemaphore = new(1); + private readonly ConcurrentDictionary _queueRemoval = new(); + private readonly SemaphoreSlim _queueSemaphore = new(1); + private readonly UserQueueEntry[] _userQueueRequests; private int _queueLimitForReset; private System.Timers.Timer _queueTimer; - private readonly ConcurrentDictionary _queueRemoval = new(); public RequestQueueService(MareMetrics metrics, IConfigurationService configurationService, ILogger logger, IHubContext hubContext) { @@ -32,6 +32,12 @@ public class RequestQueueService : IHostedService _hubContext = hubContext; } + public void ActivateRequest(Guid request) + { + _logger.LogDebug("Activating request {guid}", request); + _userQueueRequests.First(f => f != null && f.UserRequest.RequestId == request).IsActive = true; + } + public async Task EnqueueUser(UserRequest request) { _logger.LogDebug("Enqueueing req {guid} from {user} for {file}", request.RequestId, request.User, request.FileId); @@ -39,20 +45,13 @@ public class RequestQueueService : IHostedService if (_queueProcessingSemaphore.CurrentCount == 0) { _queue.Enqueue(request); + return; } try { await _queueSemaphore.WaitAsync().ConfigureAwait(false); - var idx = Array.FindIndex(_userQueueRequests, r => r == null); - if (idx == -1) - { - _queue.Enqueue(request); - } - else - { - await DequeueIntoSlotAsync(request, idx).ConfigureAwait(false); - } + _queue.Enqueue(request); return; } @@ -68,6 +67,21 @@ public class RequestQueueService : IHostedService throw new Exception("Error during EnqueueUser"); } + public void FinishRequest(Guid request) + { + var req = _userQueueRequests.First(f => f != null && f.UserRequest.RequestId == request); + var idx = Array.IndexOf(_userQueueRequests, req); + _logger.LogDebug("Finishing Request {guid}, clearing slot {idx}", request, idx); + _userQueueRequests[idx] = null; + } + + public bool IsActiveProcessing(Guid request, string user, out UserRequest userRequest) + { + var userQueueRequest = _userQueueRequests.FirstOrDefault(u => u != null && u.UserRequest.RequestId == request && string.Equals(u.UserRequest.User, user, StringComparison.Ordinal)); + userRequest = userQueueRequest?.UserRequest ?? null; + return userQueueRequest != null && userRequest != null && userQueueRequest.ExpirationDate > DateTime.UtcNow; + } + public void RemoveFromQueue(Guid requestId, string user) { if (!_queue.Any(f => f.RequestId == requestId && string.Equals(f.User, user, StringComparison.Ordinal))) @@ -87,30 +101,31 @@ public class RequestQueueService : IHostedService _queueRemoval[requestId] = user; } + public Task StartAsync(CancellationToken cancellationToken) + { + _queueTimer = new System.Timers.Timer(250); + _queueTimer.Elapsed += ProcessQueue; + _queueTimer.AutoReset = true; + _queueTimer.Start(); + return Task.CompletedTask; + } + public bool StillEnqueued(Guid request, string user) { return _queue.Any(c => c.RequestId == request && string.Equals(c.User, user, StringComparison.Ordinal)); } - public bool IsActiveProcessing(Guid request, string user, out UserRequest userRequest) + public Task StopAsync(CancellationToken cancellationToken) { - var userQueueRequest = _userQueueRequests.FirstOrDefault(u => u != null && u.UserRequest.RequestId == request && string.Equals(u.UserRequest.User, user, StringComparison.Ordinal)); - userRequest = userQueueRequest?.UserRequest ?? null; - return userQueueRequest != null && userRequest != null && userQueueRequest.ExpirationDate > DateTime.UtcNow; + _queueTimer.Stop(); + return Task.CompletedTask; } - public void FinishRequest(Guid request) + private async Task DequeueIntoSlotAsync(UserRequest userRequest, int slot) { - var req = _userQueueRequests.First(f => f != null && f.UserRequest.RequestId == request); - var idx = Array.IndexOf(_userQueueRequests, req); - _logger.LogDebug("Finishing Request {guid}, clearing slot {idx}", request, idx); - _userQueueRequests[idx] = null; - } - - public void ActivateRequest(Guid request) - { - _logger.LogDebug("Activating request {guid}", request); - _userQueueRequests.First(f => f != null && f.UserRequest.RequestId == request).IsActive = true; + _logger.LogDebug("Dequeueing {req} into {i}: {user} with {file}", userRequest.RequestId, slot, userRequest.User, userRequest.FileId); + _userQueueRequests[slot] = new(userRequest, DateTime.UtcNow.AddSeconds(_queueExpirationSeconds)); + await _hubContext.Clients.User(userRequest.User).SendAsync(nameof(IMareHub.Client_DownloadReady), userRequest.RequestId).ConfigureAwait(false); } private async void ProcessQueue(object src, ElapsedEventArgs e) @@ -161,7 +176,6 @@ public class RequestQueueService : IHostedService } } }); - } catch (Exception ex) { @@ -174,26 +188,4 @@ public class RequestQueueService : IHostedService _metrics.SetGaugeTo(MetricsAPI.GaugeDownloadQueue, _queue.Count); } - - private async Task DequeueIntoSlotAsync(UserRequest userRequest, int slot) - { - _logger.LogDebug("Dequeueing {req} into {i}: {user} with {file}", userRequest.RequestId, slot, userRequest.User, userRequest.FileId); - _userQueueRequests[slot] = new(userRequest, DateTime.UtcNow.AddSeconds(_queueExpirationSeconds)); - await _hubContext.Clients.User(userRequest.User).SendAsync(nameof(IMareHub.Client_DownloadReady), userRequest.RequestId).ConfigureAwait(false); - } - - public Task StartAsync(CancellationToken cancellationToken) - { - _queueTimer = new System.Timers.Timer(250); - _queueTimer.Elapsed += ProcessQueue; - _queueTimer.AutoReset = true; - _queueTimer.Start(); - return Task.CompletedTask; - } - - public Task StopAsync(CancellationToken cancellationToken) - { - _queueTimer.Stop(); - return Task.CompletedTask; - } -} +} \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs index 89bfbe2..ef49719 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs @@ -1,4 +1,3 @@ -using Grpc.Net.Client.Configuration; using MareSynchronosShared.Data; using MareSynchronosShared.Metrics; using MareSynchronosShared.Services; @@ -16,6 +15,10 @@ using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; using Microsoft.IdentityModel.Tokens; using Prometheus; +using StackExchange.Redis.Extensions.Core.Configuration; +using StackExchange.Redis.Extensions.System.Text.Json; +using StackExchange.Redis; +using System.Net; using System.Text; namespace MareSynchronosStaticFilesServer; @@ -77,12 +80,6 @@ public class Startup options.EnableThreadSafetyChecks(false); }, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024)); - var noRetryConfig = new MethodConfig - { - Names = { MethodName.Default }, - RetryPolicy = null, - }; - services.AddOptions(JwtBearerDefaults.AuthenticationScheme) .Configure>((o, s) => { @@ -111,11 +108,6 @@ public class Startup if (_isMain) { - services.AddGrpc(o => - { - o.MaxReceiveMessageSize = null; - }); - services.AddSingleton, MareConfigurationServiceServer>(); } else @@ -174,6 +166,39 @@ public class Startup var redisConnection = mareConfig.GetValue(nameof(ServerConfiguration.RedisConnectionString), string.Empty); signalRServiceBuilder.AddStackExchangeRedis(redisConnection, options => { }); + var options = ConfigurationOptions.Parse(redisConnection); + + var endpoint = options.EndPoints[0]; + string address = ""; + int port = 0; + if (endpoint is DnsEndPoint dnsEndPoint) { address = dnsEndPoint.Host; port = dnsEndPoint.Port; } + if (endpoint is IPEndPoint ipEndPoint) { address = ipEndPoint.Address.ToString(); port = ipEndPoint.Port; } + var redisConfiguration = new RedisConfiguration() + { + AbortOnConnectFail = true, + KeyPrefix = "", + Hosts = new RedisHost[] + { + new RedisHost(){ Host = address, Port = port }, + }, + AllowAdmin = true, + ConnectTimeout = options.ConnectTimeout, + Database = 0, + Ssl = false, + Password = options.Password, + ServerEnumerationStrategy = new ServerEnumerationStrategy() + { + Mode = ServerEnumerationStrategy.ModeOptions.All, + TargetRole = ServerEnumerationStrategy.TargetRoleOptions.Any, + UnreachableServerAction = ServerEnumerationStrategy.UnreachableServerActionOptions.Throw, + }, + MaxValueLength = 1024, + PoolSize = mareConfig.GetValue(nameof(ServerConfiguration.RedisPool), 50), + SyncTimeout = options.SyncTimeout, + }; + + services.AddStackExchangeRedisExtensions(redisConfiguration); + services.AddHealthChecks(); } @@ -195,10 +220,6 @@ public class Startup app.UseEndpoints(e => { - if (_isMain) - { - e.MapGrpcService(); - } e.MapHub("/dummyhub"); e.MapControllers(); e.MapHealthChecks("/health").WithMetadata(new AllowAnonymousAttribute());