[Draft] Update 0.8 (#25)

* get rid of file handling through grpc and signalr

* fix upload on controller

* adapt usersetpairpermissions

* send user perms

* server-side fixes

* rework file upload

* adjust log level to debug in docker standalone json

* update dependencies

---------

Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>

Docker/run/compose/mare-sharded.yml

@@ -37,11 +37,7 @@ services:
     image: darkarchon/mare-synchronos-server:latest
     restart: on-failure
     environment:
-      MareSynchronos__CdnFullUrl: "http://darkarchon.internet-box.ch:9999"
-      MareSynchronos__CdnShardConfiguration__0__CdnFullUrl: "${DEV_MARE_CDNURL}"
-      MareSynchronos__CdnShardConfiguration__0__FileMatch: "^[012345678]"
-      MareSynchronos__CdnShardConfiguration__1__CdnFullUrl: "${DEV_MARE_CDNURL2}"
-      MareSynchronos__CdnShardConfiguration__1__FileMatch: "^[789ABCDEF]"
+      MareSynchronos__CdnFullUrl: "${DEV_MARE_CDNURL}"
     volumes:
       - ../config/sharded/server-shard-main.json:/opt/MareSynchronosServer/appsettings.json
       - ../log/server-shard-main/:/opt/MareSynchronosServer/logs/:rw
@@ -94,6 +90,13 @@ services:
   mare-files:
     image: darkarchon/mare-synchronos-staticfilesserver:latest
     restart: on-failure
+    ports:
+      - 6200:6200/tcp
+    environment:
+      MareSynchronos__CdnShardConfiguration__0__CdnFullUrl: "${DEV_MARE_FILES1}"
+      MareSynchronos__CdnShardConfiguration__0__FileMatch: "^[012345678]"
+      MareSynchronos__CdnShardConfiguration__1__CdnFullUrl: "${DEV_MARE_FILES2}"
+      MareSynchronos__CdnShardConfiguration__1__FileMatch: "^[789ABCDEF]"
     volumes:
       - ../config/sharded/files-shard-main.json:/opt/MareSynchronosStaticFilesServer/appsettings.json
       - ../log/files-standalone/:/opt/MareSynchronosStaticFilesServer/logs/:rw
@@ -117,7 +120,7 @@ services:
       - postgres_socket:/var/run/postgresql/:rw
       - ../data/files-shard-1/:/marecache/:rw
     ports: 
-      - 6200:6200/tcp
+      - 6201:6200/tcp
     depends_on:
       mare-files:
         condition: service_healthy
@@ -131,7 +134,7 @@ services:
       - postgres_socket:/var/run/postgresql/:rw
       - ../data/files-shard-2/:/marecache/:rw
     ports: 
-      - 6201:6200/tcp
+      - 6202:6200/tcp
     depends_on:
       mare-files:
         condition: service_healthy

Docker/run/compose/mare-standalone.yml

@@ -65,6 +65,7 @@ services:
       - 6200:6200/tcp
     restart: on-failure
     environment:
+      MareSynchronos__CdnFullUrl: "${DEV_MARE_CDNURL}"
       DOTNET_USE_POLLING_FILE_WATCHER: 1
     volumes:
       - ../config/standalone/files-standalone.json:/opt/MareSynchronosStaticFilesServer/appsettings.json

Docker/run/config/standalone/files-standalone.json

@@ -4,11 +4,11 @@
     },
     "Logging": {
         "LogLevel": {
-            "Default": "Warning",
-            "Microsoft": "Warning",
+            "Default": "Debug",
+            "Microsoft": "Debug",
             "Microsoft.Hosting.Lifetime": "Information",
-            "MareSynchronosStaticFilesServer": "Information",
-            "MareSynchronosShared": "Information",
+            "MareSynchronosStaticFilesServer": "Debug",
+            "MareSynchronosShared": "Debug",
             "System.IO": "Information"
         },
         "File": {
@@ -44,10 +44,6 @@
         "Endpoints": {
             "Http": {
                 "Url": "http://+:6200"
-            },
-            "Grpc": {
-                "Protocols": "Http2",
-                "Url": "http://+:6205"
             }
         }
     },

Docker/run/config/standalone/server-standalone.json

@@ -36,13 +36,13 @@
             ""
         ],
         "RedisConnectionString": "redis,password=secretredispassword",
-        "CdnFullUrl": "http://localhost:6200/",
-        "StaticFileServiceAddress": "http://mare-files:6205",
+        "CdnFullUrl": "http://localhost:6200",
         "MaxExistingGroupsByUser": 3,
         "MaxJoinedGroupsByUser": 6,
         "MaxGroupUserCount": 100,
         "PurgeUnusedAccounts": false,
-        "PurgeUnusedAccountsPeriodInDays": 14
+        "PurgeUnusedAccountsPeriodInDays": 14,
+        "ExpectedClientVersion": "0.8.0"
     },
     "AllowedHosts": "*",
     "Kestrel": {

Docker/run/config/standalone/services-standalone.json

@@ -28,6 +28,7 @@
         "DbContextPoolSize": 512,
         "ShardName": "Services",
         "MetricsPort": 6150,
+        "CdnFullUrl": "http://localhost:6200/",
         "MainServerAddress": "http://mare-server:6000/",
         "MainServerGrpcAddress": "http://mare-server:6005/",
         "DiscordBotToken": "",

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs

@@ -1,253 +0,0 @@
-using System.Security.Cryptography;
-using System.Text.RegularExpressions;
-using Google.Protobuf;
-using Grpc.Core;
-using MareSynchronos.API.Dto.Files;
-using MareSynchronosServer.Utils;
-using MareSynchronosShared.Models;
-using MareSynchronosShared.Protos;
-using MareSynchronosShared.Utils;
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.EntityFrameworkCore;
-
-namespace MareSynchronosServer.Hubs;
-
-public partial class MareHub
-{
-    private static readonly SemaphoreSlim _uploadSemaphore = new(20);
-
-    [Authorize(Policy = "Identified")]
-    public async Task FilesAbortUpload()
-    {
-        _logger.LogCallInfo();
-        var notUploadedFiles = await _dbContext.Files.Where(f => !f.Uploaded && f.Uploader.UID == UserUID).ToListAsync();
-        if (notUploadedFiles.Any())
-        {
-            _dbContext.RemoveRange(notUploadedFiles);
-            await _dbContext.SaveChangesAsync().ConfigureAwait(false);
-        }
-    }
-
-    [Authorize(Policy = "Identified")]
-    public async Task FilesDeleteAll()
-    {
-        _logger.LogCallInfo();
-
-        var ownFiles = await _dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == UserUID).ToListAsync().ConfigureAwait(false);
-        var request = new DeleteFilesRequest();
-        request.Hash.AddRange(ownFiles.Select(f => f.Hash));
-        Metadata headers = new Metadata()
-            {
-                { "Authorization", "Bearer " + _generator.Token },
-            };
-        _ = await _fileServiceClient.DeleteFilesAsync(request, headers).ConfigureAwait(false);
-    }
-
-    [Authorize(Policy = "Identified")]
-    public async Task<List<DownloadFileDto>> FilesGetSizes(List<string> hashes)
-    {
-        _logger.LogCallInfo(MareHubLogger.Args(hashes.Count.ToString()));
-
-        var allFiles = await _dbContext.Files.Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false);
-        var forbiddenFiles = await _dbContext.ForbiddenUploadEntries.
-            Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false);
-        List<DownloadFileDto> response = new();
-
-        var cacheFile = await _dbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false);
-
-        var shardConfig = new List<CdnShardConfiguration>(_configurationService.GetValueOrDefault(nameof(ServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>()));
-
-        foreach (var file in cacheFile)
-        {
-            var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
-
-            var matchedShardConfig = shardConfig.OrderBy(g => Guid.NewGuid()).FirstOrDefault(f => new Regex(f.FileMatch).IsMatch(file.Hash));
-            var baseUrl = matchedShardConfig?.CdnFullUrl ?? _mainCdnFullUrl;
-
-            response.Add(new DownloadFileDto
-            {
-                FileExists = file.Size > 0,
-                ForbiddenBy = forbiddenFile?.ForbiddenBy ?? string.Empty,
-                IsForbidden = forbiddenFile != null,
-                Hash = file.Hash,
-                Size = file.Size,
-                Url = baseUrl.ToString(),
-            });
-        }
-
-        return response;
-    }
-
-    [Authorize(Policy = "Identified")]
-    public async Task<bool> FilesIsUploadFinished()
-    {
-        _logger.LogCallInfo();
-        return await _dbContext.Files.AsNoTracking()
-            .AnyAsync(f => f.Uploader.UID == UserUID && !f.Uploaded).ConfigureAwait(false);
-    }
-
-    [Authorize(Policy = "Identified")]
-    public async Task<List<UploadFileDto>> FilesSend(List<string> fileListHashes)
-    {
-        var userSentHashes = new HashSet<string>(fileListHashes.Distinct(StringComparer.Ordinal).Select(s => string.Concat(s.Where(c => char.IsLetterOrDigit(c)))), StringComparer.Ordinal);
-        _logger.LogCallInfo(MareHubLogger.Args(userSentHashes.Count.ToString()));
-        var notCoveredFiles = new Dictionary<string, UploadFileDto>(StringComparer.Ordinal);
-        var forbiddenFiles = await _dbContext.ForbiddenUploadEntries.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
-        var existingFiles = await _dbContext.Files.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
-        var uploader = await _dbContext.Users.SingleAsync(u => u.UID == UserUID).ConfigureAwait(false);
-
-        List<FileCache> fileCachesToUpload = new();
-        foreach (var hash in userSentHashes)
-        {
-            // Skip empty file hashes, duplicate file hashes, forbidden file hashes and existing file hashes
-            if (string.IsNullOrEmpty(hash)) { continue; }
-            if (notCoveredFiles.ContainsKey(hash)) { continue; }
-            if (forbiddenFiles.ContainsKey(hash))
-            {
-                notCoveredFiles[hash] = new UploadFileDto()
-                {
-                    ForbiddenBy = forbiddenFiles[hash].ForbiddenBy,
-                    Hash = hash,
-                    IsForbidden = true,
-                };
-
-                continue;
-            }
-            if (existingFiles.TryGetValue(hash, out var file) && file.Uploaded) { continue; }
-
-            _logger.LogCallInfo(MareHubLogger.Args(hash, "Missing"));
-
-            if (file == null)
-            {
-                fileCachesToUpload.Add(new FileCache()
-                {
-                    Hash = hash,
-                    Uploaded = false,
-                    Uploader = uploader,
-                    UploadDate = DateTime.UtcNow,
-                });
-            }
-
-            notCoveredFiles[hash] = new UploadFileDto()
-            {
-                Hash = hash,
-            };
-        }
-        //Save bulk
-        await _dbContext.Files.AddRangeAsync(fileCachesToUpload).ConfigureAwait(false);
-        await _dbContext.SaveChangesAsync().ConfigureAwait(false);
-        return notCoveredFiles.Values.ToList();
-    }
-
-    [Authorize(Policy = "Identified")]
-    public async Task FilesUploadStreamAsync(string hash, IAsyncEnumerable<byte[]> fileContent)
-    {
-        _logger.LogCallInfo(MareHubLogger.Args(hash));
-
-        await _uploadSemaphore.WaitAsync(Context.ConnectionAborted).ConfigureAwait(false);
-
-        var relatedFile = _dbContext.Files.SingleOrDefault(f => f.Hash == hash && f.Uploader.UID == UserUID && !f.Uploaded);
-        if (relatedFile == null)
-        {
-            _uploadSemaphore.Release();
-            return;
-        }
-        var forbiddenFile = _dbContext.ForbiddenUploadEntries.SingleOrDefault(f => f.Hash == hash);
-        if (forbiddenFile != null)
-        {
-            _uploadSemaphore.Release();
-            return;
-        }
-
-        var tempFileName = Path.GetTempFileName();
-        using var fileStream = new FileStream(tempFileName, FileMode.OpenOrCreate);
-        long length = 0;
-        try
-        {
-            await foreach (var chunk in fileContent.ConfigureAwait(false))
-            {
-                length += chunk.Length;
-                await fileStream.WriteAsync(chunk).ConfigureAwait(false);
-            }
-
-            await fileStream.FlushAsync().ConfigureAwait(false);
-            await fileStream.DisposeAsync().ConfigureAwait(false);
-        }
-        catch
-        {
-            try
-            {
-                await fileStream.FlushAsync().ConfigureAwait(false);
-                await fileStream.DisposeAsync().ConfigureAwait(false);
-                _dbContext.Files.Remove(relatedFile);
-                await _dbContext.SaveChangesAsync().ConfigureAwait(false);
-            }
-            catch
-            {
-                // already removed
-            }
-            finally
-            {
-                File.Delete(tempFileName);
-            }
-
-            _uploadSemaphore.Release();
-            return;
-        }
-
-        _logger.LogCallInfo(MareHubLogger.Args(hash, "Uploaded"));
-
-        try
-        {
-            var decodedFile = LZ4.LZ4Codec.Unwrap(await File.ReadAllBytesAsync(tempFileName).ConfigureAwait(false));
-            using var sha1 = SHA1.Create();
-            using var ms = new MemoryStream(decodedFile);
-            var computedHash = await sha1.ComputeHashAsync(ms).ConfigureAwait(false);
-            var computedHashString = BitConverter.ToString(computedHash).Replace("-", "", StringComparison.Ordinal);
-            if (!string.Equals(hash, computedHashString, StringComparison.Ordinal))
-            {
-                _logger.LogCallWarning(MareHubLogger.Args(hash, "Invalid", computedHashString));
-                _dbContext.Remove(relatedFile);
-                await _dbContext.SaveChangesAsync().ConfigureAwait(false);
-
-                _uploadSemaphore.Release();
-                return;
-            }
-
-            Metadata headers = new Metadata()
-            {
-                { "Authorization", "Bearer " + _generator.Token },
-            };
-            var streamingCall = _fileServiceClient.UploadFile(headers);
-            using var tempFileStream = new FileStream(tempFileName, FileMode.Open, FileAccess.Read);
-            int size = 1024 * 1024;
-            byte[] data = new byte[size];
-            int readBytes;
-            while ((readBytes = tempFileStream.Read(data, 0, size)) > 0)
-            {
-                await streamingCall.RequestStream.WriteAsync(new UploadFileRequest()
-                {
-                    FileData = ByteString.CopyFrom(data, 0, readBytes),
-                    Hash = computedHashString,
-                    Uploader = UserUID,
-                }).ConfigureAwait(false);
-            }
-            await streamingCall.RequestStream.CompleteAsync().ConfigureAwait(false);
-            tempFileStream.Close();
-            await tempFileStream.DisposeAsync().ConfigureAwait(false);
-
-            _logger.LogCallInfo(MareHubLogger.Args(hash, "Pushed"));
-        }
-        catch (Exception ex)
-        {
-            _logger.LogCallWarning(MareHubLogger.Args("Failed", hash, ex.Message));
-            _dbContext.Remove(relatedFile);
-            await _dbContext.SaveChangesAsync().ConfigureAwait(false);
-        }
-        finally
-        {
-            _uploadSemaphore.Release();
-            File.Delete(tempFileName);
-        }
-    }
-}

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Functions.cs

@@ -30,7 +30,7 @@ public partial class MareHub
 
     private async Task<Dictionary<string, string>> GetOnlineUsers(List<string> uids)
     {
-        var result = await _redis.GetAllAsync<string>(uids.Select(u => "UID:" + u).ToArray()).ConfigureAwait(false);
+        var result = await _redis.GetAllAsync<string>(uids.Select(u => "UID:" + u).ToHashSet(StringComparer.Ordinal)).ConfigureAwait(false);
         return uids.Where(u => result.TryGetValue("UID:" + u, out var ident) && !string.IsNullOrEmpty(ident)).ToDictionary(u => u, u => result["UID:" + u], StringComparer.Ordinal);
     }
 

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.User.cs

@@ -68,6 +68,10 @@ public partial class MareHub
                 OtherIsPaused = otherEntry != null && otherEntry.IsPaused,
                 userToOther.OtherUserUID,
                 IsSynced = otherEntry != null,
+                DisableOwnAnimations = userToOther.DisableAnimations,
+                DisableOwnSounds = userToOther.DisableSounds,
+                DisableOtherAnimations = otherEntry == null ? false : otherEntry.DisableAnimations,
+                DisableOtherSounds = otherEntry == null ? false : otherEntry.DisableSounds
             };
 
         var results = await query.AsNoTracking().ToListAsync().ConfigureAwait(false);
@@ -76,9 +80,13 @@ public partial class MareHub
         {
             var ownPerm = UserPermissions.Paired;
             ownPerm.SetPaused(c.IsPaused);
+            ownPerm.SetDisableAnimations(c.DisableOwnAnimations);
+            ownPerm.SetDisableSounds(c.DisableOwnSounds);
             var otherPerm = UserPermissions.NoneSet;
             otherPerm.SetPaired(c.IsSynced);
             otherPerm.SetPaused(c.OtherIsPaused);
+            otherPerm.SetDisableAnimations(c.DisableOtherAnimations);
+            otherPerm.SetDisableSounds(c.DisableOtherSounds);
             return new UserPairDto(new(c.OtherUserUID, c.Alias), ownPerm, otherPerm);
         }).ToList();
     }
@@ -215,7 +223,11 @@ public partial class MareHub
         ClientPair pair = await _dbContext.ClientPairs.SingleOrDefaultAsync(w => w.UserUID == UserUID && w.OtherUserUID == dto.User.UID).ConfigureAwait(false);
         if (pair == null) return;
 
+        var pauseChange = pair.IsPaused != dto.Permissions.IsPaused();
+
         pair.IsPaused = dto.Permissions.IsPaused();
+        pair.DisableAnimations = dto.Permissions.IsDisableAnimations();
+        pair.DisableSounds = dto.Permissions.IsDisableSounds();
         _dbContext.Update(pair);
         await _dbContext.SaveChangesAsync().ConfigureAwait(false);
 
@@ -229,6 +241,8 @@ public partial class MareHub
         {
             await Clients.User(dto.User.UID).Client_UserUpdateOtherPairPermissions(new UserPermissionsDto(new UserData(UserUID), dto.Permissions)).ConfigureAwait(false);
 
+            if (pauseChange)
+            {
                 var otherCharaIdent = await GetUserIdent(pair.OtherUserUID).ConfigureAwait(false);
 
                 if (UserCharaIdent == null || otherCharaIdent == null || otherEntry.IsPaused) return;
@@ -245,6 +259,7 @@ public partial class MareHub
                 }
             }
         }
+    }
 
     [Authorize(Policy = "Identified")]
     public async Task UserRemovePair(UserDto dto)

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs

@@ -7,7 +7,6 @@ using MareSynchronosServer.Utils;
 using MareSynchronosShared;
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
-using MareSynchronosShared.Protos;
 using MareSynchronosShared.Services;
 using MareSynchronosShared.Utils;
 using Microsoft.AspNetCore.Authorization;
@@ -20,37 +19,33 @@ namespace MareSynchronosServer.Hubs;
 public partial class MareHub : Hub<IMareHub>, IMareHub
 {
     private readonly MareMetrics _mareMetrics;
-    private readonly FileService.FileServiceClient _fileServiceClient;
     private readonly SystemInfoService _systemInfoService;
     private readonly IHttpContextAccessor _contextAccessor;
     private readonly MareHubLogger _logger;
     private readonly MareDbContext _dbContext;
-    private readonly Uri _mainCdnFullUrl;
     private readonly string _shardName;
     private readonly int _maxExistingGroupsByUser;
     private readonly int _maxJoinedGroupsByUser;
     private readonly int _maxGroupUserCount;
-    private readonly IConfigurationService<ServerConfiguration> _configurationService;
     private readonly IRedisDatabase _redis;
-    private readonly ServerTokenGenerator _generator;
+    private readonly Uri _fileServerAddress;
+    private readonly Version _expectedClientVersion;
 
-    public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient,
+    public MareHub(MareMetrics mareMetrics,
         MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService,
         IConfigurationService<ServerConfiguration> configuration, IHttpContextAccessor contextAccessor,
-        IRedisDatabase redisDb, ServerTokenGenerator generator)
+        IRedisDatabase redisDb)
     {
         _mareMetrics = mareMetrics;
-        _fileServiceClient = fileServiceClient;
         _systemInfoService = systemInfoService;
-        _configurationService = configuration;
-        _mainCdnFullUrl = configuration.GetValue<Uri>(nameof(ServerConfiguration.CdnFullUrl));
         _shardName = configuration.GetValue<string>(nameof(ServerConfiguration.ShardName));
         _maxExistingGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxExistingGroupsByUser), 3);
         _maxJoinedGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxJoinedGroupsByUser), 6);
         _maxGroupUserCount = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxGroupUserCount), 100);
+        _fileServerAddress = configuration.GetValue<Uri>(nameof(ServerConfiguration.CdnFullUrl));
+        _expectedClientVersion = configuration.GetValueOrDefault(nameof(ServerConfiguration.ExpectedClientVersion), new Version(0, 0, 0));
         _contextAccessor = contextAccessor;
         _redis = redisDb;
-        _generator = generator;
         _logger = new MareHubLogger(this, logger);
         _dbContext = mareDbContext;
     }
@@ -73,6 +68,7 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
 
         return new ConnectionDto(new UserData(dbUser.UID, string.IsNullOrWhiteSpace(dbUser.Alias) ? null : dbUser.Alias))
         {
+            CurrentClientVersion = _expectedClientVersion,
             ServerVersion = IMareHub.ApiVersion,
             IsAdmin = dbUser.IsAdmin,
             IsModerator = dbUser.IsModerator,
@@ -82,6 +78,7 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
                 ShardName = _shardName,
                 MaxGroupsJoinedByUser = _maxJoinedGroupsByUser,
                 MaxGroupUserCount = _maxGroupUserCount,
+                FileServerAddress = _fileServerAddress
             },
         };
     }

MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj

@@ -21,29 +21,18 @@
 
   <ItemGroup>
     <PackageReference Include="AspNetCoreRateLimit" Version="5.0.0" />
-    <PackageReference Include="Bazinga.AspNetCore.Authentication.Basic" Version="2.0.1" />
-    <PackageReference Include="EFCore.NamingConventions" Version="7.0.2" />
     <PackageReference Include="Grpc.AspNetCore" Version="2.51.0" />
     <PackageReference Include="Grpc.Net.Client" Version="2.51.0" />
-    <PackageReference Include="Karambolo.Extensions.Logging.File" Version="3.3.1" />
     <PackageReference Include="lz4net" Version="1.0.15.93" />
-    <PackageReference Include="Meziantou.Analyzer" Version="2.0.14">
+    <PackageReference Include="Meziantou.Analyzer" Version="2.0.19">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="7.0.2" />
-    <PackageReference Include="Microsoft.AspNetCore.Diagnostics.EntityFrameworkCore" Version="7.0.2" />
-    <PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="7.0.2" />
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.StackExchangeRedis" Version="7.0.2" />
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="7.0.2" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.StackExchangeRedis" Version="7.0.4" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="7.0.4" />
     <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="7.0.0" />
-    <PackageReference Include="Microsoft.IdentityModel.Tokens" Version="6.26.0" />
-    <PackageReference Include="prometheus-net.AspNetCore" Version="7.0.0" />
-    <PackageReference Include="StackExchange.Redis" Version="2.6.90" />
-    <PackageReference Include="StackExchange.Redis.Extensions.AspNetCore" Version="8.0.5" />
-    <PackageReference Include="StackExchange.Redis.Extensions.Core" Version="8.0.5" />
-    <PackageReference Include="StackExchange.Redis.Extensions.System.Text.Json" Version="8.0.5" />
-    <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.26.0" />
+    <PackageReference Include="Microsoft.IdentityModel.Tokens" Version="6.27.0" />
+    <PackageReference Include="prometheus-net.AspNetCore" Version="8.0.0" />
   </ItemGroup>
 
   <ItemGroup>

MareSynchronosServer/MareSynchronosServer/Startup.cs

@@ -9,7 +9,6 @@ using MareSynchronosShared.Protos;
 using Grpc.Net.Client.Configuration;
 using MareSynchronosShared.Metrics;
 using MareSynchronosServer.Services;
-using MareSynchronosServer.RequirementHandlers;
 using MareSynchronosShared.Utils;
 using MareSynchronosShared.Services;
 using Prometheus;
@@ -26,6 +25,7 @@ using MessagePack;
 using MessagePack.Resolvers;
 using Microsoft.AspNetCore.Mvc.Controllers;
 using MareSynchronosServer.Controllers;
+using MareSynchronosShared.RequirementHandlers;
 
 namespace MareSynchronosServer;
 
@@ -52,9 +52,6 @@ public class Startup
         // configure metrics
         ConfigureMetrics(services);
 
-        // configure file service grpc connection
-        ConfigureFileServiceGrpcClient(services);
-
         // configure database
         ConfigureDatabase(services, mareConfig);
 
@@ -294,30 +291,6 @@ public class Startup
         }
     }
 
-    private static void ConfigureFileServiceGrpcClient(IServiceCollection services)
-    {
-        var defaultMethodConfig = new MethodConfig
-        {
-            Names = { MethodName.Default },
-            RetryPolicy = new RetryPolicy
-            {
-                MaxAttempts = 1000,
-                InitialBackoff = TimeSpan.FromSeconds(1),
-                MaxBackoff = TimeSpan.FromSeconds(5),
-                BackoffMultiplier = 1.5,
-                RetryableStatusCodes = { Grpc.Core.StatusCode.Unavailable },
-            },
-        };
-        services.AddGrpcClient<FileService.FileServiceClient>((serviceProvider, c) =>
-        {
-            c.Address = serviceProvider.GetRequiredService<IConfigurationService<ServerConfiguration>>()
-                .GetValue<Uri>(nameof(ServerConfiguration.StaticFileServiceAddress));
-        }).ConfigureChannel(c =>
-        {
-            c.ServiceConfig = new ServiceConfig { MethodConfigs = { defaultMethodConfig } };
-        });
-    }
-
     public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILogger<Startup> logger)
     {
         logger.LogInformation("Running Configure");

MareSynchronosServer/MareSynchronosServices/Discord/MareModule.cs

@@ -461,7 +461,7 @@ public class MareModule : InteractionModuleBase
         };
 
         await db.Users.AddAsync(newUser).ConfigureAwait(false);
-        await db.Auth.AddAsync(auth).ConfigureAwait(false); ;
+        await db.Auth.AddAsync(auth).ConfigureAwait(false);
 
         await db.SaveChangesAsync().ConfigureAwait(false);
 

MareSynchronosServer/MareSynchronosServices/MareSynchronosServices.csproj

@@ -22,14 +22,15 @@
 
   <ItemGroup>
     <PackageReference Include="Discord.Net" Version="3.9.0" />
-    <PackageReference Include="EFCore.NamingConventions" Version="7.0.2" />
     <PackageReference Include="Grpc.AspNetCore" Version="2.51.0" />
-    <PackageReference Include="Karambolo.Extensions.Logging.File" Version="3.3.1" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="7.0.2" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.2" />
+    <PackageReference Include="Meziantou.Analyzer" Version="2.0.19">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="7.0.4" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.4" />
     <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="7.0.0" />
-    <PackageReference Include="prometheus-net.AspNetCore" Version="7.0.0" />
-    <PackageReference Include="StackExchange.Redis" Version="2.6.90" />
+    <PackageReference Include="prometheus-net.AspNetCore" Version="8.0.0" />
   </ItemGroup>
 
   <ItemGroup>

MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj

@@ -21,7 +21,8 @@
 		<PackageReference Include="ByteSize" Version="2.1.1" />
 		<PackageReference Include="EFCore.NamingConventions" Version="7.0.2" />
 		<PackageReference Include="Grpc.AspNetCore" Version="2.51.0" />
-		<PackageReference Include="Meziantou.Analyzer" Version="2.0.14">
+		<PackageReference Include="Karambolo.Extensions.Logging.File" Version="3.4.0" />
+		<PackageReference Include="Meziantou.Analyzer" Version="2.0.19">
 		  <PrivateAssets>all</PrivateAssets>
 		  <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
 		</PackageReference>
@@ -30,16 +31,21 @@
 	<ItemGroup>
 		<PackageReference Include="Microsoft.AspNetCore.Authentication" Version="2.2.0" />
 		<PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
-		<PackageReference Include="Microsoft.EntityFrameworkCore" Version="7.0.2" />
-		<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="7.0.2">
+		<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="7.0.4" />
+		<PackageReference Include="Microsoft.EntityFrameworkCore" Version="7.0.4" />
+		<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="7.0.4">
 			<PrivateAssets>all</PrivateAssets>
 			<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
 		</PackageReference>
-		<PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="7.0.2" />
-		<PackageReference Include="Microsoft.IdentityModel.Tokens" Version="6.26.0" />
-		<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="7.0.1" />
-		<PackageReference Include="prometheus-net" Version="7.0.0" />
-		<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.26.0" />
+		<PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="7.0.4" />
+		<PackageReference Include="Microsoft.IdentityModel.Tokens" Version="6.27.0" />
+		<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="7.0.3" />
+		<PackageReference Include="prometheus-net" Version="8.0.0" />
+		<PackageReference Include="StackExchange.Redis" Version="2.6.96" />
+		<PackageReference Include="StackExchange.Redis.Extensions.AspNetCore" Version="9.1.0" />
+		<PackageReference Include="StackExchange.Redis.Extensions.Core" Version="9.1.0" />
+		<PackageReference Include="StackExchange.Redis.Extensions.System.Text.Json" Version="9.1.0" />
+		<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.27.0" />
 		<PackageReference Include="System.Linq.Async" Version="6.0.1" />
 	</ItemGroup>
 

MareSynchronosServer/MareSynchronosShared/Migrations/20230228001033_UserPerms.Designer.cs

@@ -0,0 +1,552 @@
+// <auto-generated />
+using System;
+using MareSynchronosShared.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+#nullable disable
+
+namespace MareSynchronosServer.Migrations
+{
+    [DbContext(typeof(MareDbContext))]
+    [Migration("20230228001033_UserPerms")]
+    partial class UserPerms
+    {
+        /// <inheritdoc />
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("ProductVersion", "7.0.3")
+                .HasAnnotation("Relational:MaxIdentifierLength", 63);
+
+            NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
+
+            modelBuilder.Entity("MareSynchronosShared.Models.Auth", b =>
+                {
+                    b.Property<string>("HashedKey")
+                        .HasMaxLength(64)
+                        .HasColumnType("character varying(64)")
+                        .HasColumnName("hashed_key");
+
+                    b.Property<bool>("IsBanned")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_banned");
+
+                    b.Property<string>("PrimaryUserUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("primary_user_uid");
+
+                    b.Property<string>("UserUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("user_uid");
+
+                    b.HasKey("HashedKey")
+                        .HasName("pk_auth");
+
+                    b.HasIndex("PrimaryUserUID")
+                        .HasDatabaseName("ix_auth_primary_user_uid");
+
+                    b.HasIndex("UserUID")
+                        .HasDatabaseName("ix_auth_user_uid");
+
+                    b.ToTable("auth", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.Banned", b =>
+                {
+                    b.Property<string>("CharacterIdentification")
+                        .HasMaxLength(100)
+                        .HasColumnType("character varying(100)")
+                        .HasColumnName("character_identification");
+
+                    b.Property<string>("Reason")
+                        .HasColumnType("text")
+                        .HasColumnName("reason");
+
+                    b.Property<byte[]>("Timestamp")
+                        .IsConcurrencyToken()
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("bytea")
+                        .HasColumnName("timestamp");
+
+                    b.HasKey("CharacterIdentification")
+                        .HasName("pk_banned_users");
+
+                    b.ToTable("banned_users", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.BannedRegistrations", b =>
+                {
+                    b.Property<string>("DiscordIdOrLodestoneAuth")
+                        .HasMaxLength(100)
+                        .HasColumnType("character varying(100)")
+                        .HasColumnName("discord_id_or_lodestone_auth");
+
+                    b.HasKey("DiscordIdOrLodestoneAuth")
+                        .HasName("pk_banned_registrations");
+
+                    b.ToTable("banned_registrations", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b =>
+                {
+                    b.Property<string>("UserUID")
+                        .HasMaxLength(10)
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("user_uid");
+
+                    b.Property<string>("OtherUserUID")
+                        .HasMaxLength(10)
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("other_user_uid");
+
+                    b.Property<bool>("AllowReceivingMessages")
+                        .HasColumnType("boolean")
+                        .HasColumnName("allow_receiving_messages");
+
+                    b.Property<bool>("DisableAnimations")
+                        .HasColumnType("boolean")
+                        .HasColumnName("disable_animations");
+
+                    b.Property<bool>("DisableSounds")
+                        .HasColumnType("boolean")
+                        .HasColumnName("disable_sounds");
+
+                    b.Property<bool>("IsPaused")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_paused");
+
+                    b.Property<byte[]>("Timestamp")
+                        .IsConcurrencyToken()
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("bytea")
+                        .HasColumnName("timestamp");
+
+                    b.HasKey("UserUID", "OtherUserUID")
+                        .HasName("pk_client_pairs");
+
+                    b.HasIndex("OtherUserUID")
+                        .HasDatabaseName("ix_client_pairs_other_user_uid");
+
+                    b.HasIndex("UserUID")
+                        .HasDatabaseName("ix_client_pairs_user_uid");
+
+                    b.ToTable("client_pairs", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b =>
+                {
+                    b.Property<string>("Hash")
+                        .HasMaxLength(40)
+                        .HasColumnType("character varying(40)")
+                        .HasColumnName("hash");
+
+                    b.Property<long>("Size")
+                        .HasColumnType("bigint")
+                        .HasColumnName("size");
+
+                    b.Property<byte[]>("Timestamp")
+                        .IsConcurrencyToken()
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("bytea")
+                        .HasColumnName("timestamp");
+
+                    b.Property<DateTime>("UploadDate")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("upload_date");
+
+                    b.Property<bool>("Uploaded")
+                        .HasColumnType("boolean")
+                        .HasColumnName("uploaded");
+
+                    b.Property<string>("UploaderUID")
+                        .HasMaxLength(10)
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("uploader_uid");
+
+                    b.HasKey("Hash")
+                        .HasName("pk_file_caches");
+
+                    b.HasIndex("UploaderUID")
+                        .HasDatabaseName("ix_file_caches_uploader_uid");
+
+                    b.ToTable("file_caches", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.ForbiddenUploadEntry", b =>
+                {
+                    b.Property<string>("Hash")
+                        .HasMaxLength(40)
+                        .HasColumnType("character varying(40)")
+                        .HasColumnName("hash");
+
+                    b.Property<string>("ForbiddenBy")
+                        .HasMaxLength(100)
+                        .HasColumnType("character varying(100)")
+                        .HasColumnName("forbidden_by");
+
+                    b.Property<byte[]>("Timestamp")
+                        .IsConcurrencyToken()
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("bytea")
+                        .HasColumnName("timestamp");
+
+                    b.HasKey("Hash")
+                        .HasName("pk_forbidden_upload_entries");
+
+                    b.ToTable("forbidden_upload_entries", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.Group", b =>
+                {
+                    b.Property<string>("GID")
+                        .HasMaxLength(20)
+                        .HasColumnType("character varying(20)")
+                        .HasColumnName("gid");
+
+                    b.Property<string>("Alias")
+                        .HasMaxLength(50)
+                        .HasColumnType("character varying(50)")
+                        .HasColumnName("alias");
+
+                    b.Property<bool>("DisableAnimations")
+                        .HasColumnType("boolean")
+                        .HasColumnName("disable_animations");
+
+                    b.Property<bool>("DisableSounds")
+                        .HasColumnType("boolean")
+                        .HasColumnName("disable_sounds");
+
+                    b.Property<string>("HashedPassword")
+                        .HasColumnType("text")
+                        .HasColumnName("hashed_password");
+
+                    b.Property<bool>("InvitesEnabled")
+                        .HasColumnType("boolean")
+                        .HasColumnName("invites_enabled");
+
+                    b.Property<string>("OwnerUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("owner_uid");
+
+                    b.HasKey("GID")
+                        .HasName("pk_groups");
+
+                    b.HasIndex("OwnerUID")
+                        .HasDatabaseName("ix_groups_owner_uid");
+
+                    b.ToTable("groups", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b =>
+                {
+                    b.Property<string>("GroupGID")
+                        .HasColumnType("character varying(20)")
+                        .HasColumnName("group_gid");
+
+                    b.Property<string>("BannedUserUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("banned_user_uid");
+
+                    b.Property<string>("BannedByUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("banned_by_uid");
+
+                    b.Property<DateTime>("BannedOn")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("banned_on");
+
+                    b.Property<string>("BannedReason")
+                        .HasColumnType("text")
+                        .HasColumnName("banned_reason");
+
+                    b.HasKey("GroupGID", "BannedUserUID")
+                        .HasName("pk_group_bans");
+
+                    b.HasIndex("BannedByUID")
+                        .HasDatabaseName("ix_group_bans_banned_by_uid");
+
+                    b.HasIndex("BannedUserUID")
+                        .HasDatabaseName("ix_group_bans_banned_user_uid");
+
+                    b.HasIndex("GroupGID")
+                        .HasDatabaseName("ix_group_bans_group_gid");
+
+                    b.ToTable("group_bans", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b =>
+                {
+                    b.Property<string>("GroupGID")
+                        .HasColumnType("character varying(20)")
+                        .HasColumnName("group_gid");
+
+                    b.Property<string>("GroupUserUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("group_user_uid");
+
+                    b.Property<bool>("DisableAnimations")
+                        .HasColumnType("boolean")
+                        .HasColumnName("disable_animations");
+
+                    b.Property<bool>("DisableSounds")
+                        .HasColumnType("boolean")
+                        .HasColumnName("disable_sounds");
+
+                    b.Property<bool>("IsModerator")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_moderator");
+
+                    b.Property<bool>("IsPaused")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_paused");
+
+                    b.Property<bool>("IsPinned")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_pinned");
+
+                    b.HasKey("GroupGID", "GroupUserUID")
+                        .HasName("pk_group_pairs");
+
+                    b.HasIndex("GroupGID")
+                        .HasDatabaseName("ix_group_pairs_group_gid");
+
+                    b.HasIndex("GroupUserUID")
+                        .HasDatabaseName("ix_group_pairs_group_user_uid");
+
+                    b.ToTable("group_pairs", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b =>
+                {
+                    b.Property<string>("GroupGID")
+                        .HasColumnType("character varying(20)")
+                        .HasColumnName("group_gid");
+
+                    b.Property<string>("Invite")
+                        .HasMaxLength(64)
+                        .HasColumnType("character varying(64)")
+                        .HasColumnName("invite");
+
+                    b.Property<DateTime>("ExpirationDate")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("expiration_date");
+
+                    b.HasKey("GroupGID", "Invite")
+                        .HasName("pk_group_temp_invites");
+
+                    b.HasIndex("GroupGID")
+                        .HasDatabaseName("ix_group_temp_invites_group_gid");
+
+                    b.HasIndex("Invite")
+                        .HasDatabaseName("ix_group_temp_invites_invite");
+
+                    b.ToTable("group_temp_invites", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b =>
+                {
+                    b.Property<decimal>("DiscordId")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("numeric(20,0)")
+                        .HasColumnName("discord_id");
+
+                    b.Property<string>("HashedLodestoneId")
+                        .HasMaxLength(100)
+                        .HasColumnType("character varying(100)")
+                        .HasColumnName("hashed_lodestone_id");
+
+                    b.Property<string>("LodestoneAuthString")
+                        .HasMaxLength(100)
+                        .HasColumnType("character varying(100)")
+                        .HasColumnName("lodestone_auth_string");
+
+                    b.Property<DateTime?>("StartedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("started_at");
+
+                    b.Property<string>("UserUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("user_uid");
+
+                    b.HasKey("DiscordId")
+                        .HasName("pk_lodestone_auth");
+
+                    b.HasIndex("UserUID")
+                        .HasDatabaseName("ix_lodestone_auth_user_uid");
+
+                    b.ToTable("lodestone_auth", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.User", b =>
+                {
+                    b.Property<string>("UID")
+                        .HasMaxLength(10)
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("uid");
+
+                    b.Property<string>("Alias")
+                        .HasMaxLength(15)
+                        .HasColumnType("character varying(15)")
+                        .HasColumnName("alias");
+
+                    b.Property<bool>("IsAdmin")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_admin");
+
+                    b.Property<bool>("IsModerator")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_moderator");
+
+                    b.Property<DateTime>("LastLoggedIn")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("last_logged_in");
+
+                    b.Property<byte[]>("Timestamp")
+                        .IsConcurrencyToken()
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("bytea")
+                        .HasColumnName("timestamp");
+
+                    b.HasKey("UID")
+                        .HasName("pk_users");
+
+                    b.ToTable("users", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.Auth", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "PrimaryUser")
+                        .WithMany()
+                        .HasForeignKey("PrimaryUserUID")
+                        .HasConstraintName("fk_auth_users_primary_user_temp_id");
+
+                    b.HasOne("MareSynchronosShared.Models.User", "User")
+                        .WithMany()
+                        .HasForeignKey("UserUID")
+                        .HasConstraintName("fk_auth_users_user_temp_id1");
+
+                    b.Navigation("PrimaryUser");
+
+                    b.Navigation("User");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "OtherUser")
+                        .WithMany()
+                        .HasForeignKey("OtherUserUID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_client_pairs_users_other_user_temp_id2");
+
+                    b.HasOne("MareSynchronosShared.Models.User", "User")
+                        .WithMany()
+                        .HasForeignKey("UserUID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_client_pairs_users_user_temp_id3");
+
+                    b.Navigation("OtherUser");
+
+                    b.Navigation("User");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "Uploader")
+                        .WithMany()
+                        .HasForeignKey("UploaderUID")
+                        .HasConstraintName("fk_file_caches_users_uploader_uid");
+
+                    b.Navigation("Uploader");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.Group", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "Owner")
+                        .WithMany()
+                        .HasForeignKey("OwnerUID")
+                        .HasConstraintName("fk_groups_users_owner_temp_id8");
+
+                    b.Navigation("Owner");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "BannedBy")
+                        .WithMany()
+                        .HasForeignKey("BannedByUID")
+                        .HasConstraintName("fk_group_bans_users_banned_by_temp_id5");
+
+                    b.HasOne("MareSynchronosShared.Models.User", "BannedUser")
+                        .WithMany()
+                        .HasForeignKey("BannedUserUID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_group_bans_users_banned_user_temp_id6");
+
+                    b.HasOne("MareSynchronosShared.Models.Group", "Group")
+                        .WithMany()
+                        .HasForeignKey("GroupGID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_group_bans_groups_group_temp_id");
+
+                    b.Navigation("BannedBy");
+
+                    b.Navigation("BannedUser");
+
+                    b.Navigation("Group");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.Group", "Group")
+                        .WithMany()
+                        .HasForeignKey("GroupGID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_group_pairs_groups_group_temp_id1");
+
+                    b.HasOne("MareSynchronosShared.Models.User", "GroupUser")
+                        .WithMany()
+                        .HasForeignKey("GroupUserUID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_group_pairs_users_group_user_temp_id7");
+
+                    b.Navigation("Group");
+
+                    b.Navigation("GroupUser");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.Group", "Group")
+                        .WithMany()
+                        .HasForeignKey("GroupGID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_group_temp_invites_groups_group_gid");
+
+                    b.Navigation("Group");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "User")
+                        .WithMany()
+                        .HasForeignKey("UserUID")
+                        .HasConstraintName("fk_lodestone_auth_users_user_uid");
+
+                    b.Navigation("User");
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}

MareSynchronosServer/MareSynchronosShared/Migrations/20230228001033_UserPerms.cs

@@ -0,0 +1,40 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace MareSynchronosServer.Migrations
+{
+    /// <inheritdoc />
+    public partial class UserPerms : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<bool>(
+                name: "disable_animations",
+                table: "client_pairs",
+                type: "boolean",
+                nullable: false,
+                defaultValue: false);
+
+            migrationBuilder.AddColumn<bool>(
+                name: "disable_sounds",
+                table: "client_pairs",
+                type: "boolean",
+                nullable: false,
+                defaultValue: false);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                name: "disable_animations",
+                table: "client_pairs");
+
+            migrationBuilder.DropColumn(
+                name: "disable_sounds",
+                table: "client_pairs");
+        }
+    }
+}

MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs

@@ -17,7 +17,7 @@ namespace MareSynchronosServer.Migrations
         {
 #pragma warning disable 612, 618
             modelBuilder
-                .HasAnnotation("ProductVersion", "7.0.0")
+                .HasAnnotation("ProductVersion", "7.0.3")
                 .HasAnnotation("Relational:MaxIdentifierLength", 63);
 
             NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
@@ -105,6 +105,14 @@ namespace MareSynchronosServer.Migrations
                         .HasColumnType("boolean")
                         .HasColumnName("allow_receiving_messages");
 
+                    b.Property<bool>("DisableAnimations")
+                        .HasColumnType("boolean")
+                        .HasColumnName("disable_animations");
+
+                    b.Property<bool>("DisableSounds")
+                        .HasColumnType("boolean")
+                        .HasColumnName("disable_sounds");
+
                     b.Property<bool>("IsPaused")
                         .HasColumnType("boolean")
                         .HasColumnName("is_paused");

MareSynchronosServer/MareSynchronosShared/Models/ClientPair.cs

@@ -14,4 +14,6 @@ public class ClientPair
     public bool AllowReceivingMessages { get; set; } = false;
     [Timestamp]
     public byte[] Timestamp { get; set; }
+    public bool DisableSounds { get; set; } = false;
+    public bool DisableAnimations { get; set; } = false;
 }

MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto

@@ -4,11 +4,6 @@ option csharp_namespace = "MareSynchronosShared.Protos";
 
 package mareservices;
 
-service FileService {
-	rpc UploadFile (stream UploadFileRequest) returns (Empty);
-	rpc DeleteFiles (DeleteFilesRequest) returns (Empty);
-}
-
 service ClientMessageService {
 	rpc SendClientMessage (ClientMessage) returns (Empty);
 }
@@ -26,78 +21,3 @@ enum MessageType {
 }
 
 message Empty { }
-
-message MultiUidMessage {
-	repeated UidMessage uids = 1;
-}
-
-message ServerIdentMessage {
-	repeated SetIdentMessage idents = 1;
-}
-
-message IdentChangeMessage {
-	oneof payload {
-		ServerMessage server = 1;
-		IdentChange identChange = 2;
-	}
-}
-
-message IdentChange {
-	UidWithIdent uidWithIdent = 1;
-	bool isOnline = 2;
-}
-
-message UidWithIdentMessage {
-	repeated UidWithIdent uidWithIdent = 1;
-}
-
-message UidWithIdent {
-	UidMessage uid = 1;
-	CharacterIdentMessage ident = 2;
-}
-
-message UidMessage {
-	string uid = 1;
-}
-
-message ServerMessage {
-	string server_id = 1;
-}
-
-message OnlineUserCountResponse {
-	int64 count = 1;
-}
-
-message RemoveIdentMessage {
-	string uid = 1;
-	string server_id = 2;
-}
-
-message SetIdentMessage {
-	UidWithIdent uidWithIdent = 1;
-}
-
-message CharacterIdentMessage {
-	string server_id = 1;
-	string ident = 2;
-}
-
-message UploadFileRequest {
-	string hash = 1;
-	string uploader = 2;
-	bytes fileData = 3;
-}
-
-message DeleteFilesRequest {
-	repeated string hash = 1;
-}
-
-message AuthRequest {
-	string ip = 1;
-	string secretKey = 2;
-}
-
-message AuthReply {
-  bool success = 1;
-  UidMessage uid = 2;
-}

MareSynchronosServer/MareSynchronosShared/RequirementHandlers/UserRequirement.cs

@@ -1,6 +1,6 @@
 using Microsoft.AspNetCore.Authorization;
 
-namespace MareSynchronosServer.RequirementHandlers;
+namespace MareSynchronosShared.RequirementHandlers;
 
 public class UserRequirement : IAuthorizationRequirement
 {

MareSynchronosServer/MareSynchronosShared/RequirementHandlers/UserRequirementHandler.cs

@@ -5,8 +5,9 @@ using Microsoft.EntityFrameworkCore;
 using MareSynchronosShared.Utils;
 using StackExchange.Redis;
 using StackExchange.Redis.Extensions.Core.Abstractions;
+using Microsoft.Extensions.Logging;
 
-namespace MareSynchronosServer.RequirementHandlers;
+namespace MareSynchronosShared.RequirementHandlers;
 
 public class UserRequirementHandler : AuthorizationHandler<UserRequirement, HubInvocationContext>
 {

MareSynchronosServer/MareSynchronosShared/RequirementHandlers/UserRequirements.cs

@@ -1,4 +1,4 @@
-namespace MareSynchronosServer.RequirementHandlers;
+namespace MareSynchronosShared.RequirementHandlers;
 
 public enum UserRequirements
 {

MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs

@@ -6,12 +6,11 @@ public class ServerConfiguration : MareConfigurationAuthBase
 {
     public string RedisConnectionString { get; set; } = string.Empty;
     public int RedisPool { get; set; } = 50;
-
+    [RemoteConfiguration]
+    public Version ExpectedClientVersion { get; set; } = new Version(0, 0, 0);
     [RemoteConfiguration]
     public Uri CdnFullUrl { get; set; } = null;
     [RemoteConfiguration]
-    public Uri StaticFileServiceAddress { get; set; } = null;
-    [RemoteConfiguration]
     public int MaxExistingGroupsByUser { get; set; } = 3;
     [RemoteConfiguration]
     public int MaxJoinedGroupsByUser { get; set; } = 6;
@@ -21,18 +20,15 @@ public class ServerConfiguration : MareConfigurationAuthBase
     public bool PurgeUnusedAccounts { get; set; } = false;
     [RemoteConfiguration]
     public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14;
-    [RemoteConfiguration]
-    public List<CdnShardConfiguration> CdnShardConfiguration { get; set; } = new();
 
     public override string ToString()
     {
         StringBuilder sb = new();
         sb.AppendLine(base.ToString());
         sb.AppendLine($"{nameof(CdnFullUrl)} => {CdnFullUrl}");
-        sb.AppendLine($"{nameof(CdnShardConfiguration)} => {string.Join(", ", CdnShardConfiguration.Select(c => c.ToString()))}");
-        sb.AppendLine($"{nameof(StaticFileServiceAddress)} => {StaticFileServiceAddress}");
         sb.AppendLine($"{nameof(RedisConnectionString)} => {RedisConnectionString}");
         sb.AppendLine($"{nameof(RedisPool)} => {RedisPool}");
+        sb.AppendLine($"{nameof(ExpectedClientVersion)} => {ExpectedClientVersion}");
         sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");
         sb.AppendLine($"{nameof(MaxJoinedGroupsByUser)} => {MaxJoinedGroupsByUser}");
         sb.AppendLine($"{nameof(MaxGroupUserCount)} => {MaxGroupUserCount}");

MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs

@@ -14,6 +14,10 @@ public class StaticFilesServerConfiguration : MareConfigurationBase
     public int DownloadTimeoutSeconds { get; set; } = 5;
     public int DownloadQueueReleaseSeconds { get; set; } = 15;
     public int DownloadQueueClearLimit { get; set; } = 15000;
+    [RemoteConfiguration]
+    public Uri CdnFullUrl { get; set; } = null;
+    [RemoteConfiguration]
+    public List<CdnShardConfiguration> CdnShardConfiguration { get; set; } = new();
     public override string ToString()
     {
         StringBuilder sb = new();

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/RequestController.cs

@@ -60,18 +60,19 @@ public class RequestController : ControllerBase
     [Route(MareFiles.Request_RequestFile)]
     public async Task<IActionResult> RequestFile(string file)
     {
+        Guid g = Guid.NewGuid();
+
         try
         {
             await _parallelRequestSemaphore.WaitAsync(HttpContext.RequestAborted);
-            Guid g = Guid.NewGuid();
             _cachedFileProvider.DownloadFileWhenRequired(file);
-            await _requestQueue.EnqueueUser(new(g, MareUser, file));
             return Ok(g);
         }
         catch (OperationCanceledException) { return BadRequest(); }
         finally
         {
             _parallelRequestSemaphore.Release();
+            await _requestQueue.EnqueueUser(new(g, MareUser, file));
         }
     }
 

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs

@@ -1,18 +1,146 @@
-using MareSynchronos.API.Routes;
+using LZ4;
+using MareSynchronos.API.Dto.Files;
+using MareSynchronos.API.Routes;
+using MareSynchronos.API.SignalR;
+using MareSynchronosServer.Hubs;
+using MareSynchronosShared.Data;
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Models;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils;
 using MareSynchronosStaticFilesServer.Services;
+using MareSynchronosStaticFilesServer.Utils;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.SignalR;
+using Microsoft.EntityFrameworkCore;
+using System.Collections.Concurrent;
+using System.Security.Cryptography;
+using System.Text.Json;
+using System.Text.RegularExpressions;
 
 namespace MareSynchronosStaticFilesServer.Controllers;
 
 [Route(MareFiles.ServerFiles)]
 public class ServerFilesController : ControllerBase
 {
+    private static readonly ConcurrentDictionary<string, SemaphoreSlim> _fileUploadLocks = new(StringComparer.Ordinal);
+    private readonly string _basePath;
     private readonly CachedFileProvider _cachedFileProvider;
+    private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
+    private readonly IHubContext<MareHub> _hubContext;
+    private readonly MareDbContext _mareDbContext;
+    private readonly MareMetrics _metricsClient;
 
-    public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider) : base(logger)
+    public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider,
+        IConfigurationService<StaticFilesServerConfiguration> configuration,
+        IHubContext<MareSynchronosServer.Hubs.MareHub> hubContext,
+        MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger)
     {
+        _basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
         _cachedFileProvider = cachedFileProvider;
+        _configuration = configuration;
+        _hubContext = hubContext;
+        _mareDbContext = mareDbContext;
+        _metricsClient = metricsClient;
+    }
+
+    [HttpPost(MareFiles.ServerFiles_DeleteAll)]
+    public async Task<IActionResult> FilesDeleteAll()
+    {
+        var ownFiles = await _mareDbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == MareUser).ToListAsync().ConfigureAwait(false);
+
+        foreach (var dbFile in ownFiles)
+        {
+            var fi = FilePathUtil.GetFileInfoForHash(_basePath, dbFile.Hash);
+            if (fi != null)
+            {
+                _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1);
+                _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0);
+
+                fi?.Delete();
+            }
+        }
+
+        _mareDbContext.Files.RemoveRange(ownFiles);
+        await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
+
+        return Ok();
+    }
+
+    [HttpGet(MareFiles.ServerFiles_GetSizes)]
+    public async Task<IActionResult> FilesGetSizes([FromBody] List<string> hashes)
+    {
+        var allFiles = await _mareDbContext.Files.Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false);
+        var forbiddenFiles = await _mareDbContext.ForbiddenUploadEntries.
+            Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false);
+        List<DownloadFileDto> response = new();
+
+        var cacheFile = await _mareDbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false);
+
+        var shardConfig = new List<CdnShardConfiguration>(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>()));
+
+        foreach (var file in cacheFile)
+        {
+            var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
+
+            var matchedShardConfig = shardConfig.OrderBy(g => Guid.NewGuid()).FirstOrDefault(f => new Regex(f.FileMatch).IsMatch(file.Hash));
+            var baseUrl = matchedShardConfig?.CdnFullUrl ?? _configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.CdnFullUrl));
+
+            response.Add(new DownloadFileDto
+            {
+                FileExists = file.Size > 0,
+                ForbiddenBy = forbiddenFile?.ForbiddenBy ?? string.Empty,
+                IsForbidden = forbiddenFile != null,
+                Hash = file.Hash,
+                Size = file.Size,
+                Url = baseUrl.ToString(),
+            });
+        }
+
+        return Ok(JsonSerializer.Serialize(response));
+    }
+
+    [HttpPost(MareFiles.ServerFiles_FilesSend)]
+    public async Task<IActionResult> FilesSend([FromBody] FilesSendDto filesSendDto)
+    {
+        var userSentHashes = new HashSet<string>(filesSendDto.FileHashes.Distinct(StringComparer.Ordinal).Select(s => string.Concat(s.Where(c => char.IsLetterOrDigit(c)))), StringComparer.Ordinal);
+        var notCoveredFiles = new Dictionary<string, UploadFileDto>(StringComparer.Ordinal);
+        var forbiddenFiles = await _mareDbContext.ForbiddenUploadEntries.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
+        var existingFiles = await _mareDbContext.Files.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
+
+        List<FileCache> fileCachesToUpload = new();
+        foreach (var hash in userSentHashes)
+        {
+            // Skip empty file hashes, duplicate file hashes, forbidden file hashes and existing file hashes
+            if (string.IsNullOrEmpty(hash)) { continue; }
+            if (notCoveredFiles.ContainsKey(hash)) { continue; }
+            if (forbiddenFiles.ContainsKey(hash))
+            {
+                notCoveredFiles[hash] = new UploadFileDto()
+                {
+                    ForbiddenBy = forbiddenFiles[hash].ForbiddenBy,
+                    Hash = hash,
+                    IsForbidden = true,
+                };
+
+                continue;
+            }
+            if (existingFiles.TryGetValue(hash, out var file) && file.Uploaded) { continue; }
+
+            notCoveredFiles[hash] = new UploadFileDto()
+            {
+                Hash = hash,
+            };
+        }
+
+        if (notCoveredFiles.Any())
+        {
+            await _hubContext.Clients.Users(filesSendDto.UIDs).SendAsync(nameof(IMareHub.Client_UserReceiveUploadStatus), new MareSynchronos.API.Dto.User.UserDto(new(MareUser)))
+                .ConfigureAwait(false);
+        }
+
+        return Ok(JsonSerializer.Serialize(notCoveredFiles.Values.ToList()));
     }
 
     [HttpGet(MareFiles.ServerFiles_Get + "/{fileId}")]
@@ -26,4 +154,93 @@ public class ServerFilesController : ControllerBase
 
         return File(fs, "application/octet-stream");
     }
+
+    [HttpPost(MareFiles.ServerFiles_Upload + "/{hash}")]
+    [RequestSizeLimit(200 * 1024 * 1024)]
+    public async Task<IActionResult> UploadFile(string hash, CancellationToken requestAborted)
+    {
+        _logger.LogInformation("{user} uploading file {file}", MareUser, hash);
+        bool initiated = false;
+        hash = hash.ToUpperInvariant();
+        var existingFile = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
+        if (existingFile != null) return Ok();
+
+        if (!_fileUploadLocks.TryGetValue(hash, out var fileLock))
+        {
+            initiated = true;
+            _fileUploadLocks[hash] = fileLock = new SemaphoreSlim(1);
+            await fileLock.WaitAsync(requestAborted).ConfigureAwait(false);
+        }
+
+        if (!initiated)
+        {
+            try
+            {
+                await fileLock.WaitAsync(requestAborted).ConfigureAwait(false);
+                var file = await _mareDbContext.Files.SingleOrDefaultAsync(c => c.Hash == hash).ConfigureAwait(false);
+                if (file != null)
+                {
+                    fileLock.Release();
+                    return Ok();
+                }
+            }
+            catch (OperationCanceledException)
+            {
+                fileLock.Release();
+                return Ok();
+            }
+        }
+
+        try
+        {
+            // copy the request body to memory
+            using var compressedFileStream = new MemoryStream();
+            await Request.Body.CopyToAsync(compressedFileStream, requestAborted).ConfigureAwait(false);
+
+            // decompress and copy the decompressed stream to memory
+            var data = LZ4Codec.Unwrap(compressedFileStream.ToArray());
+
+            // reset streams
+            compressedFileStream.Seek(0, SeekOrigin.Begin);
+
+            // compute hash to verify
+            var hashString = BitConverter.ToString(SHA1.HashData(data))
+                .Replace("-", "", StringComparison.Ordinal).ToUpperInvariant();
+            if (!string.Equals(hashString, hash, StringComparison.Ordinal))
+                throw new InvalidOperationException($"Hash does not match file, computed: {hashString}, expected: {hash}");
+
+            // save file
+            var path = FilePathUtil.GetFilePath(_basePath, hash);
+            using var fileStream = new FileStream(path, FileMode.Create);
+            await compressedFileStream.CopyToAsync(fileStream).ConfigureAwait(false);
+
+            // update on db
+            await _mareDbContext.Files.AddAsync(new FileCache()
+            {
+                Hash = hash,
+                UploadDate = DateTime.UtcNow,
+                UploaderUID = MareUser,
+                Size = compressedFileStream.Length,
+                Uploaded = true
+            }).ConfigureAwait(false);
+            await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
+
+            _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotal, 1);
+            _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotalSize, compressedFileStream.Length);
+
+            _fileUploadLocks.Remove(hash, out _);
+        }
+        catch (Exception e)
+        {
+            _logger.LogError(e, "Error during file upload");
+            return BadRequest();
+        }
+        finally
+        {
+            fileLock.Release();
+            _fileUploadLocks.TryRemove(hash, out _);
+        }
+
+        return Ok();
+    }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj

@@ -18,17 +18,15 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Grpc.AspNetCore" Version="2.51.0" />
-    <PackageReference Include="Grpc.Net.Client" Version="2.51.0" />
-    <PackageReference Include="Meziantou.Analyzer" Version="2.0.14">
+    <PackageReference Include="lz4net" Version="1.0.15.93" />
+    <PackageReference Include="Meziantou.Analyzer" Version="2.0.19">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="7.0.2" />
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="7.0.2" />
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.StackExchangeRedis" Version="7.0.2" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="7.0.4" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.StackExchangeRedis" Version="7.0.4" />
     <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="7.0.0" />
-    <PackageReference Include="prometheus-net.AspNetCore" Version="7.0.0" />
+    <PackageReference Include="prometheus-net.AspNetCore" Version="8.0.0" />
   </ItemGroup>
 
   <ItemGroup>

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/GrpcFileService.cs

@@ -1,105 +0,0 @@
-using Grpc.Core;
-using MareSynchronosShared.Data;
-using MareSynchronosShared.Metrics;
-using MareSynchronosShared.Protos;
-using MareSynchronosShared.Services;
-using MareSynchronosStaticFilesServer.Utils;
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.EntityFrameworkCore;
-
-namespace MareSynchronosStaticFilesServer.Services;
-
-[Authorize(Policy = "Internal")]
-public class GrpcFileService : FileService.FileServiceBase
-{
-    private readonly string _basePath;
-    private readonly MareDbContext _mareDbContext;
-    private readonly ILogger<GrpcFileService> _logger;
-    private readonly MareMetrics _metricsClient;
-
-    public GrpcFileService(MareDbContext mareDbContext, IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<GrpcFileService> logger, MareMetrics metricsClient)
-    {
-        _basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
-        _mareDbContext = mareDbContext;
-        _logger = logger;
-        _metricsClient = metricsClient;
-    }
-
-    [Authorize(Policy = "Internal")]
-    public override async Task<Empty> UploadFile(IAsyncStreamReader<UploadFileRequest> requestStream, ServerCallContext context)
-    {
-        _ = await requestStream.MoveNext().ConfigureAwait(false);
-        var uploadMsg = requestStream.Current;
-        var filePath = FilePathUtil.GetFilePath(_basePath, uploadMsg.Hash);
-        using var fileWriter = File.OpenWrite(filePath);
-        var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == uploadMsg.Hash && f.UploaderUID == uploadMsg.Uploader).ConfigureAwait(false);
-        try
-        {
-            if (file != null)
-            {
-                await fileWriter.WriteAsync(uploadMsg.FileData.ToArray()).ConfigureAwait(false);
-
-                while (await requestStream.MoveNext().ConfigureAwait(false))
-                {
-                    await fileWriter.WriteAsync(requestStream.Current.FileData.ToArray()).ConfigureAwait(false);
-                }
-
-                await fileWriter.FlushAsync().ConfigureAwait(false);
-                fileWriter.Close();
-
-                var fileSize = new FileInfo(filePath).Length;
-                file.Uploaded = true;
-                file.Size = fileSize;
-
-                await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
-
-                _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotal, 1);
-                _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotalSize, fileSize);
-
-                _logger.LogInformation("User {user} uploaded file {hash}", uploadMsg.Uploader, uploadMsg.Hash);
-            }
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning(ex, "Error during UploadFile");
-            var fileNew = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == uploadMsg.Hash && f.UploaderUID == uploadMsg.Uploader).ConfigureAwait(false);
-            if (fileNew != null)
-            {
-                _mareDbContext.Files.Remove(fileNew);
-            }
-
-            await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
-        }
-
-        return new Empty();
-    }
-
-    [Authorize(Policy = "Internal")]
-    public override async Task<Empty> DeleteFiles(DeleteFilesRequest request, ServerCallContext context)
-    {
-        foreach (var hash in request.Hash)
-        {
-            try
-            {
-                var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
-                var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash).ConfigureAwait(false);
-                if (file != null && fi != null)
-                {
-                    _mareDbContext.Files.Remove(file);
-                    await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
-
-                    _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1);
-                    _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0);
-
-                    fi?.Delete();
-                }
-            }
-            catch (Exception ex)
-            {
-                _logger.LogError(ex, "Could not delete file for hash {hash}", hash);
-            }
-        }
-
-        return new Empty();
-    }
-}

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/RequestQueueService.cs

@@ -10,17 +10,17 @@ namespace MareSynchronosStaticFilesServer.Services;
 
 public class RequestQueueService : IHostedService
 {
-    private readonly UserQueueEntry[] _userQueueRequests;
-    private readonly ConcurrentQueue<UserRequest> _queue = new();
-    private readonly MareMetrics _metrics;
-    private readonly ILogger<RequestQueueService> _logger;
     private readonly IHubContext<MareSynchronosServer.Hubs.MareHub> _hubContext;
+    private readonly ILogger<RequestQueueService> _logger;
+    private readonly MareMetrics _metrics;
+    private readonly ConcurrentQueue<UserRequest> _queue = new();
     private readonly int _queueExpirationSeconds;
-    private readonly SemaphoreSlim _queueSemaphore = new(1);
     private readonly SemaphoreSlim _queueProcessingSemaphore = new(1);
+    private readonly ConcurrentDictionary<Guid, string> _queueRemoval = new();
+    private readonly SemaphoreSlim _queueSemaphore = new(1);
+    private readonly UserQueueEntry[] _userQueueRequests;
     private int _queueLimitForReset;
     private System.Timers.Timer _queueTimer;
-    private readonly ConcurrentDictionary<Guid, string> _queueRemoval = new();
 
     public RequestQueueService(MareMetrics metrics, IConfigurationService<StaticFilesServerConfiguration> configurationService, ILogger<RequestQueueService> logger, IHubContext<MareSynchronosServer.Hubs.MareHub> hubContext)
     {
@@ -32,6 +32,12 @@ public class RequestQueueService : IHostedService
         _hubContext = hubContext;
     }
 
+    public void ActivateRequest(Guid request)
+    {
+        _logger.LogDebug("Activating request {guid}", request);
+        _userQueueRequests.First(f => f != null && f.UserRequest.RequestId == request).IsActive = true;
+    }
+
     public async Task EnqueueUser(UserRequest request)
     {
         _logger.LogDebug("Enqueueing req {guid} from {user} for {file}", request.RequestId, request.User, request.FileId);
@@ -39,20 +45,13 @@ public class RequestQueueService : IHostedService
         if (_queueProcessingSemaphore.CurrentCount == 0)
         {
             _queue.Enqueue(request);
+            return;
         }
 
         try
         {
             await _queueSemaphore.WaitAsync().ConfigureAwait(false);
-            var idx = Array.FindIndex(_userQueueRequests, r => r == null);
-            if (idx == -1)
-            {
             _queue.Enqueue(request);
-            }
-            else
-            {
-                await DequeueIntoSlotAsync(request, idx).ConfigureAwait(false);
-            }
 
             return;
         }
@@ -68,6 +67,21 @@ public class RequestQueueService : IHostedService
         throw new Exception("Error during EnqueueUser");
     }
 
+    public void FinishRequest(Guid request)
+    {
+        var req = _userQueueRequests.First(f => f != null && f.UserRequest.RequestId == request);
+        var idx = Array.IndexOf(_userQueueRequests, req);
+        _logger.LogDebug("Finishing Request {guid}, clearing slot {idx}", request, idx);
+        _userQueueRequests[idx] = null;
+    }
+
+    public bool IsActiveProcessing(Guid request, string user, out UserRequest userRequest)
+    {
+        var userQueueRequest = _userQueueRequests.FirstOrDefault(u => u != null && u.UserRequest.RequestId == request && string.Equals(u.UserRequest.User, user, StringComparison.Ordinal));
+        userRequest = userQueueRequest?.UserRequest ?? null;
+        return userQueueRequest != null && userRequest != null && userQueueRequest.ExpirationDate > DateTime.UtcNow;
+    }
+
     public void RemoveFromQueue(Guid requestId, string user)
     {
         if (!_queue.Any(f => f.RequestId == requestId && string.Equals(f.User, user, StringComparison.Ordinal)))
@@ -87,30 +101,31 @@ public class RequestQueueService : IHostedService
         _queueRemoval[requestId] = user;
     }
 
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        _queueTimer = new System.Timers.Timer(250);
+        _queueTimer.Elapsed += ProcessQueue;
+        _queueTimer.AutoReset = true;
+        _queueTimer.Start();
+        return Task.CompletedTask;
+    }
+
     public bool StillEnqueued(Guid request, string user)
     {
         return _queue.Any(c => c.RequestId == request && string.Equals(c.User, user, StringComparison.Ordinal));
     }
 
-    public bool IsActiveProcessing(Guid request, string user, out UserRequest userRequest)
+    public Task StopAsync(CancellationToken cancellationToken)
     {
-        var userQueueRequest = _userQueueRequests.FirstOrDefault(u => u != null && u.UserRequest.RequestId == request && string.Equals(u.UserRequest.User, user, StringComparison.Ordinal));
-        userRequest = userQueueRequest?.UserRequest ?? null;
-        return userQueueRequest != null && userRequest != null && userQueueRequest.ExpirationDate > DateTime.UtcNow;
+        _queueTimer.Stop();
+        return Task.CompletedTask;
     }
 
-    public void FinishRequest(Guid request)
+    private async Task DequeueIntoSlotAsync(UserRequest userRequest, int slot)
     {
-        var req = _userQueueRequests.First(f => f != null && f.UserRequest.RequestId == request);
-        var idx = Array.IndexOf(_userQueueRequests, req);
-        _logger.LogDebug("Finishing Request {guid}, clearing slot {idx}", request, idx);
-        _userQueueRequests[idx] = null;
-    }
-
-    public void ActivateRequest(Guid request)
-    {
-        _logger.LogDebug("Activating request {guid}", request);
-        _userQueueRequests.First(f => f != null && f.UserRequest.RequestId == request).IsActive = true;
+        _logger.LogDebug("Dequeueing {req} into {i}: {user} with {file}", userRequest.RequestId, slot, userRequest.User, userRequest.FileId);
+        _userQueueRequests[slot] = new(userRequest, DateTime.UtcNow.AddSeconds(_queueExpirationSeconds));
+        await _hubContext.Clients.User(userRequest.User).SendAsync(nameof(IMareHub.Client_DownloadReady), userRequest.RequestId).ConfigureAwait(false);
     }
 
     private async void ProcessQueue(object src, ElapsedEventArgs e)
@@ -161,7 +176,6 @@ public class RequestQueueService : IHostedService
                     }
                 }
             });
-
         }
         catch (Exception ex)
         {
@@ -174,26 +188,4 @@ public class RequestQueueService : IHostedService
 
         _metrics.SetGaugeTo(MetricsAPI.GaugeDownloadQueue, _queue.Count);
     }
-
-    private async Task DequeueIntoSlotAsync(UserRequest userRequest, int slot)
-    {
-        _logger.LogDebug("Dequeueing {req} into {i}: {user} with {file}", userRequest.RequestId, slot, userRequest.User, userRequest.FileId);
-        _userQueueRequests[slot] = new(userRequest, DateTime.UtcNow.AddSeconds(_queueExpirationSeconds));
-        await _hubContext.Clients.User(userRequest.User).SendAsync(nameof(IMareHub.Client_DownloadReady), userRequest.RequestId).ConfigureAwait(false);
-    }
-
-    public Task StartAsync(CancellationToken cancellationToken)
-    {
-        _queueTimer = new System.Timers.Timer(250);
-        _queueTimer.Elapsed += ProcessQueue;
-        _queueTimer.AutoReset = true;
-        _queueTimer.Start();
-        return Task.CompletedTask;
-    }
-
-    public Task StopAsync(CancellationToken cancellationToken)
-    {
-        _queueTimer.Stop();
-        return Task.CompletedTask;
-    }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs

@@ -1,4 +1,3 @@
-using Grpc.Net.Client.Configuration;
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
 using MareSynchronosShared.Services;
@@ -16,6 +15,10 @@ using Microsoft.AspNetCore.SignalR;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.IdentityModel.Tokens;
 using Prometheus;
+using StackExchange.Redis.Extensions.Core.Configuration;
+using StackExchange.Redis.Extensions.System.Text.Json;
+using StackExchange.Redis;
+using System.Net;
 using System.Text;
 
 namespace MareSynchronosStaticFilesServer;
@@ -77,12 +80,6 @@ public class Startup
             options.EnableThreadSafetyChecks(false);
         }, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
 
-        var noRetryConfig = new MethodConfig
-        {
-            Names = { MethodName.Default },
-            RetryPolicy = null,
-        };
-
         services.AddOptions<JwtBearerOptions>(JwtBearerDefaults.AuthenticationScheme)
             .Configure<IConfigurationService<MareConfigurationAuthBase>>((o, s) =>
             {
@@ -111,11 +108,6 @@ public class Startup
 
         if (_isMain)
         {
-            services.AddGrpc(o =>
-            {
-                o.MaxReceiveMessageSize = null;
-            });
-
             services.AddSingleton<IConfigurationService<StaticFilesServerConfiguration>, MareConfigurationServiceServer<StaticFilesServerConfiguration>>();
         }
         else
@@ -174,6 +166,39 @@ public class Startup
         var redisConnection = mareConfig.GetValue(nameof(ServerConfiguration.RedisConnectionString), string.Empty);
         signalRServiceBuilder.AddStackExchangeRedis(redisConnection, options => { });
 
+        var options = ConfigurationOptions.Parse(redisConnection);
+
+        var endpoint = options.EndPoints[0];
+        string address = "";
+        int port = 0;
+        if (endpoint is DnsEndPoint dnsEndPoint) { address = dnsEndPoint.Host; port = dnsEndPoint.Port; }
+        if (endpoint is IPEndPoint ipEndPoint) { address = ipEndPoint.Address.ToString(); port = ipEndPoint.Port; }
+        var redisConfiguration = new RedisConfiguration()
+        {
+            AbortOnConnectFail = true,
+            KeyPrefix = "",
+            Hosts = new RedisHost[]
+            {
+                new RedisHost(){ Host = address, Port = port },
+            },
+            AllowAdmin = true,
+            ConnectTimeout = options.ConnectTimeout,
+            Database = 0,
+            Ssl = false,
+            Password = options.Password,
+            ServerEnumerationStrategy = new ServerEnumerationStrategy()
+            {
+                Mode = ServerEnumerationStrategy.ModeOptions.All,
+                TargetRole = ServerEnumerationStrategy.TargetRoleOptions.Any,
+                UnreachableServerAction = ServerEnumerationStrategy.UnreachableServerActionOptions.Throw,
+            },
+            MaxValueLength = 1024,
+            PoolSize = mareConfig.GetValue(nameof(ServerConfiguration.RedisPool), 50),
+            SyncTimeout = options.SyncTimeout,
+        };
+
+        services.AddStackExchangeRedisExtensions<SystemTextJsonSerializer>(redisConfiguration);
+
         services.AddHealthChecks();
     }
 
@@ -195,10 +220,6 @@ public class Startup
 
         app.UseEndpoints(e =>
         {
-            if (_isMain)
-            {
-                e.MapGrpcService<GrpcFileService>();
-            }
             e.MapHub<MareSynchronosServer.Hubs.MareHub>("/dummyhub");
             e.MapControllers();
             e.MapHealthChecks("/health").WithMetadata(new AllowAnonymousAttribute());