rework server responsibilities (#18)

* rework server responsibilities add remote configuration * start metrics only when compiled as not debug * add some more logging to discord bot * fixes of some casts * make metrics port configurable, minor fixes * add docker bullshit * md formatting * adjustments to docker stuff * fix docker json files, fix some stuff in discord bot, add /useradd for Discord bot * adjust docker configs and fix sharded.bat * fixes for logs, cache file provider repeat trying to open filestream Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>
2022-12-27 13:48:05 +01:00
parent 7ee7fdaf48
commit 9eb5967935
101 changed files with 2470 additions and 585 deletions
--- a/MareSynchronosServer/MareSynchronosServer/Services/GrpcClientIdentificationService.cs
+++ b/MareSynchronosServer/MareSynchronosServer/Services/GrpcClientIdentificationService.cs
@@ -2,17 +2,12 @@
 using MareSynchronosShared.Metrics;
 using MareSynchronosShared.Protos;
 using MareSynchronosShared.Services;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-using System;
+using MareSynchronosShared.Utils;
 using System.Collections.Concurrent;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;

 namespace MareSynchronosServer.Services;

-public class GrpcClientIdentificationService : GrpcBaseService
+public class GrpcClientIdentificationService : GrpcBaseService, IClientIdentificationService
 {
    private readonly string _shardName;
    private readonly ILogger<GrpcClientIdentificationService> _logger;
@@ -22,13 +17,15 @@ public class GrpcClientIdentificationService : GrpcBaseService
    private readonly MareMetrics _metrics;
    protected readonly ConcurrentDictionary<string, UidWithIdent> OnlineClients = new(StringComparer.Ordinal);
    private readonly ConcurrentDictionary<string, UidWithIdent> RemoteCachedIdents = new(StringComparer.Ordinal);
-    private ConcurrentQueue<IdentChange> _identChangeQueue = new();
+    private readonly ConcurrentQueue<IdentChange> _identChangeQueue = new();

-    public GrpcClientIdentificationService(ILogger<GrpcClientIdentificationService> logger, IdentificationService.IdentificationServiceClient gprcIdentClient,
+    public GrpcClientIdentificationService(ILogger<GrpcClientIdentificationService> logger,
+        IdentificationService.IdentificationServiceClient gprcIdentClient,
        IdentificationService.IdentificationServiceClient gprcIdentClientStreamOut,
-        IdentificationService.IdentificationServiceClient gprcIdentClientStreamIn, MareMetrics metrics, IOptions<ServerConfiguration> configuration) : base(logger)
+        IdentificationService.IdentificationServiceClient gprcIdentClientStreamIn, 
+        MareMetrics metrics, IConfigurationService<ServerConfiguration> configuration) : base(logger)
    {
-        _shardName = configuration.Value.ShardName;
+        _shardName = configuration.GetValueOrDefault(nameof(ServerConfiguration.ShardName), string.Empty);
        _logger = logger;
        _grpcIdentClient = gprcIdentClient;
        _grpcIdentClientStreamOut = gprcIdentClientStreamOut;
@@ -171,7 +168,7 @@ public class GrpcClientIdentificationService : GrpcBaseService
            using var stream = _grpcIdentClientStreamIn.ReceiveStreamIdentStatusChange(new ServerMessage()
            {
                ServerId = _shardName,
-            });
+            }, cancellationToken: cts);
            _logger.LogInformation("Starting Receive Online Client Data stream");
            await foreach (var cur in stream.ResponseStream.ReadAllAsync(cts).ConfigureAwait(false))
            {
@@ -201,7 +198,7 @@ public class GrpcClientIdentificationService : GrpcBaseService

    protected override async Task StopAsyncInternal(CancellationToken cancellationToken)
    {
-        await ExecuteOnGrpc(_grpcIdentClient.ClearIdentsForServerAsync(new ServerMessage() { ServerId = _shardName })).ConfigureAwait(false);
+        await ExecuteOnGrpc(_grpcIdentClient.ClearIdentsForServerAsync(new ServerMessage() { ServerId = _shardName }, cancellationToken: cancellationToken)).ConfigureAwait(false);
    }

    protected override async Task OnGrpcRestore()
--- a/MareSynchronosServer/MareSynchronosServer/Services/IClientIdentificationService.cs
+++ b/MareSynchronosServer/MareSynchronosServer/Services/IClientIdentificationService.cs
@@ -0,0 +1,11 @@
+namespace MareSynchronosServer.Services;
+
+public interface IClientIdentificationService : IHostedService
+{
+    string GetCharacterIdentForUid(string uid);
+    Task<long> GetOnlineUsers();
+    string GetServerForUid(string uid);
+    bool IsOnCurrentServer(string uid);
+    void MarkUserOffline(string uid);
+    void MarkUserOnline(string uid, string charaIdent);
+}
--- a/MareSynchronosServer/MareSynchronosServer/Services/IdentityService.cs
+++ b/MareSynchronosServer/MareSynchronosServer/Services/IdentityService.cs
@@ -0,0 +1,161 @@
+using Grpc.Core;
+using MareSynchronosServer.Identity;
+using MareSynchronosShared.Protos;
+using Microsoft.AspNetCore.Authorization;
+
+namespace MareSynchronosServer.Services;
+
+internal class GrpcIdentityService : IdentificationService.IdentificationServiceBase
+{
+    private readonly ILogger<GrpcIdentityService> _logger;
+    private readonly IdentityHandler _handler;
+
+    public GrpcIdentityService(ILogger<GrpcIdentityService> logger, IdentityHandler handler)
+    {
+        _logger = logger;
+        _handler = handler;
+    }
+
+    public override async Task<CharacterIdentMessage> GetIdentForUid(UidMessage request, ServerCallContext context)
+    {
+        var result = await _handler.GetIdentForUid(request.Uid).ConfigureAwait(false);
+        return new CharacterIdentMessage()
+        {
+            Ident = result.CharacterIdent,
+            ServerId = result.ServerId
+        };
+    }
+
+    [AllowAnonymous]
+    public override Task<OnlineUserCountResponse> GetOnlineUserCount(ServerMessage request, ServerCallContext context)
+    {
+        return Task.FromResult(new OnlineUserCountResponse() { Count = _handler.GetOnlineUsers(request.ServerId) });
+    }
+
+    public override Task<Empty> ClearIdentsForServer(ServerMessage request, ServerCallContext context)
+    {
+        var idents = _handler.GetIdentsForServer(request.ServerId);
+        foreach (var entry in idents)
+        {
+            EnqueueIdentOffline(new UidWithIdent()
+            {
+                Ident = new CharacterIdentMessage()
+                {
+                    Ident = entry.Value.CharacterIdent,
+                    ServerId = entry.Value.ServerId
+                },
+                Uid = new UidMessage()
+                {
+                    Uid = entry.Key
+                }
+            });
+        }
+
+        _handler.ClearIdentsForServer(request.ServerId);
+        return Task.FromResult(new Empty());
+    }
+
+    public override Task<Empty> RecreateServerIdents(ServerIdentMessage request, ServerCallContext context)
+    {
+        foreach (var identMsg in request.Idents)
+        {
+            _handler.SetIdent(identMsg.UidWithIdent.Uid.Uid, identMsg.UidWithIdent.Ident.ServerId, identMsg.UidWithIdent.Ident.Ident);
+            EnqueueIdentOnline(identMsg.UidWithIdent);
+        }
+        return Task.FromResult(new Empty());
+    }
+
+    public override async Task<Empty> SendStreamIdentStatusChange(IAsyncStreamReader<IdentChangeMessage> requestStream, ServerCallContext context)
+    {
+        await requestStream.MoveNext().ConfigureAwait(false);
+        var server = requestStream.Current.Server;
+        if (server == null) throw new System.Exception("First message needs to be server message");
+        _handler.RegisterServerForQueue(server.ServerId);
+        _logger.LogInformation("Registered Server " + server.ServerId + " input stream");
+
+        while (await requestStream.MoveNext(context.CancellationToken).ConfigureAwait(false))
+        {
+            var cur = requestStream.Current.IdentChange;
+            if (cur == null) throw new System.Exception("Expected client ident change");
+            _handler.EnqueueIdentChange(cur);
+
+            if (cur.IsOnline)
+            {
+                _handler.SetIdent(cur.UidWithIdent.Uid.Uid, cur.UidWithIdent.Ident.ServerId, cur.UidWithIdent.Ident.Ident);
+            }
+            else
+            {
+                _handler.RemoveIdent(cur.UidWithIdent.Uid.Uid, cur.UidWithIdent.Ident.ServerId);
+            }
+        }
+
+        _logger.LogInformation("Server input stream from " + server + " finished");
+
+        return new Empty();
+    }
+
+    public override async Task ReceiveStreamIdentStatusChange(ServerMessage request, IServerStreamWriter<IdentChange> responseStream, ServerCallContext context)
+    {
+        var server = request.ServerId;
+        _logger.LogInformation("Registered Server " + server + " output stream");
+
+        try
+        {
+            while (!context.CancellationToken.IsCancellationRequested)
+            {
+                while (_handler.DequeueIdentChange(server, out var cur))
+                {
+                    await responseStream.WriteAsync(cur).ConfigureAwait(false);
+                }
+
+                await Task.Delay(10).ConfigureAwait(false);
+            }
+        }
+        catch
+        {
+            _logger.LogInformation("Server output stream to " + server + " is faulty");
+        }
+
+        _logger.LogInformation("Server output stream to " + server + " is finished");
+    }
+
+    public override Task<UidWithIdentMessage> GetAllIdents(ServerMessage request, ServerCallContext context)
+    {
+        var response = new UidWithIdentMessage();
+        foreach (var item in _handler.GetIdentsForAllExcept(request.ServerId))
+        {
+            response.UidWithIdent.Add(new UidWithIdent()
+            {
+                Uid = new UidMessage()
+                {
+                    Uid = item.Key
+                },
+                Ident = new CharacterIdentMessage()
+                {
+                    Ident = item.Value.CharacterIdent,
+                    ServerId = item.Value.ServerId
+                }
+            });
+        }
+
+        return Task.FromResult(response);
+    }
+
+    private void EnqueueIdentOnline(UidWithIdent ident)
+    {
+        _handler.EnqueueIdentChange(new IdentChange()
+        {
+            IsOnline = true,
+            UidWithIdent = ident
+        });
+    }
+
+    private void EnqueueIdentOffline(UidWithIdent ident)
+    {
+        _handler.EnqueueIdentChange(new IdentChange()
+        {
+            IsOnline = false,
+            UidWithIdent = ident
+        });
+    }
+}
--- a/MareSynchronosServer/MareSynchronosServer/Services/LocalClientIdentificationService.cs
+++ b/MareSynchronosServer/MareSynchronosServer/Services/LocalClientIdentificationService.cs
@@ -0,0 +1,60 @@
+using MareSynchronosShared.Protos;
+using System.Collections.Concurrent;
+using MareSynchronosServer.Identity;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils;
+
+namespace MareSynchronosServer.Services;
+
+public class LocalClientIdentificationService : IClientIdentificationService
+{
+    protected readonly ConcurrentDictionary<string, UidWithIdent> OnlineClients = new(StringComparer.Ordinal);
+    private readonly IdentityHandler _identityHandler;
+    private readonly string _shardName;
+
+    public LocalClientIdentificationService(IdentityHandler identityHandler, IConfigurationService<ServerConfiguration> config)
+    {
+        _identityHandler = identityHandler;
+        _shardName = config.GetValueOrDefault(nameof(ServerConfiguration.ShardName), string.Empty);
+    }
+
+    public string GetCharacterIdentForUid(string uid)
+    {
+        return _identityHandler.GetIdentForUid(uid).Result.CharacterIdent;
+    }
+
+    public Task<long> GetOnlineUsers()
+    {
+        return Task.FromResult((long)_identityHandler.GetOnlineUsers(string.Empty));
+    }
+
+    public string GetServerForUid(string uid)
+    {
+        return _identityHandler.GetIdentForUid(uid).Result.ServerId;
+    }
+
+    public bool IsOnCurrentServer(string uid)
+    {
+        return string.Equals(_identityHandler.GetIdentForUid(uid).Result.ServerId, _shardName, StringComparison.Ordinal);
+    }
+
+    public void MarkUserOffline(string uid)
+    {
+        _identityHandler.RemoveIdent(uid, _shardName);
+    }
+
+    public void MarkUserOnline(string uid, string charaIdent)
+    {
+        _identityHandler.SetIdent(uid, _shardName, charaIdent);
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        return Task.CompletedTask;
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        return Task.CompletedTask;
+    }
+}
--- a/MareSynchronosServer/MareSynchronosServer/Services/SystemInfoService.cs
+++ b/MareSynchronosServer/MareSynchronosServer/Services/SystemInfoService.cs
@@ -1,32 +1,30 @@
-using System;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using MareSynchronos.API;
+using MareSynchronos.API;
 using MareSynchronosServer.Hubs;
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils;
 using Microsoft.AspNetCore.SignalR;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;

 namespace MareSynchronosServer.Services;

 public class SystemInfoService : IHostedService, IDisposable
 {
    private readonly MareMetrics _mareMetrics;
+    private readonly IConfigurationService<ServerConfiguration> _config;
    private readonly IServiceProvider _services;
-    private readonly GrpcClientIdentificationService _clientIdentService;
+    private readonly IClientIdentificationService _clientIdentService;
    private readonly ILogger<SystemInfoService> _logger;
    private readonly IHubContext<MareHub, IMareHub> _hubContext;
    private Timer _timer;
    public SystemInfoDto SystemInfoDto { get; private set; } = new();

-    public SystemInfoService(MareMetrics mareMetrics, IServiceProvider services, GrpcClientIdentificationService clientIdentService, ILogger<SystemInfoService> logger, IHubContext<MareHub, IMareHub> hubContext)
+    public SystemInfoService(MareMetrics mareMetrics, IConfigurationService<ServerConfiguration> configurationService, IServiceProvider services,
+        IClientIdentificationService clientIdentService, ILogger<SystemInfoService> logger, IHubContext<MareHub, IMareHub> hubContext)
    {
        _mareMetrics = mareMetrics;
+        _config = configurationService;
        _services = services;
        _clientIdentService = clientIdentService;
        _logger = logger;
@@ -49,14 +47,16 @@ public class SystemInfoService : IHostedService, IDisposable
        _mareMetrics.SetGaugeTo(MetricsAPI.GaugeAvailableWorkerThreads, workerThreads);
        _mareMetrics.SetGaugeTo(MetricsAPI.GaugeAvailableIOWorkerThreads, ioThreads);

-        var secondaryServer = Environment.GetEnvironmentVariable("SECONDARY_SERVER");
-        if (string.IsNullOrEmpty(secondaryServer) || string.Equals(secondaryServer, "0", StringComparison.Ordinal))
+        if (_config.IsMain)
        {
+            var onlineUsers = (int)_clientIdentService.GetOnlineUsers().Result;
            SystemInfoDto = new SystemInfoDto()
            {
-                OnlineUsers = (int)_clientIdentService.GetOnlineUsers().Result,
+                OnlineUsers = onlineUsers,
            };

+            _logger.LogInformation("Sending System Info, Online Users: {onlineUsers}", onlineUsers);
+
            _hubContext.Clients.All.Client_UpdateSystemInfo(SystemInfoDto);

            using var scope = _services.CreateScope();
--- a/MareSynchronosServer/MareSynchronosServer/Services/UserCleanupService.cs
+++ b/MareSynchronosServer/MareSynchronosServer/Services/UserCleanupService.cs
@@ -0,0 +1,198 @@
+using MareSynchronosShared.Data;
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Models;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils;
+using Microsoft.EntityFrameworkCore;
+
+namespace MareSynchronosServer.Services;
+
+public class UserCleanupService : IHostedService
+{
+    private readonly MareMetrics metrics;
+    private readonly ILogger<UserCleanupService> _logger;
+    private readonly IServiceProvider _services;
+    private readonly IConfigurationService<ServerConfiguration> _configuration;
+    private CancellationTokenSource _cleanupCts;
+
+    public UserCleanupService(MareMetrics metrics, ILogger<UserCleanupService> logger, IServiceProvider services, IConfigurationService<ServerConfiguration> configuration)
+    {
+        this.metrics = metrics;
+        _logger = logger;
+        _services = services;
+        _configuration = configuration;
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Cleanup Service started");
+        _cleanupCts = new();
+
+        _ = CleanUp(_cleanupCts.Token);
+
+        return Task.CompletedTask;
+    }
+
+    private async Task CleanUp(CancellationToken ct)
+    {
+        while (!ct.IsCancellationRequested)
+        {
+            using var scope = _services.CreateScope();
+            using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
+
+            CleanUpOutdatedLodestoneAuths(dbContext);
+
+            await PurgeUnusedAccounts(dbContext).ConfigureAwait(false);
+
+            await PurgeTempInvites(dbContext).ConfigureAwait(false);
+
+            dbContext.SaveChanges();
+
+            var now = DateTime.Now;
+            TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
+            TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % 10, 0);
+            var span = futureTime.AddMinutes(10) - currentTime;
+
+            _logger.LogInformation("User Cleanup Complete, next run at {date}", now.Add(span));
+            await Task.Delay(span, ct).ConfigureAwait(false);
+        }
+    }
+
+    private async Task PurgeTempInvites(MareDbContext dbContext)
+    {
+        try
+        {
+            var tempInvites = await dbContext.GroupTempInvites.ToListAsync().ConfigureAwait(false);
+            dbContext.RemoveRange(tempInvites.Where(i => i.ExpirationDate < DateTime.UtcNow));
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error during Temp Invite purge");
+        }
+    }
+
+    private async Task PurgeUnusedAccounts(MareDbContext dbContext)
+    {
+        try
+        {
+            if (_configuration.GetValueOrDefault(nameof(ServerConfiguration.PurgeUnusedAccounts), false))
+            {
+                var usersOlderThanDays = _configuration.GetValueOrDefault(nameof(ServerConfiguration.PurgeUnusedAccountsPeriodInDays), 14);
+                var maxGroupsByUser = _configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxGroupUserCount), 3);
+
+                _logger.LogInformation("Cleaning up users older than {usersOlderThanDays} days", usersOlderThanDays);
+
+                var allUsers = dbContext.Users.Where(u => string.IsNullOrEmpty(u.Alias)).ToList();
+                List<User> usersToRemove = new();
+                foreach (var user in allUsers)
+                {
+                    if (user.LastLoggedIn < DateTime.UtcNow - TimeSpan.FromDays(usersOlderThanDays))
+                    {
+                        _logger.LogInformation("User outdated: {userUID}", user.UID);
+                        usersToRemove.Add(user);
+                    }
+                }
+
+                foreach (var user in usersToRemove)
+                {
+                    await SharedDbFunctions.PurgeUser(_logger, user, dbContext, maxGroupsByUser).ConfigureAwait(false);
+                }
+            }
+
+            _logger.LogInformation("Cleaning up unauthorized users");
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error during user purge");
+        }
+    }
+
+    private void CleanUpOutdatedLodestoneAuths(MareDbContext dbContext)
+    {
+        try
+        {
+            _logger.LogInformation($"Cleaning up expired lodestone authentications");
+            var lodestoneAuths = dbContext.LodeStoneAuth.Include(u => u.User).Where(a => a.StartedAt != null).ToList();
+            List<LodeStoneAuth> expiredAuths = new List<LodeStoneAuth>();
+            foreach (var auth in lodestoneAuths)
+            {
+                if (auth.StartedAt < DateTime.UtcNow - TimeSpan.FromMinutes(15))
+                {
+                    expiredAuths.Add(auth);
+                }
+            }
+
+            dbContext.Users.RemoveRange(expiredAuths.Where(u => u.User != null).Select(a => a.User));
+            dbContext.RemoveRange(expiredAuths);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error during expired auths cleanup");
+        }
+    }
+
+    public async Task PurgeUser(User user, MareDbContext dbContext)
+    {
+        _logger.LogInformation("Purging user: {uid}", user.UID);
+
+        var lodestone = dbContext.LodeStoneAuth.SingleOrDefault(a => a.User.UID == user.UID);
+
+        if (lodestone != null)
+        {
+            dbContext.Remove(lodestone);
+        }
+
+        var auth = dbContext.Auth.Single(a => a.UserUID == user.UID);
+
+        var userFiles = dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == user.UID).ToList();
+        dbContext.Files.RemoveRange(userFiles);
+
+        var ownPairData = dbContext.ClientPairs.Where(u => u.User.UID == user.UID).ToList();
+        dbContext.ClientPairs.RemoveRange(ownPairData);
+        var otherPairData = dbContext.ClientPairs.Include(u => u.User)
+            .Where(u => u.OtherUser.UID == user.UID).ToList();
+        dbContext.ClientPairs.RemoveRange(otherPairData);
+
+        var userJoinedGroups = await dbContext.GroupPairs.Include(g => g.Group).Where(u => u.GroupUserUID == user.UID).ToListAsync().ConfigureAwait(false);
+
+        foreach (var userGroupPair in userJoinedGroups)
+        {
+            bool ownerHasLeft = string.Equals(userGroupPair.Group.OwnerUID, user.UID, StringComparison.Ordinal);
+
+            if (ownerHasLeft)
+            {
+                var groupPairs = await dbContext.GroupPairs.Where(g => g.GroupGID == userGroupPair.GroupGID && g.GroupUserUID != user.UID).ToListAsync().ConfigureAwait(false);
+
+                if (!groupPairs.Any())
+                {
+                    _logger.LogInformation("Group {gid} has no new owner, deleting", userGroupPair.GroupGID);
+                    dbContext.Groups.Remove(userGroupPair.Group);
+                }
+                else
+                {
+                    _ = await SharedDbFunctions.MigrateOrDeleteGroup(dbContext, userGroupPair.Group, groupPairs, _configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxExistingGroupsByUser), 3)).ConfigureAwait(false);
+                }
+            }
+
+            dbContext.GroupPairs.Remove(userGroupPair);
+
+            await dbContext.SaveChangesAsync().ConfigureAwait(false);
+        }
+
+        _logger.LogInformation("User purged: {uid}", user.UID);
+
+        dbContext.Auth.Remove(auth);
+        dbContext.Users.Remove(user);
+
+        await dbContext.SaveChangesAsync().ConfigureAwait(false);
+
+        metrics.DecGauge(MetricsAPI.GaugeUsersRegistered, 1);
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        _cleanupCts.Cancel();
+
+        return Task.CompletedTask;
+    }
+}