rework server responsibilities (#18)

* rework server responsibilities
add remote configuration

* start metrics only when compiled as not debug

* add some more logging to discord bot

* fixes of some casts

* make metrics port configurable, minor fixes

* add docker bullshit

* md formatting

* adjustments to docker stuff

* fix docker json files, fix some stuff in discord bot, add /useradd for Discord bot

* adjust docker configs and fix sharded.bat

* fixes for logs, cache file provider repeat trying to open filestream

Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>
This commit is contained in:
rootdarkarchon
2022-12-27 13:48:05 +01:00
committed by GitHub
parent 7ee7fdaf48
commit 9eb5967935
101 changed files with 2470 additions and 585 deletions

View File

@@ -1,185 +0,0 @@
using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Models;
using MareSynchronosShared.Utils;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace MareSynchronosServices;
public class CleanupService : IHostedService, IDisposable
{
private readonly MareMetrics metrics;
private readonly ILogger<CleanupService> _logger;
private readonly IServiceProvider _services;
private readonly ServicesConfiguration _configuration;
private Timer? _timer;
public CleanupService(MareMetrics metrics, ILogger<CleanupService> logger, IServiceProvider services, IOptions<ServicesConfiguration> configuration)
{
this.metrics = metrics;
_logger = logger;
_services = services;
_configuration = configuration.Value;
}
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Cleanup Service started");
_timer = new Timer(CleanUp, null, TimeSpan.Zero, TimeSpan.FromMinutes(10));
return Task.CompletedTask;
}
private async void CleanUp(object state)
{
using var scope = _services.CreateScope();
using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
try
{
_logger.LogInformation($"Cleaning up expired lodestone authentications");
var lodestoneAuths = dbContext.LodeStoneAuth.Include(u => u.User).Where(a => a.StartedAt != null).ToList();
List<LodeStoneAuth> expiredAuths = new List<LodeStoneAuth>();
foreach (var auth in lodestoneAuths)
{
if (auth.StartedAt < DateTime.UtcNow - TimeSpan.FromMinutes(15))
{
expiredAuths.Add(auth);
}
}
dbContext.Users.RemoveRange(expiredAuths.Where(u => u.User != null).Select(a => a.User));
dbContext.RemoveRange(expiredAuths);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error during expired auths cleanup");
}
try
{
if (_configuration.PurgeUnusedAccounts)
{
var usersOlderThanDays = _configuration.PurgeUnusedAccountsPeriodInDays;
_logger.LogInformation("Cleaning up users older than {usersOlderThanDays} days", usersOlderThanDays);
var allUsers = dbContext.Users.Where(u => string.IsNullOrEmpty(u.Alias)).ToList();
List<User> usersToRemove = new();
foreach (var user in allUsers)
{
if (user.LastLoggedIn < (DateTime.UtcNow - TimeSpan.FromDays(usersOlderThanDays)))
{
_logger.LogInformation("User outdated: {userUID}", user.UID);
usersToRemove.Add(user);
}
}
foreach (var user in usersToRemove)
{
await PurgeUser(user, dbContext);
}
}
_logger.LogInformation("Cleaning up unauthorized users");
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error during user purge");
}
try
{
var tempInvites = await dbContext.GroupTempInvites.ToListAsync();
dbContext.RemoveRange(tempInvites.Where(i => i.ExpirationDate < DateTime.UtcNow));
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error during Temp Invite purge");
}
_logger.LogInformation($"Cleanup complete");
dbContext.SaveChanges();
}
public async Task PurgeUser(User user, MareDbContext dbContext)
{
_logger.LogInformation("Purging user: {uid}", user.UID);
var lodestone = dbContext.LodeStoneAuth.SingleOrDefault(a => a.User.UID == user.UID);
if (lodestone != null)
{
dbContext.Remove(lodestone);
}
var auth = dbContext.Auth.Single(a => a.UserUID == user.UID);
var userFiles = dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == user.UID).ToList();
dbContext.Files.RemoveRange(userFiles);
var ownPairData = dbContext.ClientPairs.Where(u => u.User.UID == user.UID).ToList();
dbContext.ClientPairs.RemoveRange(ownPairData);
var otherPairData = dbContext.ClientPairs.Include(u => u.User)
.Where(u => u.OtherUser.UID == user.UID).ToList();
dbContext.ClientPairs.RemoveRange(otherPairData);
var userJoinedGroups = await dbContext.GroupPairs.Include(g => g.Group).Where(u => u.GroupUserUID == user.UID).ToListAsync().ConfigureAwait(false);
foreach (var userGroupPair in userJoinedGroups)
{
bool ownerHasLeft = string.Equals(userGroupPair.Group.OwnerUID, user.UID, StringComparison.Ordinal);
if (ownerHasLeft)
{
var groupPairs = await dbContext.GroupPairs.Where(g => g.GroupGID == userGroupPair.GroupGID && g.GroupUserUID != user.UID).ToListAsync().ConfigureAwait(false);
if (!groupPairs.Any())
{
_logger.LogInformation("Group {gid} has no new owner, deleting", userGroupPair.GroupGID);
dbContext.Groups.Remove(userGroupPair.Group);
}
else
{
_ = await SharedDbFunctions.MigrateOrDeleteGroup(dbContext, userGroupPair.Group, groupPairs, _configuration.MaxExistingGroupsByUser).ConfigureAwait(false);
}
}
dbContext.GroupPairs.Remove(userGroupPair);
await dbContext.SaveChangesAsync();
}
_logger.LogInformation("User purged: {uid}", user.UID);
dbContext.Auth.Remove(auth);
dbContext.Users.Remove(user);
await dbContext.SaveChangesAsync();
metrics.DecGauge(MetricsAPI.GaugeUsersRegistered, 1);
}
public Task StopAsync(CancellationToken cancellationToken)
{
_timer?.Change(Timeout.Infinite, 0);
return Task.CompletedTask;
}
public void Dispose()
{
_timer?.Dispose();
}
}

View File

@@ -1,41 +1,34 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Discord;
using Discord;
using Discord.Interactions;
using Discord.Rest;
using Discord.WebSocket;
using MareSynchronosServices.Identity;
using MareSynchronosShared.Data;
using MareSynchronosShared.Services;
using MareSynchronosShared.Utils;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using static MareSynchronosShared.Protos.IdentificationService;
namespace MareSynchronosServices.Discord;
internal class DiscordBot : IHostedService
{
private readonly DiscordBotServices _botServices;
private readonly IdentityHandler _identityHandler;
private readonly IServiceProvider _services;
private readonly ServicesConfiguration _configuration;
private readonly IConfigurationService<ServicesConfiguration> _configurationService;
private readonly ILogger<DiscordBot> _logger;
private readonly IdentificationServiceClient _identificationServiceClient;
private readonly DiscordSocketClient _discordClient;
private CancellationTokenSource? _updateStatusCts;
private CancellationTokenSource? _vanityUpdateCts;
public DiscordBot(DiscordBotServices botServices, IdentityHandler identityHandler, IServiceProvider services, IOptions<ServicesConfiguration> configuration, ILogger<DiscordBot> logger)
public DiscordBot(DiscordBotServices botServices, IServiceProvider services, IConfigurationService<ServicesConfiguration> configuration,
ILogger<DiscordBot> logger, IdentificationServiceClient identificationServiceClient)
{
_botServices = botServices;
_identityHandler = identityHandler;
_services = services;
_configuration = configuration.Value;
_configurationService = configuration;
_logger = logger;
this._identificationServiceClient = identificationServiceClient;
_discordClient = new(new DiscordSocketConfig()
{
DefaultRetryMode = RetryMode.AlwaysRetry
@@ -72,7 +65,8 @@ internal class DiscordBot : IHostedService
_vanityUpdateCts = new();
var guild = (await _discordClient.Rest.GetGuildsAsync()).First();
var commands = await guild.GetApplicationCommandsAsync();
var vanityCommandId = commands.First(c => c.Name == "setvanityuid").Id;
var appId = await _discordClient.GetApplicationInfoAsync().ConfigureAwait(false);
var vanityCommandId = commands.First(c => c.ApplicationId == appId.Id && c.Name == "setvanityuid").Id;
while (!_vanityUpdateCts.IsCancellationRequested)
{
@@ -176,18 +170,19 @@ internal class DiscordBot : IHostedService
_updateStatusCts = new();
while (!_updateStatusCts.IsCancellationRequested)
{
var onlineUsers = _identityHandler.GetOnlineUsers(string.Empty);
_logger.LogInformation("Users online: " + onlineUsers);
await _discordClient.SetActivityAsync(new Game("Mare for " + onlineUsers + " Users")).ConfigureAwait(false);
var onlineUsers = await _identificationServiceClient.GetOnlineUserCountAsync(new MareSynchronosShared.Protos.ServerMessage());
_logger.LogInformation("Users online: " + onlineUsers.Count);
await _discordClient.SetActivityAsync(new Game("Mare for " + onlineUsers.Count + " Users")).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(15)).ConfigureAwait(false);
}
}
public async Task StartAsync(CancellationToken cancellationToken)
{
if (!string.IsNullOrEmpty(_configuration.DiscordBotToken))
var token = _configurationService.GetValueOrDefault(nameof(ServicesConfiguration.DiscordBotToken), string.Empty);
if (!string.IsNullOrEmpty(token))
{
await _discordClient.LoginAsync(TokenType.Bot, _configuration.DiscordBotToken).ConfigureAwait(false);
await _discordClient.LoginAsync(TokenType.Bot, token).ConfigureAwait(false);
await _discordClient.StartAsync().ConfigureAwait(false);
_discordClient.Ready += DiscordClient_Ready;
@@ -199,7 +194,7 @@ internal class DiscordBot : IHostedService
public async Task StopAsync(CancellationToken cancellationToken)
{
if (!string.IsNullOrEmpty(_configuration.DiscordBotToken))
if (!string.IsNullOrEmpty(_configurationService.GetValueOrDefault(nameof(ServicesConfiguration.DiscordBotToken), string.Empty)))
{
await _botServices.Stop();
_updateStatusCts?.Cancel();

View File

@@ -1,17 +1,11 @@
using System;
using System.Threading.Tasks;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Collections.Concurrent;
using MareSynchronosShared.Metrics;
using Microsoft.Extensions.Logging;
using System.Threading;
using Microsoft.Extensions.Options;
namespace MareSynchronosServices.Discord;
public class DiscordBotServices
{
public readonly ConcurrentQueue<KeyValuePair<ulong, Action<IServiceProvider>>> verificationQueue = new();
public ConcurrentQueue<KeyValuePair<ulong, Action<IServiceProvider>>> VerificationQueue { get; } = new();
public ConcurrentDictionary<ulong, DateTime> LastVanityChange = new();
public ConcurrentDictionary<string, DateTime> LastVanityGidChange = new();
public ConcurrentDictionary<ulong, string> DiscordLodestoneMapping = new();
@@ -19,29 +13,27 @@ public class DiscordBotServices
public readonly string[] LodestoneServers = new[] { "eu", "na", "jp", "fr", "de" };
private readonly IServiceProvider _serviceProvider;
public ServicesConfiguration Configuration { get; init; }
public ILogger<DiscordBotServices> Logger { get; init; }
public MareMetrics Metrics { get; init; }
public Random Random { get; init; }
private CancellationTokenSource? verificationTaskCts;
public DiscordBotServices(IOptions<ServicesConfiguration> configuration, IServiceProvider serviceProvider, ILogger<DiscordBotServices> logger, MareMetrics metrics)
public DiscordBotServices(IServiceProvider serviceProvider, ILogger<DiscordBotServices> logger, MareMetrics metrics)
{
Configuration = configuration.Value;
_serviceProvider = serviceProvider;
Logger = logger;
Metrics = metrics;
Random = new();
}
public async Task Start()
public Task Start()
{
_ = ProcessVerificationQueue();
return Task.CompletedTask;
}
public async Task Stop()
public Task Stop()
{
verificationTaskCts?.Cancel();
return Task.CompletedTask;
}
private async Task ProcessVerificationQueue()
@@ -49,7 +41,7 @@ public class DiscordBotServices
verificationTaskCts = new CancellationTokenSource();
while (!verificationTaskCts.IsCancellationRequested)
{
if (verificationQueue.TryDequeue(out var queueitem))
if (VerificationQueue.TryDequeue(out var queueitem))
{
try
{
@@ -61,8 +53,8 @@ public class DiscordBotServices
{
Logger.LogError(e, "Error during queue work");
}
}
await Task.Delay(TimeSpan.FromSeconds(2), verificationTaskCts.Token).ConfigureAwait(false);
}
}

View File

@@ -2,20 +2,15 @@
using Discord.Interactions;
using MareSynchronosShared.Data;
using System.Text.RegularExpressions;
using System;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.EntityFrameworkCore;
using Discord.WebSocket;
using System.Linq;
using Prometheus;
using MareSynchronosShared.Models;
using MareSynchronosServices.Identity;
using MareSynchronosShared.Metrics;
using System.Net.Http;
using MareSynchronosShared.Utils;
using System.Collections.Generic;
using Microsoft.Extensions.Logging;
using MareSynchronosShared.Services;
using static MareSynchronosShared.Protos.IdentificationService;
using static System.Formats.Asn1.AsnWriter;
namespace MareSynchronosServices.Discord;
@@ -30,22 +25,30 @@ public class LodestoneModal : IModal
public class MareModule : InteractionModuleBase
{
private readonly ILogger<MareModule> _logger;
private readonly IServiceProvider _services;
private readonly DiscordBotServices _botServices;
private readonly IdentityHandler _identityHandler;
private readonly CleanupService _cleanupService;
private readonly IdentificationServiceClient _identificationServiceClient;
private readonly IConfigurationService<ServerConfiguration> _mareClientConfigurationService;
private Random random = new();
public MareModule(IServiceProvider services, DiscordBotServices botServices, IdentityHandler identityHandler, CleanupService cleanupService)
public MareModule(ILogger<MareModule> logger, IServiceProvider services, DiscordBotServices botServices,
IdentificationServiceClient identificationServiceClient, IConfigurationService<ServerConfiguration> mareClientConfigurationService)
{
_logger = logger;
_services = services;
_botServices = botServices;
_identityHandler = identityHandler;
_cleanupService = cleanupService;
_identificationServiceClient = identificationServiceClient;
_mareClientConfigurationService = mareClientConfigurationService;
}
[SlashCommand("register", "Starts the registration process for the Mare Synchronos server of this Discord")]
public async Task Register([Summary("overwrite", "Overwrites your old account")] bool overwrite = false)
{
_logger.LogInformation("SlashCommand:{userId}:{Method}:{params}",
Context.Client.CurrentUser.Id, nameof(Register),
string.Join(",", new[] { $"{nameof(overwrite)}:{overwrite}" }));
await TryRespondAsync(async () =>
{
if (overwrite)
@@ -60,6 +63,10 @@ public class MareModule : InteractionModuleBase
[SlashCommand("setvanityuid", "Sets your Vanity UID.")]
public async Task SetVanityUid([Summary("vanity_uid", "Desired Vanity UID")] string vanityUid)
{
_logger.LogInformation("SlashCommand:{userId}:{Method}:{params}",
Context.Client.CurrentUser.Id, nameof(SetVanityUid),
string.Join(",", new[] { $"{nameof(vanityUid)}:{vanityUid}" }));
await TryRespondAsync(async () =>
{
EmbedBuilder eb = new();
@@ -75,6 +82,10 @@ public class MareModule : InteractionModuleBase
[Summary("syncshell_id", "Syncshell ID")] string syncshellId,
[Summary("vanity_syncshell_id", "Desired Vanity Syncshell ID")] string vanityId)
{
_logger.LogInformation("SlashCommand:{userId}:{Method}:{params}",
Context.Client.CurrentUser.Id, nameof(SetSyncshellVanityId),
string.Join(",", new[] { $"{nameof(syncshellId)}:{syncshellId}", $"{nameof(vanityId)}:{vanityId}" }));
await TryRespondAsync(async () =>
{
EmbedBuilder eb = new();
@@ -88,10 +99,12 @@ public class MareModule : InteractionModuleBase
[SlashCommand("verify", "Finishes the registration process for the Mare Synchronos server of this Discord")]
public async Task Verify()
{
_logger.LogInformation("SlashCommand:{userId}:{Method}",
Context.Client.CurrentUser.Id, nameof(Verify));
await TryRespondAsync(async () =>
{
EmbedBuilder eb = new();
if (_botServices.verificationQueue.Any(u => u.Key == Context.User.Id))
if (_botServices.VerificationQueue.Any(u => u.Key == Context.User.Id))
{
eb.WithTitle("Already queued for verfication");
eb.WithDescription("You are already queued for verification. Please wait.");
@@ -106,7 +119,7 @@ public class MareModule : InteractionModuleBase
else
{
await DeferAsync(ephemeral: true).ConfigureAwait(false);
_botServices.verificationQueue.Enqueue(new KeyValuePair<ulong, Action<IServiceProvider>>(Context.User.Id, async (sp) => await HandleVerifyAsync((SocketSlashCommand)Context.Interaction, sp)));
_botServices.VerificationQueue.Enqueue(new KeyValuePair<ulong, Action<IServiceProvider>>(Context.User.Id, async (sp) => await HandleVerifyAsync((SocketSlashCommand)Context.Interaction, sp)));
}
});
}
@@ -114,10 +127,12 @@ public class MareModule : InteractionModuleBase
[SlashCommand("verify_relink", "Finishes the relink process for your user on the Mare Synchronos server of this Discord")]
public async Task VerifyRelink()
{
_logger.LogInformation("SlashCommand:{userId}:{Method}",
Context.Client.CurrentUser.Id, nameof(VerifyRelink));
await TryRespondAsync(async () =>
{
EmbedBuilder eb = new();
if (_botServices.verificationQueue.Any(u => u.Key == Context.User.Id))
if (_botServices.VerificationQueue.Any(u => u.Key == Context.User.Id))
{
eb.WithTitle("Already queued for verfication");
eb.WithDescription("You are already queued for verification. Please wait.");
@@ -132,7 +147,7 @@ public class MareModule : InteractionModuleBase
else
{
await DeferAsync(ephemeral: true).ConfigureAwait(false);
_botServices.verificationQueue.Enqueue(new KeyValuePair<ulong, Action<IServiceProvider>>(Context.User.Id, async (sp) => await HandleVerifyRelinkAsync((SocketSlashCommand)Context.Interaction, sp)));
_botServices.VerificationQueue.Enqueue(new KeyValuePair<ulong, Action<IServiceProvider>>(Context.User.Id, async (sp) => await HandleVerifyRelinkAsync((SocketSlashCommand)Context.Interaction, sp)));
}
});
}
@@ -140,6 +155,8 @@ public class MareModule : InteractionModuleBase
[SlashCommand("recover", "Allows you to recover your account by generating a new secret key")]
public async Task Recover()
{
_logger.LogInformation("SlashCommand:{userId}:{Method}",
Context.Client.CurrentUser.Id, nameof(Recover));
await RespondWithModalAsync<LodestoneModal>("recover_modal").ConfigureAwait(false);
}
@@ -148,6 +165,9 @@ public class MareModule : InteractionModuleBase
[Summary("discord_user", "ADMIN ONLY: Discord User to check for")] IUser? discordUser = null,
[Summary("uid", "ADMIN ONLY: UID to check for")] string? uid = null)
{
_logger.LogInformation("SlashCommand:{userId}:{Method}",
Context.Client.CurrentUser.Id, nameof(UserInfo));
await TryRespondAsync(async () =>
{
EmbedBuilder eb = new();
@@ -161,12 +181,32 @@ public class MareModule : InteractionModuleBase
[SlashCommand("relink", "Allows you to link a new Discord account to an existing Mare account")]
public async Task Relink()
{
_logger.LogInformation("SlashCommand:{userId}:{Method}",
Context.Client.CurrentUser.Id, nameof(Relink));
await RespondWithModalAsync<LodestoneModal>("relink_modal").ConfigureAwait(false);
}
[SlashCommand("useradd", "ADMIN ONLY: add a user unconditionally to the Database")]
public async Task UserAdd([Summary("desired_uid", "Desired UID")] string desiredUid)
{
_logger.LogInformation("SlashCommand:{userId}:{Method}:{params}",
Context.Client.CurrentUser.Id, nameof(UserAdd),
string.Join(",", new[] { $"{nameof(desiredUid)}:{desiredUid}" }));
await TryRespondAsync(async () =>
{
var embed = await HandleUserAdd(desiredUid, Context.User.Id);
await RespondAsync(embeds: new[] { embed }, ephemeral: true).ConfigureAwait(false);
});
}
[ModalInteraction("recover_modal")]
public async Task RecoverModal(LodestoneModal modal)
{
_logger.LogInformation("Modal:{userId}:{Method}",
Context.Client.CurrentUser.Id, nameof(RecoverModal));
await TryRespondAsync(async () =>
{
var embed = await HandleRecoverModalAsync(modal, Context.User.Id).ConfigureAwait(false);
@@ -177,6 +217,9 @@ public class MareModule : InteractionModuleBase
[ModalInteraction("register_modal")]
public async Task RegisterModal(LodestoneModal modal)
{
_logger.LogInformation("Modal:{userId}:{Method}",
Context.Client.CurrentUser.Id, nameof(RegisterModal));
await TryRespondAsync(async () =>
{
var embed = await HandleRegisterModalAsync(modal, Context.User.Id).ConfigureAwait(false);
@@ -187,6 +230,9 @@ public class MareModule : InteractionModuleBase
[ModalInteraction("relink_modal")]
public async Task RelinkModal(LodestoneModal modal)
{
_logger.LogInformation("Modal:{userId}:{Method}",
Context.Client.CurrentUser.Id, nameof(RelinkModal));
await TryRespondAsync(async () =>
{
var embed = await HandleRelinkModalAsync(modal, Context.User.Id).ConfigureAwait(false);
@@ -194,6 +240,51 @@ public class MareModule : InteractionModuleBase
});
}
public async Task<Embed> HandleUserAdd(string desiredUid, ulong discordUserId)
{
var embed = new EmbedBuilder();
using var scope = _services.CreateScope();
using var db = scope.ServiceProvider.GetService<MareDbContext>();
if (!(await db.LodeStoneAuth.Include(u => u.User).SingleOrDefaultAsync(a => a.DiscordId == discordUserId))?.User?.IsAdmin ?? true)
{
embed.WithTitle("Failed to add user");
embed.WithDescription("No permission");
}
else if (db.Users.Any(u => u.UID == desiredUid || u.Alias == desiredUid))
{
embed.WithTitle("Failed to add user");
embed.WithDescription("Already in Database");
}
else
{
User newUser = new()
{
IsAdmin = false,
IsModerator = false,
LastLoggedIn = DateTime.UtcNow,
UID = desiredUid,
};
var computedHash = StringUtils.Sha256String(StringUtils.GenerateRandomString(64) + DateTime.UtcNow.ToString());
var auth = new Auth()
{
HashedKey = StringUtils.Sha256String(computedHash),
User = newUser,
};
await db.Users.AddAsync(newUser);
await db.Auth.AddAsync(auth);
await db.SaveChangesAsync();
embed.WithTitle("Successfully added " + desiredUid);
embed.WithDescription("Secret Key: " + computedHash);
}
return embed.Build();
}
private async Task TryRespondAsync(Action act)
{
try
@@ -258,7 +349,7 @@ public class MareModule : InteractionModuleBase
var lodestoneUser = await db.LodeStoneAuth.Include(u => u.User).SingleOrDefaultAsync(u => u.DiscordId == userToCheckForDiscordId).ConfigureAwait(false);
var dbUser = lodestoneUser.User;
var auth = await db.Auth.SingleOrDefaultAsync(u => u.UserUID == dbUser.UID).ConfigureAwait(false);
var identity = await _identityHandler.GetIdentForuid(dbUser.UID).ConfigureAwait(false);
var identity = await _identificationServiceClient.GetIdentForUidAsync(new MareSynchronosShared.Protos.UidMessage { Uid = dbUser.UID });
var groups = await db.Groups.Where(g => g.OwnerUID == dbUser.UID).ToListAsync().ConfigureAwait(false);
var groupsJoined = await db.GroupPairs.Where(g => g.GroupUserUID == dbUser.UID).ToListAsync().ConfigureAwait(false);
@@ -271,7 +362,7 @@ public class MareModule : InteractionModuleBase
eb.AddField("Vanity UID", dbUser.Alias);
}
eb.AddField("Last Online (UTC)", dbUser.LastLoggedIn.ToString("U"));
eb.AddField("Currently online: ", !string.IsNullOrEmpty(identity.CharacterIdent));
eb.AddField("Currently online: ", !string.IsNullOrEmpty(identity.Ident));
eb.AddField("Hashed Secret Key", auth.HashedKey);
eb.AddField("Joined Syncshells", groupsJoined.Count);
eb.AddField("Owned Syncshells", groups.Count);
@@ -285,9 +376,9 @@ public class MareModule : InteractionModuleBase
eb.AddField("Owned Syncshell " + group.GID + " User Count", syncShellUserCount);
}
if (isAdminCall && !string.IsNullOrEmpty(identity.CharacterIdent))
if (isAdminCall && !string.IsNullOrEmpty(identity.Ident))
{
eb.AddField("Character Ident", identity.CharacterIdent);
eb.AddField("Character Ident", identity.Ident);
}
return eb;
@@ -635,7 +726,9 @@ public class MareModule : InteractionModuleBase
{
if (discordAuthedUser.User != null)
{
await _cleanupService.PurgeUser(discordAuthedUser.User, db);
var maxGroupsByUser = _mareClientConfigurationService.GetValueOrDefault(nameof(ServerConfiguration.MaxGroupUserCount), 3);
await SharedDbFunctions.PurgeUser(_logger, discordAuthedUser.User, db, maxGroupsByUser);
}
else
{
@@ -657,7 +750,7 @@ public class MareModule : InteractionModuleBase
var lodestoneAuth = db.LodeStoneAuth.SingleOrDefault(u => u.DiscordId == cmd.User.Id);
if (lodestoneAuth != null && _botServices.DiscordRelinkLodestoneMapping.ContainsKey(cmd.User.Id))
{
var randomServer = _botServices.LodestoneServers[_botServices.Random.Next(_botServices.LodestoneServers.Length)];
var randomServer = _botServices.LodestoneServers[random.Next(_botServices.LodestoneServers.Length)];
var response = await req.GetAsync($"https://{randomServer}.finalfantasyxiv.com/lodestone/character/{_botServices.DiscordRelinkLodestoneMapping[cmd.User.Id]}").ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
@@ -731,7 +824,7 @@ public class MareModule : InteractionModuleBase
var lodestoneAuth = db.LodeStoneAuth.SingleOrDefault(u => u.DiscordId == cmd.User.Id);
if (lodestoneAuth != null && _botServices.DiscordLodestoneMapping.ContainsKey(cmd.User.Id))
{
var randomServer = _botServices.LodestoneServers[_botServices.Random.Next(_botServices.LodestoneServers.Length)];
var randomServer = _botServices.LodestoneServers[random.Next(_botServices.LodestoneServers.Length)];
var response = await req.GetAsync($"https://{randomServer}.finalfantasyxiv.com/lodestone/character/{_botServices.DiscordLodestoneMapping[cmd.User.Id]}").ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
@@ -757,11 +850,7 @@ public class MareModule : InteractionModuleBase
user.IsAdmin = true;
}
if (_botServices.Configuration.PurgeUnusedAccounts)
{
var purgedDays = _botServices.Configuration.PurgeUnusedAccountsPeriodInDays;
user.LastLoggedIn = DateTime.UtcNow - TimeSpan.FromDays(purgedDays) + TimeSpan.FromDays(1);
}
user.LastLoggedIn = DateTime.UtcNow;
var computedHash = StringUtils.Sha256String(StringUtils.GenerateRandomString(64) + DateTime.UtcNow.ToString());
var auth = new Auth()

View File

@@ -1,109 +0,0 @@
using MareSynchronosShared.Protos;
using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
namespace MareSynchronosServices.Identity;
public class IdentityHandler
{
private readonly ConcurrentDictionary<string, ServerIdentity> cachedIdentities = new();
private readonly ConcurrentDictionary<string, ConcurrentQueue<IdentChange>> identChanges = new();
private readonly ILogger<IdentityHandler> _logger;
public IdentityHandler(ILogger<IdentityHandler> logger)
{
_logger = logger;
}
internal Task<string> GetUidForCharacterIdent(string ident, string serverId)
{
var exists = cachedIdentities.Any(f => f.Value.CharacterIdent == ident && f.Value.ServerId == serverId);
return Task.FromResult(exists ? cachedIdentities.FirstOrDefault(f => f.Value.CharacterIdent == ident && f.Value.ServerId == serverId).Key : string.Empty);
}
internal Task<ServerIdentity> GetIdentForuid(string uid)
{
ServerIdentity result;
if (!cachedIdentities.TryGetValue(uid, out result))
{
result = new ServerIdentity();
}
return Task.FromResult(result);
}
internal void SetIdent(string uid, string serverId, string ident)
{
cachedIdentities[uid] = new ServerIdentity() { ServerId = serverId, CharacterIdent = ident };
}
internal void RemoveIdent(string uid, string serverId)
{
if (cachedIdentities.ContainsKey(uid) && cachedIdentities[uid].ServerId == serverId)
{
cachedIdentities.TryRemove(uid, out _);
}
}
internal int GetOnlineUsers(string serverId)
{
if (string.IsNullOrEmpty(serverId))
return cachedIdentities.Count;
return cachedIdentities.Count(c => c.Value.ServerId == serverId);
}
internal Dictionary<string, ServerIdentity> GetIdentsForAllExcept(string serverId)
{
return cachedIdentities.Where(k => k.Value.ServerId != serverId).ToDictionary(k => k.Key, k => k.Value);
}
internal Dictionary<string, ServerIdentity> GetIdentsForServer(string serverId)
{
return cachedIdentities.Where(k => k.Value.ServerId == serverId).ToDictionary(k => k.Key, k => k.Value);
}
internal void ClearIdentsForServer(string serverId)
{
var serverIdentities = cachedIdentities.Where(i => i.Value.ServerId == serverId);
foreach (var identity in serverIdentities)
{
cachedIdentities.TryRemove(identity.Key, out _);
}
}
internal void EnqueueIdentChange(IdentChange identchange)
{
_logger.LogInformation("Enqueued " + identchange.UidWithIdent.Uid.Uid + ":" + identchange.IsOnline + " from " + identchange.UidWithIdent.Ident.ServerId);
foreach (var k in identChanges.Keys)
{
if (string.Equals(k, identchange.UidWithIdent.Ident.ServerId, System.StringComparison.Ordinal)) continue;
identChanges[k].Enqueue(identchange);
}
}
internal bool DequeueIdentChange(string server, out IdentChange? cur)
{
if (!(identChanges.ContainsKey(server) && identChanges[server].TryDequeue(out cur)))
{
cur = null;
return false;
}
return true;
}
internal void RegisterServerForQueue(string serverId)
{
identChanges[serverId] = new ConcurrentQueue<IdentChange>();
}
internal record ServerIdentity
{
public string ServerId { get; set; } = string.Empty;
public string CharacterIdent { get; set; } = string.Empty;
}
}

View File

@@ -1,160 +0,0 @@
using Grpc.Core;
using MareSynchronosShared.Protos;
using Microsoft.Extensions.Logging;
using System.Threading.Tasks;
namespace MareSynchronosServices.Identity;
internal class IdentityService : IdentificationService.IdentificationServiceBase
{
private readonly ILogger<IdentityService> _logger;
private readonly IdentityHandler _handler;
public IdentityService(ILogger<IdentityService> logger, IdentityHandler handler)
{
_logger = logger;
_handler = handler;
}
public override async Task<CharacterIdentMessage> GetIdentForUid(UidMessage request, ServerCallContext context)
{
var result = await _handler.GetIdentForuid(request.Uid);
return new CharacterIdentMessage()
{
Ident = result.CharacterIdent,
ServerId = result.ServerId
};
}
public override Task<OnlineUserCountResponse> GetOnlineUserCount(ServerMessage request, ServerCallContext context)
{
return Task.FromResult(new OnlineUserCountResponse() { Count = _handler.GetOnlineUsers(request.ServerId) });
}
public override Task<Empty> ClearIdentsForServer(ServerMessage request, ServerCallContext context)
{
var idents = _handler.GetIdentsForServer(request.ServerId);
foreach (var entry in idents)
{
EnqueueIdentOffline(new UidWithIdent()
{
Ident = new CharacterIdentMessage()
{
Ident = entry.Value.CharacterIdent,
ServerId = entry.Value.ServerId
},
Uid = new UidMessage()
{
Uid = entry.Key
}
});
}
_handler.ClearIdentsForServer(request.ServerId);
return Task.FromResult(new Empty());
}
public override Task<Empty> RecreateServerIdents(ServerIdentMessage request, ServerCallContext context)
{
foreach (var identMsg in request.Idents)
{
_handler.SetIdent(identMsg.UidWithIdent.Uid.Uid, identMsg.UidWithIdent.Ident.ServerId, identMsg.UidWithIdent.Ident.Ident);
EnqueueIdentOnline(identMsg.UidWithIdent);
}
return Task.FromResult(new Empty());
}
public override async Task<Empty> SendStreamIdentStatusChange(IAsyncStreamReader<IdentChangeMessage> requestStream, ServerCallContext context)
{
await requestStream.MoveNext();
var server = requestStream.Current.Server;
if (server == null) throw new System.Exception("First message needs to be server message");
_handler.RegisterServerForQueue(server.ServerId);
_logger.LogInformation("Registered Server " + server.ServerId + " input stream");
while (await requestStream.MoveNext(context.CancellationToken).ConfigureAwait(false))
{
var cur = requestStream.Current.IdentChange;
if (cur == null) throw new System.Exception("Expected client ident change");
_handler.EnqueueIdentChange(cur);
if (cur.IsOnline)
{
_handler.SetIdent(cur.UidWithIdent.Uid.Uid, cur.UidWithIdent.Ident.ServerId, cur.UidWithIdent.Ident.Ident);
}
else
{
_handler.RemoveIdent(cur.UidWithIdent.Uid.Uid, cur.UidWithIdent.Ident.ServerId);
}
}
_logger.LogInformation("Server input stream from " + server + " finished");
return new Empty();
}
public override async Task ReceiveStreamIdentStatusChange(ServerMessage request, IServerStreamWriter<IdentChange> responseStream, ServerCallContext context)
{
var server = request.ServerId;
_logger.LogInformation("Registered Server " + server + " output stream");
try
{
while (!context.CancellationToken.IsCancellationRequested)
{
while (_handler.DequeueIdentChange(server, out var cur))
{
await responseStream.WriteAsync(cur).ConfigureAwait(false);
}
await Task.Delay(10).ConfigureAwait(false);
}
}
catch
{
_logger.LogInformation("Server output stream to " + server + " is faulty");
}
_logger.LogInformation("Server output stream to " + server + " is finished");
}
public override Task<UidWithIdentMessage> GetAllIdents(ServerMessage request, ServerCallContext context)
{
var response = new UidWithIdentMessage();
foreach (var item in _handler.GetIdentsForAllExcept(request.ServerId))
{
response.UidWithIdent.Add(new UidWithIdent()
{
Uid = new UidMessage()
{
Uid = item.Key
},
Ident = new CharacterIdentMessage()
{
Ident = item.Value.CharacterIdent,
ServerId = item.Value.ServerId
}
});
}
return Task.FromResult(response);
}
private void EnqueueIdentOnline(UidWithIdent ident)
{
_handler.EnqueueIdentChange(new IdentChange()
{
IsOnline = true,
UidWithIdent = ident
});
}
private void EnqueueIdentOffline(UidWithIdent ident)
{
_handler.EnqueueIdentChange(new IdentChange()
{
IsOnline = false,
UidWithIdent = ident
});
}
}

View File

@@ -2,6 +2,7 @@
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>

View File

@@ -1,13 +1,8 @@
using MareSynchronosServices;
using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System;
using System.Linq;
using MareSynchronosShared.Services;
using MareSynchronosShared.Utils;
public class Program
{
@@ -24,10 +19,13 @@ public class Program
metrics.SetGaugeTo(MetricsAPI.GaugeUsersRegistered, dbContext.Users.Count());
var options = host.Services.GetService<IOptions<ServicesConfiguration>>();
var options = host.Services.GetService<IConfigurationService<ServicesConfiguration>>();
var optionsServer = host.Services.GetService<IConfigurationService<ServerConfiguration>>();
var logger = host.Services.GetService<ILogger<Program>>();
logger.LogInformation("Loaded MareSynchronos Services Configuration");
logger.LogInformation(options.Value.ToString());
logger.LogInformation("Loaded MareSynchronos Services Configuration (IsMain: {isMain})", options.IsMain);
logger.LogInformation(options.ToString());
logger.LogInformation("Loaded MareSynchronos Server Configuration (IsMain: {isMain})", optionsServer.IsMain);
logger.LogInformation(optionsServer.ToString());
}
host.Run();

View File

@@ -1,23 +0,0 @@
using MareSynchronosShared.Utils;
using System.Text;
namespace MareSynchronosServices;
public class ServicesConfiguration : MareConfigurationBase
{
public string DiscordBotToken { get; set; } = string.Empty;
public bool PurgeUnusedAccounts { get; set; } = false;
public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14;
public int MaxExistingGroupsByUser { get; set; } = 3;
public override string ToString()
{
StringBuilder sb = new();
sb.AppendLine(base.ToString());
sb.AppendLine($"{nameof(DiscordBotToken)} => {DiscordBotToken}");
sb.AppendLine($"{nameof(PurgeUnusedAccounts)} => {PurgeUnusedAccounts}");
sb.AppendLine($"{nameof(PurgeUnusedAccountsPeriodInDays)} => {PurgeUnusedAccountsPeriodInDays}");
sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");
return sb.ToString();
}
}

View File

@@ -1,16 +1,12 @@
using MareSynchronosServices.Discord;
using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Prometheus;
using System.Collections.Generic;
using MareSynchronosServices.Identity;
using Microsoft.Extensions.Logging;
using MareSynchronosShared.Utils;
using Grpc.Net.Client.Configuration;
using MareSynchronosShared.Protos;
using MareSynchronosShared.Services;
namespace MareSynchronosServices;
@@ -25,6 +21,8 @@ public class Startup
public void ConfigureServices(IServiceCollection services)
{
var mareConfig = Configuration.GetSection("MareSynchronos");
services.AddDbContextPool<MareDbContext>(options =>
{
options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder =>
@@ -34,32 +32,58 @@ public class Startup
options.EnableThreadSafetyChecks(false);
}, Configuration.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
services.AddSingleton(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string> {
}, new List<string>
services.AddSingleton(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string> { },
new List<string>
{
MetricsAPI.GaugeUsersRegistered
}));
var noRetryConfig = new MethodConfig
{
Names = { MethodName.Default },
RetryPolicy = null
};
services.AddGrpcClient<IdentificationService.IdentificationServiceClient>(c =>
{
c.Address = new Uri(mareConfig.GetValue<string>(nameof(ServicesConfiguration.MainServerGrpcAddress)));
}).ConfigureChannel(c =>
{
c.ServiceConfig = new ServiceConfig { MethodConfigs = { noRetryConfig } };
c.HttpHandler = new SocketsHttpHandler()
{
EnableMultipleHttp2Connections = true
};
});
services.AddGrpcClient<ConfigurationService.ConfigurationServiceClient>(c =>
{
c.Address = new Uri(mareConfig.GetValue<string>(nameof(ServicesConfiguration.MainServerGrpcAddress)));
}).ConfigureChannel(c =>
{
c.ServiceConfig = new ServiceConfig { MethodConfigs = { noRetryConfig } };
c.HttpHandler = new SocketsHttpHandler()
{
EnableMultipleHttp2Connections = true
};
});
services.Configure<ServicesConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
services.Configure<ServerConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
services.Configure<MareConfigurationAuthBase>(Configuration.GetRequiredSection("MareSynchronos"));
services.AddSingleton(Configuration);
services.AddSingleton<DiscordBotServices>();
services.AddSingleton<IdentityHandler>();
services.AddSingleton<CleanupService>();
services.AddHostedService(provider => provider.GetService<CleanupService>());
services.AddHostedService<DiscordBot>();
services.AddGrpc();
services.AddSingleton<IConfigurationService<ServicesConfiguration>, MareConfigurationServiceServer<ServicesConfiguration>>();
services.AddSingleton<IConfigurationService<ServerConfiguration>, MareConfigurationServiceClient<ServerConfiguration>>();
services.AddSingleton<IConfigurationService<MareConfigurationAuthBase>, MareConfigurationServiceClient<MareConfigurationAuthBase>>();
}
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
app.UseRouting();
var config = app.ApplicationServices.GetRequiredService<IConfigurationService<MareConfigurationAuthBase>>();
var metricServer = new KestrelMetricServer(4982);
var metricServer = new KestrelMetricServer(config.GetValueOrDefault<int>(nameof(MareConfigurationBase.MetricsPort), 4982));
metricServer.Start();
app.UseEndpoints(endpoints =>
{
endpoints.MapGrpcService<IdentityService>();
});
}
}