This commit is contained in:
Stanley Dimant
2022-10-04 14:51:49 +02:00
56 changed files with 3759 additions and 1237 deletions

View File

@@ -2,6 +2,7 @@
using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Models;
using MareSynchronosShared.Utils;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
@@ -9,156 +10,180 @@ using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace MareSynchronosServices
namespace MareSynchronosServices;
public class CleanupService : IHostedService, IDisposable
{
public class CleanupService : IHostedService, IDisposable
private readonly MareMetrics metrics;
private readonly SecretKeyAuthenticationHandler _authService;
private readonly ILogger<CleanupService> _logger;
private readonly IServiceProvider _services;
private readonly IConfiguration _configuration;
private Timer? _timer;
public CleanupService(MareMetrics metrics, SecretKeyAuthenticationHandler authService, ILogger<CleanupService> logger, IServiceProvider services, IConfiguration configuration)
{
private readonly MareMetrics metrics;
private readonly SecretKeyAuthenticationHandler _authService;
private readonly ILogger<CleanupService> _logger;
private readonly IServiceProvider _services;
private readonly IConfiguration _configuration;
private Timer? _timer;
this.metrics = metrics;
_authService = authService;
_logger = logger;
_services = services;
_configuration = configuration.GetRequiredSection("MareSynchronos");
}
public CleanupService(MareMetrics metrics, SecretKeyAuthenticationHandler authService, ILogger<CleanupService> logger, IServiceProvider services, IConfiguration configuration)
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Cleanup Service started");
_timer = new Timer(CleanUp, null, TimeSpan.Zero, TimeSpan.FromMinutes(10));
return Task.CompletedTask;
}
private async void CleanUp(object state)
{
using var scope = _services.CreateScope();
using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
try
{
this.metrics = metrics;
_authService = authService;
_logger = logger;
_services = services;
_configuration = configuration.GetRequiredSection("MareSynchronos");
_logger.LogInformation($"Cleaning up expired lodestone authentications");
var lodestoneAuths = dbContext.LodeStoneAuth.Include(u => u.User).Where(a => a.StartedAt != null).ToList();
List<LodeStoneAuth> expiredAuths = new List<LodeStoneAuth>();
foreach (var auth in lodestoneAuths)
{
if (auth.StartedAt < DateTime.UtcNow - TimeSpan.FromMinutes(15))
{
expiredAuths.Add(auth);
}
}
dbContext.Users.RemoveRange(expiredAuths.Where(u => u.User != null).Select(a => a.User));
dbContext.RemoveRange(expiredAuths);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error during expired auths cleanup");
}
public Task StartAsync(CancellationToken cancellationToken)
try
{
_logger.LogInformation("Cleanup Service started");
if (!bool.TryParse(_configuration["PurgeUnusedAccounts"], out var purgeUnusedAccounts))
{
purgeUnusedAccounts = false;
}
_timer = new Timer(CleanUp, null, TimeSpan.Zero, TimeSpan.FromMinutes(10));
if (purgeUnusedAccounts)
{
if (!int.TryParse(_configuration["PurgeUnusedAccountsPeriodInDays"], out var usersOlderThanDays))
{
usersOlderThanDays = 14;
}
return Task.CompletedTask;
_logger.LogInformation("Cleaning up users older than {usersOlderThanDays} days", usersOlderThanDays);
var allUsers = dbContext.Users.Where(u => string.IsNullOrEmpty(u.Alias)).ToList();
List<User> usersToRemove = new();
foreach (var user in allUsers)
{
if (user.LastLoggedIn < (DateTime.UtcNow - TimeSpan.FromDays(usersOlderThanDays)))
{
_logger.LogInformation("User outdated: {userUID}", user.UID);
usersToRemove.Add(user);
}
}
foreach (var user in usersToRemove)
{
await PurgeUser(user, dbContext);
}
}
_logger.LogInformation("Cleaning up unauthorized users");
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error during user purge");
}
private void CleanUp(object state)
_authService.ClearUnauthorizedUsers();
_logger.LogInformation($"Cleanup complete");
dbContext.SaveChanges();
}
public async Task PurgeUser(User user, MareDbContext dbContext)
{
var lodestone = dbContext.LodeStoneAuth.SingleOrDefault(a => a.User.UID == user.UID);
if (lodestone != null)
{
using var scope = _services.CreateScope();
using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
dbContext.Remove(lodestone);
}
try
_authService.RemoveAuthentication(user.UID);
var auth = dbContext.Auth.Single(a => a.UserUID == user.UID);
var userFiles = dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == user.UID).ToList();
dbContext.Files.RemoveRange(userFiles);
var ownPairData = dbContext.ClientPairs.Where(u => u.User.UID == user.UID).ToList();
dbContext.RemoveRange(ownPairData);
var otherPairData = dbContext.ClientPairs.Include(u => u.User)
.Where(u => u.OtherUser.UID == user.UID).ToList();
var userGroupPairs = dbContext.GroupPairs.Include(g => g.Group).Where(u => u.GroupUserUID == user.UID);
foreach (var groupPair in userGroupPairs)
{
bool ownerHasLeft = string.Equals(groupPair.Group.OwnerUID, user.UID, StringComparison.Ordinal);
if (ownerHasLeft)
{
_logger.LogInformation($"Cleaning up expired lodestone authentications");
var lodestoneAuths = dbContext.LodeStoneAuth.Include(u => u.User).Where(a => a.StartedAt != null).ToList();
List<LodeStoneAuth> expiredAuths = new List<LodeStoneAuth>();
foreach (var auth in lodestoneAuths)
var groupPairs = await dbContext.GroupPairs.Where(g => g.GroupGID == groupPair.GroupGID).ToListAsync().ConfigureAwait(false);
if (!groupPairs.Any())
{
if (auth.StartedAt < DateTime.UtcNow - TimeSpan.FromMinutes(15))
{
expiredAuths.Add(auth);
}
_logger.LogInformation("Group {gid} has no new owner, deleting", groupPair.GroupGID);
dbContext.Remove(groupPair.Group);
}
dbContext.Users.RemoveRange(expiredAuths.Where(u => u.User != null).Select(a => a.User));
dbContext.RemoveRange(expiredAuths);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error during expired auths cleanup");
}
try
{
if (!bool.TryParse(_configuration["PurgeUnusedAccounts"], out var purgeUnusedAccounts))
else
{
purgeUnusedAccounts = false;
var groupHasMigrated = await SharedDbFunctions.MigrateOrDeleteGroup(dbContext, groupPair.Group, groupPairs, _configuration.GetValue<int>("MaxExistingGroupsByUser", 3)).ConfigureAwait(false);
continue;
}
if (purgeUnusedAccounts)
{
if (!int.TryParse(_configuration["PurgeUnusedAccountsPeriodInDays"], out var usersOlderThanDays))
{
usersOlderThanDays = 14;
}
_logger.LogInformation("Cleaning up users older than {usersOlderThanDays} days", usersOlderThanDays);
var allUsers = dbContext.Users.Where(u => string.IsNullOrEmpty(u.Alias)).ToList();
List<User> usersToRemove = new();
foreach (var user in allUsers)
{
if (user.LastLoggedIn < (DateTime.UtcNow - TimeSpan.FromDays(usersOlderThanDays)))
{
_logger.LogInformation("User outdated: {userUID}", user.UID);
usersToRemove.Add(user);
}
}
foreach (var user in usersToRemove)
{
PurgeUser(user, dbContext);
}
}
_logger.LogInformation("Cleaning up unauthorized users");
}
catch (Exception ex)
else
{
_logger.LogWarning(ex, "Error during user purge");
dbContext.Remove(groupPair);
}
_authService.ClearUnauthorizedUsers();
_logger.LogInformation($"Cleanup complete");
dbContext.SaveChanges();
}
public void PurgeUser(User user, MareDbContext dbContext)
{
var lodestone = dbContext.LodeStoneAuth.SingleOrDefault(a => a.User.UID == user.UID);
_logger.LogInformation("User purged: {uid}", user.UID);
if (lodestone != null)
{
dbContext.Remove(lodestone);
}
metrics.DecGauge(MetricsAPI.GaugeUsersRegistered, 1);
_authService.RemoveAuthentication(user.UID);
dbContext.RemoveRange(otherPairData);
dbContext.Remove(auth);
dbContext.Remove(user);
}
var auth = dbContext.Auth.Single(a => a.UserUID == user.UID);
public Task StopAsync(CancellationToken cancellationToken)
{
_timer?.Change(Timeout.Infinite, 0);
var userFiles = dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == user.UID).ToList();
dbContext.Files.RemoveRange(userFiles);
return Task.CompletedTask;
}
var ownPairData = dbContext.ClientPairs.Where(u => u.User.UID == user.UID).ToList();
dbContext.RemoveRange(ownPairData);
var otherPairData = dbContext.ClientPairs.Include(u => u.User)
.Where(u => u.OtherUser.UID == user.UID).ToList();
_logger.LogInformation("User purged: {uid}", user.UID);
metrics.DecGauge(MetricsAPI.GaugePairs, ownPairData.Count + otherPairData.Count);
metrics.DecGauge(MetricsAPI.GaugePairsPaused, ownPairData.Count(c => c.IsPaused) + otherPairData.Count(c => c.IsPaused));
metrics.DecGauge(MetricsAPI.GaugeUsersRegistered, 1);
dbContext.RemoveRange(otherPairData);
dbContext.Remove(auth);
dbContext.Remove(user);
}
public Task StopAsync(CancellationToken cancellationToken)
{
_timer?.Change(Timeout.Infinite, 0);
return Task.CompletedTask;
}
public void Dispose()
{
_timer?.Dispose();
}
public void Dispose()
{
_timer?.Dispose();
}
}

View File

@@ -9,6 +9,7 @@ using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Discord;
using Discord.Rest;
using Discord.WebSocket;
using MareSynchronosServices.Authentication;
using MareSynchronosShared.Data;
@@ -41,7 +42,9 @@ public class DiscordBot : IHostedService
private readonly string[] LodestoneServers = new[] { "eu", "na", "jp", "fr", "de" };
private readonly ConcurrentQueue<SocketSlashCommand> verificationQueue = new();
private ConcurrentDictionary<ulong, DateTime> LastVanityChange = new();
private ConcurrentDictionary<string, DateTime> LastVanityGidChange = new();
private ulong vanityCommandId;
private ulong vanityGidCommandId;
private Task cleanUpUserTask = null;
private SemaphoreSlim semaphore;
@@ -128,6 +131,18 @@ public class DiscordBot : IHostedService
eb = await HandleVanityUid(eb, arg.User.Id, newUid);
await arg.RespondAsync(embeds: new[] { eb.Build() }, ephemeral: true).ConfigureAwait(false);
break;
}
case "setsyncshellvanityid":
{
EmbedBuilder eb = new();
var oldGid = (string)arg.Data.Options.First(f => f.Name == "syncshell_id").Value;
var newGid = (string)arg.Data.Options.First(f => f.Name == "vanity_syncshell_id").Value;
eb = await HandleVanityGid(eb, arg.User.Id, oldGid, newGid);
await arg.RespondAsync(embeds: new[] { eb.Build() }, ephemeral: true).ConfigureAwait(false);
break;
}
default:
@@ -141,11 +156,75 @@ public class DiscordBot : IHostedService
}
}
private async Task<EmbedBuilder> HandleVanityUid(EmbedBuilder eb, ulong id, string newUid)
private async Task<EmbedBuilder> HandleVanityGid(EmbedBuilder eb, ulong id, string oldGid, string newGid)
{
if (LastVanityGidChange.TryGetValue(oldGid, out var lastChange))
{
var dateTimeDiff = DateTime.UtcNow.Subtract(lastChange);
if (dateTimeDiff.TotalHours < 24)
{
eb.WithTitle(("Failed to set Vanity Syncshell Id"));
eb.WithDescription(
$"You can only change the Vanity Syncshell Id once every 24h. Your last change is {dateTimeDiff} ago.");
}
}
Regex rgx = new(@"[_\-a-zA-Z0-9]{5,20}", RegexOptions.ECMAScript);
if (!rgx.Match(newGid).Success || newGid.Length < 5 || newGid.Length > 20)
{
eb.WithTitle("Failed to set Vanity Syncshell Id");
eb.WithDescription("The Vanity Syncshell Id must be between 5 and 20 characters and only contain letters A-Z, numbers 0-9 as well as - and _.");
return eb;
}
using var scope = services.CreateScope();
await using var db = scope.ServiceProvider.GetRequiredService<MareDbContext>();
var lodestoneUser = await db.LodeStoneAuth.Include(u => u.User).SingleOrDefaultAsync(u => u.DiscordId == id).ConfigureAwait(false);
if (lodestoneUser == null)
{
eb.WithTitle("Failed to set Vanity Syncshell Id");
eb.WithDescription("You do not have a registered account on this server.");
return eb;
}
var group = await db.Groups.FirstOrDefaultAsync(g => g.GID == oldGid || g.Alias == oldGid).ConfigureAwait(false);
if (group == null)
{
eb.WithTitle("Failed to set Vanity Syncshell Id");
eb.WithDescription("The provided Syncshell Id does not exist.");
return eb;
}
if (lodestoneUser.User.UID != group.OwnerUID)
{
eb.WithTitle("Failed to set Vanity Syncshell Id");
eb.WithDescription("You are not the owner of this Syncshell");
return eb;
}
var uidExists = await db.Groups.AnyAsync(u => u.GID == newGid || u.Alias == newGid).ConfigureAwait(false);
if (uidExists)
{
eb.WithTitle("Failed to set Vanity Syncshell Id");
eb.WithDescription("This Syncshell Id is already taken.");
return eb;
}
group.Alias = newGid;
db.Update(group);
await db.SaveChangesAsync();
LastVanityGidChange[newGid] = DateTime.UtcNow;
LastVanityGidChange[oldGid] = DateTime.UtcNow;
eb.WithTitle("Vanity Syncshell Id set");
eb.WithDescription("The Vanity Syncshell Id was set to **" + newGid + "**." + Environment.NewLine + "For those changes to apply you will have to reconnect to Mare.");
return eb;
}
private async Task<EmbedBuilder> HandleVanityUid(EmbedBuilder eb, ulong id, string newUid)
{
if (LastVanityChange.TryGetValue(id, out var lastChange))
{
var timeRemaining = DateTime.UtcNow.Subtract(lastChange);
@@ -157,14 +236,17 @@ public class DiscordBot : IHostedService
}
}
Regex rgx = new("[A-Z0-9]{10}", RegexOptions.ECMAScript);
if (!rgx.Match(newUid).Success || newUid.Length != 10)
Regex rgx = new(@"[_\-a-zA-Z0-9]{5,15}", RegexOptions.ECMAScript);
if (!rgx.Match(newUid).Success || newUid.Length < 5 || newUid.Length > 15)
{
eb.WithTitle("Failed to set Vanity UID");
eb.WithDescription("The Vanity UID must be between 5 and 15 characters and only contain letters A-Z, numbers 0-9, as well as - and _.");
return eb;
}
using var scope = services.CreateScope();
await using var db = scope.ServiceProvider.GetRequiredService<MareDbContext>();
var lodestoneUser = await db.LodeStoneAuth.Include("User").SingleOrDefaultAsync(u => u.DiscordId == id).ConfigureAwait(false);
if (lodestoneUser == null)
{
@@ -202,7 +284,7 @@ public class DiscordBot : IHostedService
{
if (discordAuthedUser.User != null)
{
cleanupService.PurgeUser(discordAuthedUser.User, db);
await cleanupService.PurgeUser(discordAuthedUser.User, db);
}
else
{
@@ -508,6 +590,12 @@ public class DiscordBot : IHostedService
vanityuid.WithDescription("Sets your Vanity UID.");
vanityuid.AddOption("vanity_uid", ApplicationCommandOptionType.String, "Desired Vanity UID", isRequired: true);
var vanitygid = new SlashCommandBuilder();
vanitygid.WithName("setsyncshellvanityid");
vanitygid.WithDescription("Sets a Vanity GID for a Syncshell");
vanitygid.AddOption("syncshell_id", ApplicationCommandOptionType.String, "Syncshell ID", isRequired: true);
vanitygid.AddOption("vanity_syncshell_id", ApplicationCommandOptionType.String, "Desired Vanity Syncshell ID", isRequired: true);
var recover = new SlashCommandBuilder();
recover.WithName("recover");
recover.WithDescription("Allows you to recover your account by generating a new secret key");
@@ -528,7 +616,6 @@ public class DiscordBot : IHostedService
}
if (!commands.Any(c => c.Name.Contains("setvanityuid")))
{
await guild.CreateApplicationCommandAsync(recover.Build()).ConfigureAwait(false);
var vanityCommand = await guild.CreateApplicationCommandAsync(vanityuid.Build()).ConfigureAwait(false);
vanityCommandId = vanityCommand.Id;
}
@@ -536,6 +623,10 @@ public class DiscordBot : IHostedService
{
vanityCommandId = commands.First(c => c.Name.Contains("setvanityuid")).Id;
}
if (!commands.Any(c => c.Name.Contains("setsyncshellvanityid")))
{
await guild.CreateApplicationCommandAsync(vanitygid.Build()).ConfigureAwait(false);
}
if (!commands.Any(c => c.Name.Contains("recover")))
{
await guild.CreateApplicationCommandAsync(recover.Build()).ConfigureAwait(false);
@@ -651,6 +742,8 @@ public class DiscordBot : IHostedService
{
var aliasedUsers = db.LodeStoneAuth.Include("User")
.Where(c => c.User != null && !string.IsNullOrEmpty(c.User.Alias));
var aliasedGroups = db.Groups.Include(u => u.Owner)
.Where(c => !string.IsNullOrEmpty(c.Alias));
foreach (var lodestoneAuth in aliasedUsers)
{
@@ -665,9 +758,30 @@ public class DiscordBot : IHostedService
}
await Task.Delay(100);
await db.SaveChangesAsync().ConfigureAwait(false);
}
await db.SaveChangesAsync().ConfigureAwait(false);
foreach (var group in aliasedGroups)
{
var lodestoneUser = await db.LodeStoneAuth.Include(u => u.User).SingleOrDefaultAsync(f => f.User.UID == group.OwnerUID);
RestGuildUser discordUser = null;
if (lodestoneUser != null)
{
discordUser = await restGuild.GetUserAsync(lodestoneUser.DiscordId).ConfigureAwait(false);
}
logger.LogInformation($"Checking Group: {group.GID}, owned by {lodestoneUser?.User?.UID ?? string.Empty} ({lodestoneUser?.User?.Alias ?? string.Empty}), User in Roles: {string.Join(", ", discordUser?.RoleIds ?? new List<ulong>())}");
if (lodestoneUser == null || discordUser == null || !discordUser.RoleIds.Any(u => allowedRoleIds.Contains(u)))
{
logger.LogInformation($"User {lodestoneUser.User.UID} not in allowed roles, deleting group alias");
group.Alias = null;
db.Update(group);
}
await Task.Delay(100);
await db.SaveChangesAsync().ConfigureAwait(false);
}
}
}
else
@@ -690,7 +804,7 @@ public class DiscordBot : IHostedService
updateStatusCts = new();
while (!updateStatusCts.IsCancellationRequested)
{
var onlineUsers = clientService.GetOnlineUsers();
var onlineUsers = await clientService.GetOnlineUsers();
logger.LogInformation("Users online: " + onlineUsers);
await discordClient.SetActivityAsync(new Game("Mare for " + onlineUsers + " Users")).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(15)).ConfigureAwait(false);

View File

@@ -5,38 +5,37 @@ using MareSynchronosShared.Protos;
using Microsoft.Extensions.Logging;
using System.Threading.Tasks;
namespace MareSynchronosServices.Services
namespace MareSynchronosServices.Services;
public class AuthenticationService : AuthService.AuthServiceBase
{
public class AuthenticationService : AuthService.AuthServiceBase
private readonly ILogger<AuthenticationService> _logger;
private readonly MareDbContext _dbContext;
private readonly SecretKeyAuthenticationHandler _authHandler;
public AuthenticationService(ILogger<AuthenticationService> logger, MareDbContext dbContext, SecretKeyAuthenticationHandler authHandler)
{
private readonly ILogger<AuthenticationService> _logger;
private readonly MareDbContext _dbContext;
private readonly SecretKeyAuthenticationHandler _authHandler;
_logger = logger;
_dbContext = dbContext;
_authHandler = authHandler;
}
public AuthenticationService(ILogger<AuthenticationService> logger, MareDbContext dbContext, SecretKeyAuthenticationHandler authHandler)
{
_logger = logger;
_dbContext = dbContext;
_authHandler = authHandler;
}
public override async Task<AuthReply> Authorize(AuthRequest request, ServerCallContext context)
{
return await _authHandler.AuthenticateAsync(_dbContext, request.Ip, request.SecretKey);
}
public override async Task<AuthReply> Authorize(AuthRequest request, ServerCallContext context)
{
return await _authHandler.AuthenticateAsync(_dbContext, request.Ip, request.SecretKey);
}
public override Task<Empty> RemoveAuth(RemoveAuthRequest request, ServerCallContext context)
{
_logger.LogInformation("Removing Authentication for {uid}", request.Uid);
_authHandler.RemoveAuthentication(request.Uid);
return Task.FromResult(new Empty());
}
public override Task<Empty> RemoveAuth(RemoveAuthRequest request, ServerCallContext context)
{
_logger.LogInformation("Removing Authentication for {uid}", request.Uid);
_authHandler.RemoveAuthentication(request.Uid);
return Task.FromResult(new Empty());
}
public override Task<Empty> ClearUnauthorized(Empty request, ServerCallContext context)
{
_logger.LogInformation("Clearing unauthorized users");
_authHandler.ClearUnauthorizedUsers();
return Task.FromResult(new Empty());
}
public override Task<Empty> ClearUnauthorized(Empty request, ServerCallContext context)
{
_logger.LogInformation("Clearing unauthorized users");
_authHandler.ClearUnauthorizedUsers();
return Task.FromResult(new Empty());
}
}