rework server responsibilities (#18)

* rework server responsibilities
add remote configuration

* start metrics only when compiled as not debug

* add some more logging to discord bot

* fixes of some casts

* make metrics port configurable, minor fixes

* add docker bullshit

* md formatting

* adjustments to docker stuff

* fix docker json files, fix some stuff in discord bot, add /useradd for Discord bot

* adjust docker configs and fix sharded.bat

* fixes for logs, cache file provider repeat trying to open filestream

Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>

.dockerignore

@@ -0,0 +1,284 @@
+# Created by https://www.gitignore.io/api/csharp
+
+### Csharp ###
+## Ignore Visual Studio temporary files, build results, and
+## files generated by popular Visual Studio add-ons.
+
+# User-specific files
+*.suo
+*.user
+*.userosscache
+*.sln.docstates
+
+# User-specific files (MonoDevelop/Xamarin Studio)
+*.userprefs
+
+# Build results
+[Dd]ebug/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+bld/
+[Bb]in/
+[Oo]bj/
+[Ll]og/
+
+# Visual Studio 2015 cache/options directory
+.vs/
+# Uncomment if you have tasks that create the project's static files in wwwroot
+#wwwroot/
+
+# MSTest test Results
+[Tt]est[Rr]esult*/
+[Bb]uild[Ll]og.*
+
+# NUNIT
+*.VisualState.xml
+TestResult.xml
+
+# Build Results of an ATL Project
+[Dd]ebugPS/
+[Rr]eleasePS/
+dlldata.c
+
+# DNX
+project.lock.json
+project.fragment.lock.json
+artifacts/
+Properties/launchSettings.json
+
+*_i.c
+*_p.c
+*_i.h
+*.ilk
+*.meta
+*.obj
+*.pch
+*.pdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp
+*.tmp_proj
+*.log
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.svclog
+*.scc
+
+# Chutzpah Test files
+_Chutzpah*
+
+# Visual C++ cache files
+ipch/
+*.aps
+*.ncb
+*.opendb
+*.opensdf
+*.sdf
+*.cachefile
+*.VC.db
+*.VC.VC.opendb
+
+# Visual Studio profiler
+*.psess
+*.vsp
+*.vspx
+*.sap
+
+# TFS 2012 Local Workspace
+$tf/
+
+# Guidance Automation Toolkit
+*.gpState
+
+# ReSharper is a .NET coding add-in
+_ReSharper*/
+*.[Rr]e[Ss]harper
+*.DotSettings.user
+
+# JustCode is a .NET coding add-in
+.JustCode
+
+# TeamCity is a build add-in
+_TeamCity*
+
+# DotCover is a Code Coverage Tool
+*.dotCover
+
+# Visual Studio code coverage results
+*.coverage
+*.coveragexml
+
+# NCrunch
+_NCrunch_*
+.*crunch*.local.xml
+nCrunchTemp_*
+
+# MightyMoose
+*.mm.*
+AutoTest.Net/
+
+# Web workbench (sass)
+.sass-cache/
+
+# Installshield output folder
+[Ee]xpress/
+
+# DocProject is a documentation generator add-in
+DocProject/buildhelp/
+DocProject/Help/*.HxT
+DocProject/Help/*.HxC
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/Html2
+DocProject/Help/html
+
+# Click-Once directory
+publish/
+
+# Publish Web Output
+*.[Pp]ublish.xml
+*.azurePubxml
+# TODO: Comment the next line if you want to checkin your web deploy settings
+# but database connection strings (with potential passwords) will be unencrypted
+*.pubxml
+*.publishproj
+
+# Microsoft Azure Web App publish settings. Comment the next line if you want to
+# checkin your Azure Web App publish settings, but sensitive information contained
+# in these scripts will be unencrypted
+PublishScripts/
+
+# NuGet Packages
+*.nupkg
+# The packages folder can be ignored because of Package Restore
+**/packages/*
+# except build/, which is used as an MSBuild target.
+!**/packages/build/
+# Uncomment if necessary however generally it will be regenerated when needed
+#!**/packages/repositories.config
+# NuGet v3's project.json files produces more ignoreable files
+*.nuget.props
+*.nuget.targets
+
+# Microsoft Azure Build Output
+csx/
+*.build.csdef
+
+# Microsoft Azure Emulator
+ecf/
+rcf/
+
+# Windows Store app package directories and files
+AppPackages/
+BundleArtifacts/
+Package.StoreAssociation.xml
+_pkginfo.txt
+
+# Visual Studio cache files
+# files ending in .cache can be ignored
+*.[Cc]ache
+# but keep track of directories ending in .cache
+!*.[Cc]ache/
+
+# Others
+ClientBin/
+~$*
+*~
+*.dbmdl
+*.dbproj.schemaview
+*.jfm
+*.pfx
+*.publishsettings
+node_modules/
+orleans.codegen.cs
+
+# Since there are multiple workflows, uncomment next line to ignore bower_components
+# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
+#bower_components/
+
+# RIA/Silverlight projects
+Generated_Code/
+
+# Backup & report files from converting an old project file
+# to a newer Visual Studio version. Backup files are not needed,
+# because we have git ;-)
+_UpgradeReport_Files/
+Backup*/
+UpgradeLog*.XML
+UpgradeLog*.htm
+
+# SQL Server files
+*.mdf
+*.ldf
+
+# Business Intelligence projects
+*.rdl.data
+*.bim.layout
+*.bim_*.settings
+
+# Microsoft Fakes
+FakesAssemblies/
+
+# GhostDoc plugin setting file
+*.GhostDoc.xml
+
+# Node.js Tools for Visual Studio
+.ntvs_analysis.dat
+
+# Visual Studio 6 build log
+*.plg
+
+# Visual Studio 6 workspace options file
+*.opt
+
+# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
+*.vbw
+
+# Visual Studio LightSwitch build output
+**/*.HTMLClient/GeneratedArtifacts
+**/*.DesktopClient/GeneratedArtifacts
+**/*.DesktopClient/ModelManifest.xml
+**/*.Server/GeneratedArtifacts
+**/*.Server/ModelManifest.xml
+_Pvt_Extensions
+
+# Paket dependency manager
+.paket/paket.exe
+paket-files/
+
+# FAKE - F# Make
+.fake/
+
+# JetBrains Rider
+.idea/
+*.sln.iml
+
+# CodeRush
+.cr/
+
+# Python Tools for Visual Studio (PTVS)
+__pycache__/
+*.pyc
+
+# Cake - Uncomment if you are using it
+# tools/
+tools/Cake.CoreCLR
+.vscode
+tools
+.dotnet
+Dockerfile
+
+# .env file contains default environment variables for docker
+.env
+.git/

.gitignore

@@ -348,3 +348,6 @@ MigrationBackup/
 
 # Ionide (cross platform F# VS Code tools) working folder
 .ionide/
+
+# docker run data
+Docker/run/data/

Docker/Readme.md

@@ -0,0 +1,38 @@
+# Mare Synchronos Docker Setup
+This is primarily aimed at developers who want to spin up their own local server for development purposes without having to spin up a VM.
+Obligatory requires Docker to be installed on the machine.
+
+There are two directories: `build` and `run`
+
+## 1. build images
+There is two ways to build the necessary docker images which are differentiated by the folders `-local` and `-git`  
+- -local will run the image build against the current locally present sources  
+- -git will run the image build against the latest git main commit  
+It is possible to build all required images at once by running `docker-build.bat/sh` (Server, Servies, StaticFilesServer) or all 3 separately with `docker-build-<whatever>.bat/sh`
+
+## 2. Configure ports + token
+You should set up 2 environment variables that hold server specific configuration and open up ports.  
+The default ports used through the provided configuration are `6000` for the main server and `6200` for the files downloads.  
+Both ports should be open to your computer through your router if you wish to test this with clients.
+
+Furthermore there are two environment variables `DEV_MARE_CDNURL` and `DEV_MARE_DISCORDTOKEN` which you are required to set.  
+`DEV_MARE_CDNURL` should point to `http://<yourip or dyndns>:6200/cache/` and `DEV_MARE_DISCORDTOKEN` is an oauth token from a bot you need to create through the Discord bot portal.  
+It is enough to set them as User variables. The compose files refer to those environment variables to overwrite configuration settings for the Server and Services to set those respective values.  
+It is also possible to set those values in the configuration.json files themselves.  
+Without a valid Discord bot you will not be able to register accounts without fumbling around in the PostgreSQL database.
+
+## 3. Run Mare Server
+The run folder contains two major Mare configurations which is `standalone` and `sharded`.  
+Both configurations default to port `6000` for the main server connection and `6200` for the files downloads. No HTTPS.  
+All `appsettings.json` configurations provided are extensive at the point of writing, note the differences between the shard configurations and the main servers respectively.  
+They can be used as examples if you want to spin up your own servers otherwise.
+
+The scripts to start the respective services are divided by name, the `daemon-start/stop` files use `compose up -d` to run it in the background and to be able to stop the containers as well.  
+The respective docker-compose files lie in the `compose` folder. I would not recommend editing them unless you know what you are doing.  
+All data (postgresql and files uploads) will be thrown into the `data` folder after startup.  
+All logs from the mare services will be thrown into `logs`, divided by shard, where applicable.
+
+The `standalone` configuration features PostgeSQL, Mare Server, Mare StaticFilesServer and Mare Services.  
+The `sharded` configuration features PostgreSQL, Redis, HAProxy, Mare Server Main, 2 Mare Server Shards, Mare Services, Mare StaticFilesServer Main and 2 Mare StaticFilesServer Shards.  
+Haproxy is set up that it takes the same ports as the `standalone` configuration and distributes the connections between the shards.  
+In theory it should be possible to switch between the `standalone` and `sharded` configuration by shutting down one composition container and starting up the other. They share the same Database.

Docker/build/Dockerfile-MareSynchronosServer

@@ -0,0 +1,31 @@
+FROM mcr.microsoft.com/dotnet/sdk:7.0 as BUILD
+
+COPY MareAPI /server/MareAPI
+COPY MareSynchronosServer/MareSynchronosShared /server/MareSynchronosServer/MareSynchronosShared
+COPY MareSynchronosServer/MareSynchronosServer /server/MareSynchronosServer/MareSynchronosServer
+
+WORKDIR /server/MareSynchronosServer/MareSynchronosServer/
+
+RUN dotnet publish \
+        --configuration=Release \
+        --os=linux \
+        --output=/build \
+        MareSynchronosServer.csproj
+
+FROM mcr.microsoft.com/dotnet/aspnet:7.0
+
+RUN adduser \
+        --disabled-password \
+        --group \
+        --no-create-home \
+        --quiet \
+        --system \
+        mare
+
+COPY --from=BUILD /build /opt/MareSynchronosServer
+RUN chown -R mare:mare /opt/MareSynchronosServer
+
+USER mare:mare
+WORKDIR /opt/MareSynchronosServer
+
+CMD ["./MareSynchronosServer"]

Docker/build/Dockerfile-MareSynchronosServer-git

@@ -0,0 +1,29 @@
+FROM mcr.microsoft.com/dotnet/sdk:7.0 as BUILD
+
+RUN git clone --recurse-submodules https://github.com/Penumbra-Sync/server
+
+WORKDIR /server/MareSynchronosServer/MareSynchronosServer/
+
+RUN dotnet publish \
+        --configuration=Release \
+        --os=linux \
+        --output=/MareSynchronosServer \
+        MareSynchronosServer.csproj
+
+FROM mcr.microsoft.com/dotnet/aspnet:7.0
+
+RUN adduser \
+        --disabled-password \
+        --group \
+        --no-create-home \
+        --quiet \
+        --system \
+        mare
+
+COPY --from=BUILD /MareSynchronosServer /opt/MareSynchronosServer
+RUN chown -R mare:mare /opt/MareSynchronosServer
+
+USER mare:mare
+WORKDIR /opt/MareSynchronosServer
+
+CMD ["./MareSynchronosServer"]

Docker/build/Dockerfile-MareSynchronosServices

@@ -0,0 +1,31 @@
+FROM mcr.microsoft.com/dotnet/sdk:7.0 as BUILD
+
+COPY MareAPI /server/MareAPI
+COPY MareSynchronosServer/MareSynchronosShared /server/MareSynchronosServer/MareSynchronosShared
+COPY MareSynchronosServer/MareSynchronosServices /server/MareSynchronosServer/MareSynchronosServices
+
+WORKDIR /server/MareSynchronosServer/MareSynchronosServices/
+
+RUN dotnet publish \
+        --configuration=Release \
+        --os=linux \
+        --output=/build \
+        MareSynchronosServices.csproj
+
+FROM mcr.microsoft.com/dotnet/aspnet:7.0
+
+RUN adduser \
+        --disabled-password \
+        --group \
+        --no-create-home \
+        --quiet \
+        --system \
+        mare
+
+COPY --from=BUILD /build /opt/MareSynchronosServices
+RUN chown -R mare:mare /opt/MareSynchronosServices
+
+USER mare:mare
+WORKDIR /opt/MareSynchronosServices
+
+CMD ["./MareSynchronosServices"]

Docker/build/Dockerfile-MareSynchronosServices-git

@@ -0,0 +1,29 @@
+FROM mcr.microsoft.com/dotnet/sdk:7.0 as BUILD
+
+RUN git clone --recurse-submodules https://github.com/Penumbra-Sync/server
+
+WORKDIR /server/MareSynchronosServer/MareSynchronosServices/
+
+RUN dotnet publish \
+        --configuration=Release \
+        --os=linux \
+        --output=/MareSynchronosServices \
+        MareSynchronosServices.csproj
+
+FROM mcr.microsoft.com/dotnet/aspnet:7.0
+
+RUN adduser \
+        --disabled-password \
+        --group \
+        --no-create-home \
+        --quiet \
+        --system \
+        mare
+
+COPY --from=BUILD /MareSynchronosServices /opt/MareSynchronosServices
+RUN chown -R mare:mare /opt/MareSynchronosServices
+
+USER mare:mare
+WORKDIR /opt/MareSynchronosServices
+
+CMD ["./MareSynchronosServices"]

Docker/build/Dockerfile-MareSynchronosStaticFilesServer

@@ -0,0 +1,31 @@
+FROM mcr.microsoft.com/dotnet/sdk:7.0 as BUILD
+
+COPY MareAPI /server/MareAPI
+COPY MareSynchronosServer/MareSynchronosShared /server/MareSynchronosServer/MareSynchronosShared
+COPY MareSynchronosServer/MareSynchronosStaticFilesServer /server/MareSynchronosServer/MareSynchronosStaticFilesServer
+
+WORKDIR /server/MareSynchronosServer/MareSynchronosStaticFilesServer/
+
+RUN dotnet publish \
+        --configuration=Release \
+        --os=linux \
+        --output=/build \
+        MareSynchronosStaticFilesServer.csproj
+
+FROM mcr.microsoft.com/dotnet/aspnet:7.0
+
+RUN adduser \
+        --disabled-password \
+        --group \
+        --no-create-home \
+        --quiet \
+        --system \
+        mare
+
+COPY --from=BUILD /build /opt/MareSynchronosStaticFilesServer
+RUN chown -R mare:mare /opt/MareSynchronosStaticFilesServer
+
+USER mare:mare
+WORKDIR /opt/MareSynchronosStaticFilesServer
+
+CMD ["./MareSynchronosStaticFilesServer"]

Docker/build/Dockerfile-MareSynchronosStaticFilesServer-git

@@ -0,0 +1,29 @@
+FROM mcr.microsoft.com/dotnet/sdk:7.0 as BUILD
+
+RUN git clone --recurse-submodules https://github.com/Penumbra-Sync/server
+
+WORKDIR /server/MareSynchronosServer/MareSynchronosStaticFilesServer/
+
+RUN dotnet publish \
+        --configuration=Release \
+        --os=linux \
+        --output=/MareSynchronosStaticFilesServer \
+        MareSynchronosStaticFilesServer.csproj
+
+FROM mcr.microsoft.com/dotnet/aspnet:7.0
+
+RUN adduser \
+        --disabled-password \
+        --group \
+        --no-create-home \
+        --quiet \
+        --system \
+        mare
+
+COPY --from=BUILD /MareSynchronosStaticFilesServer /opt/MareSynchronosStaticFilesServer
+RUN chown -R mare:mare /opt/MareSynchronosStaticFilesServer
+
+USER mare:mare
+WORKDIR /opt/MareSynchronosStaticFilesServer
+
+CMD ["./MareSynchronosStaticFilesServer"]

Docker/build/linux-git/docker-build-server.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+docker build -t darkarchon/mare-synchronos-server:latest . -f ../Dockerfile-MareSynchronosServer-git --no-cache --pull --force-rm

Docker/build/linux-git/docker-build-services.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+docker build -t darkarchon/mare-synchronos-services:latest . -f ../Dockerfile-MareSynchronosServices-git --no-cache --pull --force-rm

Docker/build/linux-git/docker-build-staticfilesserver.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+docker build -t darkarchon/mare-synchronos-staticfilesserver:latest . -f ../Dockerfile-MareSynchronosStaticFilesServer-git --no-cache --pull --force-rm

Docker/build/linux-git/docker-build.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+./docker-build-server.sh
+./docker-build-services.sh
+./docker-build-staticfilesserver.sh

Docker/build/linux-local/docker-build-server.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+cd ../../../
+docker build -t darkarchon/mare-synchronos-server:latest . -f ../Dockerfile-MareSynchronosServer --no-cache --pull --force-rm
+cd Docker/build/linux-local

Docker/build/linux-local/docker-build-services.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+cd ../../../
+docker build -t darkarchon/mare-synchronos-services:latest . -f ../Dockerfile-MareSynchronosServices --no-cache --pull --force-rm
+cd Docker/build/linux-local

Docker/build/linux-local/docker-build-staticfilesserver.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+cd ../../../
+docker build -t darkarchon/mare-synchronos-staticfilesserver:latest . -f ../Dockerfile-MareSynchronosStaticFilesServer --no-cache --pull --force-rm
+cd Docker/build/linux-local

Docker/build/linux-local/docker-build.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+./docker-build-server.sh
+./docker-build-services.sh
+./docker-build-staticfilesserver.sh

Docker/build/windows-git/docker-build-server.bat

@@ -0,0 +1,2 @@
+@echo off
+docker build -t darkarchon/mare-synchronos-server:latest . -f ..\Dockerfile-MareSynchronosServer-git --no-cache --pull --force-rm

Docker/build/windows-git/docker-build-services.bat

@@ -0,0 +1,3 @@
+@echo off
+
+docker build -t darkarchon/mare-synchronos-services:latest . -f ..\Dockerfile-MareSynchronosServices-git --no-cache --pull --force-rm

Docker/build/windows-git/docker-build-staticfilesserver.bat

@@ -0,0 +1,3 @@
+@echo off
+
+docker build -t darkarchon/mare-synchronos-staticfilesserver:latest . -f ..\Dockerfile-MareSynchronosStaticFilesServer-git --no-cache --pull --force-rm

Docker/build/windows-git/docker-build.bat

@@ -0,0 +1,5 @@
+@echo off
+
+call docker-build-server.bat
+call docker-build-services.bat
+call docker-build-staticfilesserver.bat

Docker/build/windows-local/docker-build-server.bat

@@ -0,0 +1,4 @@
+@echo off
+cd ..\..\..\
+docker build -t darkarchon/mare-synchronos-server:latest . -f Docker\build\Dockerfile-MareSynchronosServer --no-cache --pull --force-rm
+cd Docker\build\windows-local

Docker/build/windows-local/docker-build-services.bat

@@ -0,0 +1,4 @@
+@echo off
+cd ..\..\..\
+docker build -t darkarchon/mare-synchronos-services:latest . -f Docker\build\Dockerfile-MareSynchronosServices --no-cache --pull --force-rm
+cd Docker\build\windows-local

Docker/build/windows-local/docker-build-staticfilesserver.bat

@@ -0,0 +1,4 @@
+@echo off
+cd ..\..\..\
+docker build -t darkarchon/mare-synchronos-staticfilesserver:latest . -f Docker\build\Dockerfile-MareSynchronosStaticFilesServer --no-cache --pull --force-rm
+cd Docker\build\windows-local

Docker/build/windows-local/docker-build.bat

@@ -0,0 +1,5 @@
+@echo off
+
+call docker-build-server.bat
+call docker-build-services.bat
+call docker-build-staticfilesserver.bat

Docker/run/compose/mare-sharded.yml

@@ -0,0 +1,114 @@
+services:
+  postgres:
+    image: postgres:latest
+    restart: always
+    environment:
+      POSTGRES_DB: mare
+      POSTGRES_USER: mare
+      POSTGRES_PASSWORD: secretdevpassword
+    volumes:
+      - ../data/postgresql/:/var/lib/postgresql/data
+      - postgres_socket:/var/run/postgresql:rw
+
+  haproxy:
+    image: haproxy:latest
+    restart: always
+    ports: 
+      - 6000:6000/tcp
+      - 6200:6200/tcp
+    volumes:
+      - ../config/sharded/haproxy-shards.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
+
+  redis:
+    image: redis:latest
+    command: [sh, -c, "rm -f /data/dump.rdb && redis-server --save \"\" --appendonly no --requirepass secretredispassword"]
+    volumes:
+      - cache:/data
+
+  mare-server:
+    image: darkarchon/mare-synchronos-server:latest
+    restart: on-failure
+    environment:
+      MareSynchronos__CdnFullUrl: "${DEV_MARE_CDNURL}"
+    volumes:
+      - ../config/sharded/server-shard-main.json:/opt/MareSynchronosServer/appsettings.json
+      - ../log/server-shard-main/:/opt/MareSynchronosServer/logs/:rw
+      - postgres_socket:/var/run/postgresql/:rw
+    depends_on:
+      - "postgres"
+
+  mare-shard-1:
+    image: darkarchon/mare-synchronos-server:latest
+    restart: on-failure
+    volumes:
+      - ../config/sharded/server-shard-1.json:/opt/MareSynchronosServer/appsettings.json
+      - ../log/server-shard-1/:/opt/MareSynchronosServer/logs/:rw
+      - postgres_socket:/var/run/postgresql/:rw
+    depends_on:
+      - "postgres"
+      - "mare-server"
+
+  mare-shard-2:
+    image: darkarchon/mare-synchronos-server:latest
+    restart: on-failure
+    volumes:
+      - ../config/sharded/server-shard-2.json:/opt/MareSynchronosServer/appsettings.json
+      - ../log/server-shard-2/:/opt/MareSynchronosServer/logs/:rw
+      - postgres_socket:/var/run/postgresql/:rw
+    depends_on:
+      - "postgres"
+      - "mare-server"
+
+  mare-services:
+    image: darkarchon/mare-synchronos-services:latest
+    restart: on-failure
+    environment:
+      MareSynchronos__DiscordBotToken: "${DEV_MARE_DISCORDTOKEN}"
+    volumes:
+      - ../config/standalone/services-standalone.json:/opt/MareSynchronosServices/appsettings.json
+      - ../log/services-standalone/:/opt/MareSynchronosServices/logs/:rw
+      - postgres_socket:/var/run/postgresql/:rw
+    depends_on:
+      - "postgres"
+      - "mare-server"
+
+  mare-files:
+    image: darkarchon/mare-synchronos-staticfilesserver:latest
+    restart: on-failure
+    volumes:
+      - ../config/sharded/files-shard-main.json:/opt/MareSynchronosStaticFilesServer/appsettings.json
+      - ../log/files-standalone/:/opt/MareSynchronosStaticFilesServer/logs/:rw
+      - postgres_socket:/var/run/postgresql/:rw
+      - ../data/files-shard-main/:/marecache/:rw
+    depends_on:
+      - "postgres"
+      - "mare-server"
+
+  mare-files-shard-1:
+    image: darkarchon/mare-synchronos-staticfilesserver:latest
+    restart: on-failure
+    volumes:
+      - ../config/sharded/files-shard-1.json:/opt/MareSynchronosStaticFilesServer/appsettings.json
+      - ../log/files-shard-1/:/opt/MareSynchronosStaticFilesServer/logs/:rw
+      - postgres_socket:/var/run/postgresql/:rw
+      - ../data/files-shard-1/:/marecache/:rw
+    depends_on:
+      - "postgres"
+      - "mare-files"
+
+  mare-files-shard-2:
+    image: darkarchon/mare-synchronos-staticfilesserver:latest
+    restart: on-failure
+    volumes:
+      - ../config/sharded/files-shard-2.json:/opt/MareSynchronosStaticFilesServer/appsettings.json
+      - ../log/files-shard-2/:/opt/MareSynchronosStaticFilesServer/logs/:rw
+      - postgres_socket:/var/run/postgresql/:rw
+      - ../data/files-shard-2/:/marecache/:rw
+    depends_on:
+      - "postgres"
+      - "mare-files"
+
+volumes:
+  cache:
+    driver: local
+  postgres_socket:

Docker/run/compose/mare-standalone.yml

@@ -0,0 +1,55 @@
+services:
+  postgres:
+    image: postgres:latest
+    restart: always
+    environment:
+      POSTGRES_DB: mare
+      POSTGRES_USER: mare
+      POSTGRES_PASSWORD: secretdevpassword
+    volumes:
+      - ../data/postgresql/:/var/lib/postgresql/data
+      - postgres_socket:/var/run/postgresql:rw
+
+  mare-server:
+    image: darkarchon/mare-synchronos-server:latest
+    restart: on-failure
+    ports:
+      - 6000:6000/tcp
+    environment:
+      MareSynchronos__CdnFullUrl: "${DEV_MARE_CDNURL}"
+    volumes:
+      - ../config/standalone/server-standalone.json:/opt/MareSynchronosServer/appsettings.json
+      - ../log/server-standalone/:/opt/MareSynchronosServer/logs/:rw
+      - postgres_socket:/var/run/postgresql/:rw
+    depends_on:
+      - "postgres"
+
+  mare-services:
+    image: darkarchon/mare-synchronos-services:latest
+    restart: on-failure
+    environment:
+      MareSynchronos__DiscordBotToken: "${DEV_MARE_DISCORDTOKEN}"
+    volumes:
+      - ../config/standalone/services-standalone.json:/opt/MareSynchronosServices/appsettings.json
+      - ../log/services-standalone/:/opt/MareSynchronosServices/logs/:rw
+      - postgres_socket:/var/run/postgresql/:rw
+    depends_on:
+      - "postgres"
+      - "mare-server"
+
+  mare-files:
+    image: darkarchon/mare-synchronos-staticfilesserver:latest
+    ports:
+      - 6200:6200/tcp
+    restart: on-failure
+    volumes:
+      - ../config/standalone/files-standalone.json:/opt/MareSynchronosStaticFilesServer/appsettings.json
+      - ../log/files-standalone/:/opt/MareSynchronosStaticFilesServer/logs/:rw
+      - postgres_socket:/var/run/postgresql/:rw
+      - ../data/files-standalone/:/marecache/:rw
+    depends_on:
+      - "postgres"
+      - "mare-server"
+
+volumes:
+  postgres_socket:

Docker/run/config/sharded/files-shard-1.json

@@ -0,0 +1,49 @@
+{
+    "ConnectionStrings": {
+        "DefaultConnection": "Host=/var/run/postgresql;Port=5432;Database=mare;Username=mare;Keepalive=15;Minimum Pool Size=10;Maximum Pool Size=50;No Reset On Close=true;Max Auto Prepare=50;Enlist=false"
+    },
+    "Logging": {
+        "LogLevel": {
+            "Default": "Warning",
+            "Microsoft": "Warning",
+            "Microsoft.Hosting.Lifetime": "Information",
+            "MareSynchronosStaticFilesServer": "Information",
+            "MareSynchronosShared": "Information",
+            "System.IO": "Information"
+        },
+        "File": {
+            "BasePath": "logs",
+            "FileAccessMode": "KeepOpenAndAutoFlush",
+            "FileEncodingName": "utf-8",
+            "DateFormat": "yyyMMdd",
+            "MaxFileSize": 104857600,
+            "Files": [
+                {
+                    "Path": "<date:yyyy>/<date:MM>/<date:dd>/mare-<date:HH>-<counter:0000>.log"
+                }
+            ]
+        }
+    },
+    "MareSynchronos": {
+        "DbContextPoolSize": 512,
+        "ShardName": "Files Shard 1",
+        "MetricsPort": 6250,
+        "FileServerGrpcAddress": "http://mare-files:6205",
+        "ForcedDeletionOfFilesAfterHours": 2,
+        "CacheSizeHardLimitInGiB": 5,
+        "UnusedFileRetentionPeriodInDays": 14,
+        "CacheDirectory": "/marecache/",
+        "RemoteCacheSourceUri": "http://mare-files:6200/cache/",
+        "MainServerGrpcAddress": "http://mare-server:6005"
+    },
+    "AllowedHosts": "*",
+    "Kestrel": {
+        "Endpoints": {
+            "Http": {
+                "Url": "http://+:6200"
+            }
+        }
+    },
+    "IpRateLimiting": {},
+    "IPRateLimitPolicies": {}
+}

Docker/run/config/sharded/files-shard-2.json

@@ -0,0 +1,49 @@
+{
+    "ConnectionStrings": {
+        "DefaultConnection": "Host=/var/run/postgresql;Port=5432;Database=mare;Username=mare;Keepalive=15;Minimum Pool Size=10;Maximum Pool Size=50;No Reset On Close=true;Max Auto Prepare=50;Enlist=false"
+    },
+    "Logging": {
+        "LogLevel": {
+            "Default": "Warning",
+            "Microsoft": "Warning",
+            "Microsoft.Hosting.Lifetime": "Information",
+            "MareSynchronosStaticFilesServer": "Information",
+            "MareSynchronosShared": "Information",
+            "System.IO": "Information"
+        },
+        "File": {
+            "BasePath": "logs",
+            "FileAccessMode": "KeepOpenAndAutoFlush",
+            "FileEncodingName": "utf-8",
+            "DateFormat": "yyyMMdd",
+            "MaxFileSize": 104857600,
+            "Files": [
+                {
+                    "Path": "<date:yyyy>/<date:MM>/<date:dd>/mare-<date:HH>-<counter:0000>.log"
+                }
+            ]
+        }
+    },
+    "MareSynchronos": {
+        "DbContextPoolSize": 512,
+        "ShardName": "Files Shard 2",
+        "MetricsPort": 6250,
+        "FileServerGrpcAddress": "http://mare-files:6205",
+        "ForcedDeletionOfFilesAfterHours": 2,
+        "CacheSizeHardLimitInGiB": 5,
+        "UnusedFileRetentionPeriodInDays": 14,
+        "CacheDirectory": "/marecache/",
+        "RemoteCacheSourceUri": "http://mare-files:6200/cache/",
+        "MainServerGrpcAddress": "http://mare-server:6005"
+    },
+    "AllowedHosts": "*",
+    "Kestrel": {
+        "Endpoints": {
+            "Http": {
+                "Url": "http://+:6200"
+            }
+        }
+    },
+    "IpRateLimiting": {},
+    "IPRateLimitPolicies": {}
+}

Docker/run/config/sharded/files-shard-main.json

@@ -0,0 +1,53 @@
+{
+    "ConnectionStrings": {
+        "DefaultConnection": "Host=/var/run/postgresql;Port=5432;Database=mare;Username=mare;Keepalive=15;Minimum Pool Size=10;Maximum Pool Size=50;No Reset On Close=true;Max Auto Prepare=50;Enlist=false"
+    },
+    "Logging": {
+        "LogLevel": {
+            "Default": "Warning",
+            "Microsoft": "Warning",
+            "Microsoft.Hosting.Lifetime": "Information",
+            "MareSynchronosStaticFilesServer": "Information",
+            "MareSynchronosShared": "Information",
+            "System.IO": "Information"
+        },
+        "File": {
+            "BasePath": "logs",
+            "FileAccessMode": "KeepOpenAndAutoFlush",
+            "FileEncodingName": "utf-8",
+            "DateFormat": "yyyMMdd",
+            "MaxFileSize": 104857600,
+            "Files": [
+                {
+                    "Path": "<date:yyyy>/<date:MM>/<date:dd>/mare-<date:HH>-<counter:0000>.log"
+                }
+            ]
+        }
+    },
+    "MareSynchronos": {
+        "DbContextPoolSize": 512,
+        "ShardName": "Files",
+        "MetricsPort": 6250,
+        "FileServerGrpcAddress": "",
+        "ForcedDeletionOfFilesAfterHours": -1,
+        "CacheSizeHardLimitInGiB": -1,
+        "UnusedFileRetentionPeriodInDays": 14,
+        "CacheDirectory": "/marecache/",
+        "RemoteCacheSourceUri": "",
+        "MainServerGrpcAddress": "http://mare-server:6005"
+    },
+    "AllowedHosts": "*",
+    "Kestrel": {
+        "Endpoints": {
+            "Http": {
+                "Url": "http://+:6200"
+            },
+            "Grpc": {
+                "Protocols": "Http2",
+                "Url": "http://+:6205"
+            }
+        }
+    },
+    "IpRateLimiting": {},
+    "IPRateLimitPolicies": {}
+}

Docker/run/config/sharded/haproxy-shards.cfg

@@ -0,0 +1,39 @@
+global
+        log /dev/log    local0
+        log /dev/log    local1 notice
+        daemon
+
+        ca-base /etc/ssl/certs
+        crt-base /etc/ssl/private
+
+        ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+        ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
+        ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
+
+defaults
+        log     global
+        mode    http
+        option  httplog
+        option  dontlognull
+        timeout connect 5000
+        timeout client  50000
+        timeout server  50000
+
+frontend mare
+        bind :6000
+        default_backend mare-servers
+
+frontend mare-files
+        bind :6200
+        default_backend mare-files
+
+backend mare-servers
+        balance leastconn
+        cookie SERVER insert indirect nocache
+        server mare1 mare-shard-1:6000 cookie mare1
+        server mare2 mare-shard-2:6000 cookie mare2
+
+backend mare-files
+        balance roundrobin
+        server files1 mare-files-shard-1:6200
+        server files2 mare-files-shard-2:6200

Docker/run/config/sharded/server-shard-1.json

@@ -0,0 +1,44 @@
+{
+    "ConnectionStrings": {
+        "DefaultConnection": "Host=/var/run/postgresql;Port=5432;Database=mare;Username=mare;Keepalive=15;Minimum Pool Size=10;Maximum Pool Size=50;No Reset On Close=true;Max Auto Prepare=50;Enlist=false"
+    },
+    "Logging": {
+        "LogLevel": {
+            "Default": "Warning",
+            "Microsoft": "Warning",
+            "Microsoft.Hosting.Lifetime": "Information",
+            "MareSynchronosServer": "Information",
+            "MareSynchronosShared": "Information",
+            "System.IO": "Information"
+        },
+        "File": {
+            "BasePath": "logs",
+            "FileAccessMode": "KeepOpenAndAutoFlush",
+            "FileEncodingName": "utf-8",
+            "DateFormat": "yyyMMdd",
+            "MaxFileSize": 104857600,
+            "Files": [
+                {
+                    "Path": "<date:yyyy>/<date:MM>/<date:dd>/mare-<date:HH>-<counter:0000>.log"
+                }
+            ]
+        }
+    },
+    "MareSynchronos": {
+        "DbContextPoolSize": 512,
+        "ShardName": "Shard 1",
+        "MetricsPort": 6050,
+        "MainServerGrpcAddress": "http://mare-server:6005",
+        "RedisConnectionString": "redis,password=secretredispassword"
+    },
+    "AllowedHosts": "*",
+    "Kestrel": {
+        "Endpoints": {
+            "Http": {
+                "Url": "http://+:6000"
+            }
+        }
+    },
+    "IpRateLimiting": {},
+    "IPRateLimitPolicies": {}
+}

Docker/run/config/sharded/server-shard-2.json

@@ -0,0 +1,44 @@
+{
+    "ConnectionStrings": {
+        "DefaultConnection": "Host=/var/run/postgresql;Port=5432;Database=mare;Username=mare;Keepalive=15;Minimum Pool Size=10;Maximum Pool Size=50;No Reset On Close=true;Max Auto Prepare=50;Enlist=false"
+    },
+    "Logging": {
+        "LogLevel": {
+            "Default": "Warning",
+            "Microsoft": "Warning",
+            "Microsoft.Hosting.Lifetime": "Information",
+            "MareSynchronosServer": "Information",
+            "MareSynchronosShared": "Information",
+            "System.IO": "Information"
+        },
+        "File": {
+            "BasePath": "logs",
+            "FileAccessMode": "KeepOpenAndAutoFlush",
+            "FileEncodingName": "utf-8",
+            "DateFormat": "yyyMMdd",
+            "MaxFileSize": 104857600,
+            "Files": [
+                {
+                    "Path": "<date:yyyy>/<date:MM>/<date:dd>/mare-<date:HH>-<counter:0000>.log"
+                }
+            ]
+        }
+    },
+    "MareSynchronos": {
+        "DbContextPoolSize": 512,
+        "ShardName": "Shard 2",
+        "MetricsPort": 6050,
+        "MainServerGrpcAddress": "http://mare-server:6005",
+        "RedisConnectionString": "redis,password=secretredispassword"
+    },
+    "AllowedHosts": "*",
+    "Kestrel": {
+        "Endpoints": {
+            "Http": {
+                "Url": "http://+:6000"
+            }
+        }
+    },
+    "IpRateLimiting": {},
+    "IPRateLimitPolicies": {}
+}

Docker/run/config/sharded/server-shard-main.json

@@ -0,0 +1,60 @@
+{
+    "ConnectionStrings": {
+        "DefaultConnection": "Host=/var/run/postgresql;Port=5432;Database=mare;Username=mare;Keepalive=15;Minimum Pool Size=10;Maximum Pool Size=50;No Reset On Close=true;Max Auto Prepare=50;Enlist=false"
+    },
+    "Logging": {
+        "LogLevel": {
+            "Default": "Warning",
+            "Microsoft": "Warning",
+            "Microsoft.Hosting.Lifetime": "Information",
+            "MareSynchronosServer": "Information",
+            "MareSynchronosShared": "Information",
+            "System.IO": "Information"
+        },
+        "File": {
+            "BasePath": "logs",
+            "FileAccessMode": "KeepOpenAndAutoFlush",
+            "FileEncodingName": "utf-8",
+            "DateFormat": "yyyMMdd",
+            "MaxFileSize": 104857600,
+            "Files": [
+                {
+                    "Path": "<date:yyyy>/<date:MM>/<date:dd>/mare-<date:HH>-<counter:0000>.log"
+                }
+            ]
+        }
+    },
+    "MareSynchronos": {
+        "DbContextPoolSize": 512,
+        "ShardName": "Main",
+        "MetricsPort": 6050,
+        "MainServerGrpcAddress": "",
+        "FailedAuthForTempBan": 5,
+        "TempBanDurationInMinutes": 5,
+        "WhitelistedIps": [
+            ""
+        ],
+        "RedisConnectionString": "redis,password=secretredispassword",
+        "CdnFullUrl": "http://localhost:6200/cache/",
+        "StaticFileServiceAddress": "http://mare-files:6205",
+        "MaxExistingGroupsByUser": 3,
+        "MaxJoinedGroupsByUser": 6,
+        "MaxGroupUserCount": 100,
+        "PurgeUnusedAccounts": false,
+        "PurgeUnusedAccountsPeriodInDays": 14
+    },
+    "AllowedHosts": "*",
+    "Kestrel": {
+        "Endpoints": {
+            "Http": {
+                "Url": "http://+:6000"
+            },
+            "Grpc": {
+                "Protocols": "Http2",
+                "Url": "http://+:6005"
+            }
+        }
+    },
+    "IpRateLimiting": {},
+    "IPRateLimitPolicies": {}
+}

Docker/run/config/standalone/files-standalone.json

@@ -0,0 +1,53 @@
+{
+    "ConnectionStrings": {
+        "DefaultConnection": "Host=/var/run/postgresql;Port=5432;Database=mare;Username=mare;Keepalive=15;Minimum Pool Size=10;Maximum Pool Size=50;No Reset On Close=true;Max Auto Prepare=50;Enlist=false"
+    },
+    "Logging": {
+        "LogLevel": {
+            "Default": "Warning",
+            "Microsoft": "Warning",
+            "Microsoft.Hosting.Lifetime": "Information",
+            "MareSynchronosStaticFilesServer": "Information",
+            "MareSynchronosShared": "Information",
+            "System.IO": "Information"
+        },
+        "File": {
+            "BasePath": "logs",
+            "FileAccessMode": "KeepOpenAndAutoFlush",
+            "FileEncodingName": "utf-8",
+            "DateFormat": "yyyMMdd",
+            "MaxFileSize": 104857600,
+            "Files": [
+                {
+                    "Path": "<date:yyyy>/<date:MM>/<date:dd>/mare-<date:HH>-<counter:0000>.log"
+                }
+            ]
+        }
+    },
+    "MareSynchronos": {
+        "DbContextPoolSize": 512,
+        "ShardName": "Files",
+        "MetricsPort": 6250,
+        "FileServerGrpcAddress": "",
+        "ForcedDeletionOfFilesAfterHours": -1,
+        "CacheSizeHardLimitInGiB": -1,
+        "UnusedFileRetentionPeriodInDays": 14,
+        "CacheDirectory": "/marecache/",
+        "RemoteCacheSourceUri": "",
+        "MainServerGrpcAddress": "http://mare-server:6005"
+    },
+    "AllowedHosts": "*",
+    "Kestrel": {
+        "Endpoints": {
+            "Http": {
+                "Url": "http://+:6200"
+            },
+            "Grpc": {
+                "Protocols": "Http2",
+                "Url": "http://+:6205"
+            }
+        }
+    },
+    "IpRateLimiting": {},
+    "IPRateLimitPolicies": {}
+}

Docker/run/config/standalone/server-standalone.json

@@ -0,0 +1,60 @@
+{
+    "ConnectionStrings": {
+        "DefaultConnection": "Host=/var/run/postgresql;Port=5432;Database=mare;Username=mare;Keepalive=15;Minimum Pool Size=10;Maximum Pool Size=50;No Reset On Close=true;Max Auto Prepare=50;Enlist=false"
+    },
+    "Logging": {
+        "LogLevel": {
+            "Default": "Warning",
+            "Microsoft": "Warning",
+            "Microsoft.Hosting.Lifetime": "Information",
+            "MareSynchronosServer": "Information",
+            "MareSynchronosShared": "Information",
+            "System.IO": "Information"
+        },
+        "File": {
+            "BasePath": "logs",
+            "FileAccessMode": "KeepOpenAndAutoFlush",
+            "FileEncodingName": "utf-8",
+            "DateFormat": "yyyMMdd",
+            "MaxFileSize": 104857600,
+            "Files": [
+                {
+                    "Path": "<date:yyyy>/<date:MM>/<date:dd>/mare-<date:HH>-<counter:0000>.log"
+                }
+            ]
+        }
+    },
+    "MareSynchronos": {
+        "DbContextPoolSize": 512,
+        "ShardName": "Main",
+        "MetricsPort": 6050,
+        "MainServerGrpcAddress": "",
+        "FailedAuthForTempBan": 5,
+        "TempBanDurationInMinutes": 5,
+        "WhitelistedIps": [
+            ""
+        ],
+        "RedisConnectionString": "",
+        "CdnFullUrl": "http://localhost:6200/cache/",
+        "StaticFileServiceAddress": "http://mare-files:6205",
+        "MaxExistingGroupsByUser": 3,
+        "MaxJoinedGroupsByUser": 6,
+        "MaxGroupUserCount": 100,
+        "PurgeUnusedAccounts": false,
+        "PurgeUnusedAccountsPeriodInDays": 14
+    },
+    "AllowedHosts": "*",
+    "Kestrel": {
+        "Endpoints": {
+            "Http": {
+                "Url": "http://+:6000"
+            },
+            "Grpc": {
+                "Protocols": "Http2",
+                "Url": "http://+:6005"
+            }
+        }
+    },
+    "IpRateLimiting": {},
+    "IPRateLimitPolicies": {}
+}

Docker/run/config/standalone/services-standalone.json

@@ -0,0 +1,39 @@
+{
+    "ConnectionStrings": {
+        "DefaultConnection": "Host=/var/run/postgresql;Port=5432;Database=mare;Username=mare;Keepalive=15;Minimum Pool Size=10;Maximum Pool Size=50;No Reset On Close=true;Max Auto Prepare=50;Enlist=false"
+    },
+    "Logging": {
+        "LogLevel": {
+            "Default": "Warning",
+            "Microsoft": "Warning",
+            "Microsoft.Hosting.Lifetime": "Information",
+            "MareSynchronosServices": "Information",
+            "MareSynchronosShared": "Information",
+            "System.IO": "Information"
+        },
+        "File": {
+            "BasePath": "logs",
+            "FileAccessMode": "KeepOpenAndAutoFlush",
+            "FileEncodingName": "utf-8",
+            "DateFormat": "yyyMMdd",
+            "MaxFileSize": 104857600,
+            "Files": [
+                {
+                    "Path": "<date:yyyy>/<date:MM>/<date:dd>/mare-<date:HH>-<counter:0000>.log"
+                }
+            ]
+        }
+    },
+    "MareSynchronos": {
+        "DbContextPoolSize": 512,
+        "ShardName": "Services",
+        "MetricsPort": 6150,
+        "MainServerGrpcAddress": "http://mare-server:6005",
+        "DiscordBotToken": ""
+    },
+    "AllowedHosts": "*",
+    "Kestrel": {
+    },
+    "IpRateLimiting": {},
+    "IPRateLimitPolicies": {}
+}

Docker/run/linux-sharded-daemon-start.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+docker compose -f compose/mare-sharded.yml -p sharded up -d

Docker/run/linux-sharded-daemon-stop.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+docker compose -f compose/mare-sharded.yml -p sharded stop

Docker/run/linux-sharded.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+docker compose -f compose/mare-sharded.yml -p sharded up

Docker/run/linux-standalone-daemon-start.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+docker compose -f compose/mare-standalone.yml -p standalone up -d

Docker/run/linux-standalone-daemon-stop.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+docker compose -f compose/mare-standalone.yml -p standalone stop

Docker/run/linux-standalone.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+docker compose -f compose/mare-standalone.yml -p standalone up

Docker/run/windows-sharded-daemon-start.bat

@@ -0,0 +1,2 @@
+@echo off
+docker compose -f compose\mare-sharded.yml -p sharded up -d

Docker/run/windows-sharded-daemon-stop.bat

@@ -0,0 +1,2 @@
+@echo off
+docker compose -f compose\mare-sharded.yml -p sharded stop

Docker/run/windows-sharded.bat

@@ -0,0 +1,2 @@
+@echo off
+docker compose -f compose\mare-sharded.yml -p sharded up

Docker/run/windows-standalone-daemon-start.bat

@@ -0,0 +1,2 @@
+@echo off
+docker compose -f compose\mare-standalone.yml -p standalone up -d

Docker/run/windows-standalone-daemon-stop.bat

@@ -0,0 +1,2 @@
+@echo off
+docker compose -f compose\mare-standalone.yml -p standalone stop

Docker/run/windows-standalone.bat

@@ -0,0 +1,2 @@
+@echo off
+docker compose -f compose\mare-standalone.yml -p standalone up

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Admin.cs

@@ -1,7 +1,4 @@
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using MareSynchronos.API;
+using MareSynchronos.API;
 using MareSynchronosShared.Models;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.SignalR;
@@ -11,6 +8,7 @@ namespace MareSynchronosServer.Hubs;
 
 public partial class MareHub
 {
+    // TODO: remove all of this and migrate it to the discord bot eventually
     private List<string> OnlineAdmins => _dbContext.Users.Where(u => (u.IsModerator || u.IsAdmin)).Select(u => u.UID).ToList();
 
     [Authorize(Policy = "Admin")]
@@ -116,11 +114,11 @@ public partial class MareHub
 
         await _dbContext.SaveChangesAsync().ConfigureAwait(false);
         await Clients.Users(OnlineAdmins).Client_AdminUpdateOrAddBannedUser(dto).ConfigureAwait(false);
-        var bannedUser = _clientIdentService.GetUidForCharacterIdent(dto.CharacterHash);
-        if (!string.IsNullOrEmpty(bannedUser))
-        {
-            await Clients.User(bannedUser).Client_AdminForcedReconnect().ConfigureAwait(false);
-        }
+        //var bannedUser = _clientIdentService.GetUidForCharacterIdent(dto.CharacterHash);
+        //if (!string.IsNullOrEmpty(bannedUser))
+        //{
+        //    await Clients.User(bannedUser).Client_AdminForcedReconnect().ConfigureAwait(false);
+        //}
     }
 
     [Authorize(Policy = "Admin")]

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.ClientStubs.cs

@@ -1,6 +1,4 @@
 using MareSynchronos.API;
-using System.Threading.Tasks;
-using System;
 
 namespace MareSynchronosServer.Hubs
 {

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs

@@ -1,11 +1,5 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Security.Claims;
+using System.Security.Claims;
 using System.Security.Cryptography;
-using System.Threading;
-using System.Threading.Tasks;
 using Google.Protobuf;
 using Grpc.Core;
 using MareSynchronos.API;

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Functions.cs

@@ -1,9 +1,5 @@
 using MareSynchronosShared.Models;
 using Microsoft.EntityFrameworkCore;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using System;
 using MareSynchronosServer.Utils;
 using System.Security.Claims;
 

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Groups.cs

@@ -5,11 +5,7 @@ using MareSynchronosShared.Utils;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.SignalR;
 using Microsoft.EntityFrameworkCore;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Security.Cryptography;
-using System.Threading.Tasks;
 
 namespace MareSynchronosServer.Hubs;
 

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.User.cs

@@ -1,8 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text.RegularExpressions;
-using System.Threading.Tasks;
+using System.Text.RegularExpressions;
 using MareSynchronos.API;
 using MareSynchronosServer.Utils;
 using MareSynchronosShared.Metrics;
@@ -72,7 +68,7 @@ public partial class MareHub
         var ownIdent = _clientIdentService.GetCharacterIdentForUid(AuthenticatedUserId);
 
         var usersToSendOnlineTo = await SendOnlineToAllPairedUsers(ownIdent).ConfigureAwait(false);
-        return usersToSendOnlineTo.Select(e => _clientIdentService.GetCharacterIdentForUid(e)).Where(t => !string.IsNullOrEmpty(t)).Distinct(System.StringComparer.Ordinal).ToList();
+        return usersToSendOnlineTo.Select(e => _clientIdentService.GetCharacterIdentForUid(e)).Where(t => !string.IsNullOrEmpty(t)).Distinct(StringComparer.Ordinal).ToList();
     }
 
     [Authorize(Policy = "Identified")]
@@ -152,8 +148,8 @@ public partial class MareHub
 
         var allPairedUsers = await GetAllPairedUnpausedUsers().ConfigureAwait(false);
 
-        var allPairedUsersDict = allPairedUsers.ToDictionary(f => f, f => _clientIdentService.GetCharacterIdentForUid(f), System.StringComparer.Ordinal)
-            .Where(f => visibleCharacterIds.Contains(f.Value, System.StringComparer.Ordinal));
+        var allPairedUsersDict = allPairedUsers.ToDictionary(f => f, f => _clientIdentService.GetCharacterIdentForUid(f), StringComparer.Ordinal)
+            .Where(f => visibleCharacterIds.Contains(f.Value, StringComparer.Ordinal));
 
         var ownIdent = _clientIdentService.GetCharacterIdentForUid(AuthenticatedUserId);
 
@@ -172,7 +168,7 @@ public partial class MareHub
 
         // don't allow adding yourself or nothing
         uid = uid.Trim();
-        if (string.Equals(uid, AuthenticatedUserId, System.StringComparison.Ordinal) || string.IsNullOrWhiteSpace(uid)) return;
+        if (string.Equals(uid, AuthenticatedUserId, StringComparison.Ordinal) || string.IsNullOrWhiteSpace(uid)) return;
 
         // grab other user, check if it exists and if a pair already exists
         var otherUser = await _dbContext.Users.SingleOrDefaultAsync(u => u.UID == uid || u.Alias == uid).ConfigureAwait(false);
@@ -231,7 +227,7 @@ public partial class MareHub
         var allUserPairs = await GetAllPairedClientsWithPauseState().ConfigureAwait(false);
 
         // if the other user has paused the main user and there was no previous group connection don't send anything
-        if (!otherEntry.IsPaused && allUserPairs.Any(p => string.Equals(p.UID, uid, System.StringComparison.Ordinal) && p.IsPausedPerGroup is PauseInfo.Paused or PauseInfo.NoConnection))
+        if (!otherEntry.IsPaused && allUserPairs.Any(p => string.Equals(p.UID, uid, StringComparison.Ordinal) && p.IsPausedPerGroup is PauseInfo.Paused or PauseInfo.NoConnection))
         {
             await Clients.User(user.UID).Client_UserChangePairedPlayer(otherIdent, true).ConfigureAwait(false);
             await Clients.User(otherUser.UID).Client_UserChangePairedPlayer(userIdent, true).ConfigureAwait(false);
@@ -243,7 +239,7 @@ public partial class MareHub
     {
         _logger.LogCallInfo(MareHubLogger.Args(otherUserUid, isPaused));
 
-        if (string.Equals(otherUserUid, AuthenticatedUserId, System.StringComparison.Ordinal)) return;
+        if (string.Equals(otherUserUid, AuthenticatedUserId, StringComparison.Ordinal)) return;
         ClientPair pair = await _dbContext.ClientPairs.SingleOrDefaultAsync(w => w.UserUID == AuthenticatedUserId && w.OtherUserUID == otherUserUid).ConfigureAwait(false);
         if (pair == null) return;
 
@@ -288,7 +284,7 @@ public partial class MareHub
     {
         _logger.LogCallInfo(MareHubLogger.Args(otherUserUid));
 
-        if (string.Equals(otherUserUid, AuthenticatedUserId, System.StringComparison.Ordinal)) return;
+        if (string.Equals(otherUserUid, AuthenticatedUserId, StringComparison.Ordinal)) return;
 
         // check if client pair even exists
         ClientPair callerPair =
@@ -331,7 +327,7 @@ public partial class MareHub
         if (!callerHadPaused && otherHadPaused) return;
 
         var allUsers = await GetAllPairedClientsWithPauseState().ConfigureAwait(false);
-        var pauseEntry = allUsers.SingleOrDefault(f => string.Equals(f.UID, otherUserUid, System.StringComparison.Ordinal));
+        var pauseEntry = allUsers.SingleOrDefault(f => string.Equals(f.UID, otherUserUid, StringComparison.Ordinal));
         var isPausedInGroup = pauseEntry == null || pauseEntry.IsPausedPerGroup is PauseInfo.Paused or PauseInfo.NoConnection;
 
         // if neither user had paused each other and both are in unpaused groups, state will be online for both, do nothing

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs

@@ -1,19 +1,16 @@
-using System;
-using System.Linq;
-using System.Security.Claims;
-using System.Threading.Tasks;
+using System.Security.Claims;
 using MareSynchronos.API;
 using MareSynchronosServer.Services;
 using MareSynchronosServer.Utils;
+using MareSynchronosShared;
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
 using MareSynchronosShared.Protos;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils;
 using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.SignalR;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
 
 namespace MareSynchronosServer.Hubs;
 
@@ -23,7 +20,7 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
     private readonly FileService.FileServiceClient _fileServiceClient;
     private readonly SystemInfoService _systemInfoService;
     private readonly IHttpContextAccessor _contextAccessor;
-    private readonly GrpcClientIdentificationService _clientIdentService;
+    private readonly IClientIdentificationService _clientIdentService;
     private readonly MareHubLogger _logger;
     private readonly MareDbContext _dbContext;
     private readonly Uri _cdnFullUri;
@@ -33,18 +30,18 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
     private readonly int _maxGroupUserCount;
 
     public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient,
-        MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService, IOptions<ServerConfiguration> configuration, IHttpContextAccessor contextAccessor,
-        GrpcClientIdentificationService clientIdentService)
+        MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService,
+        IConfigurationService<ServerConfiguration> configuration, IHttpContextAccessor contextAccessor,
+        IClientIdentificationService clientIdentService)
     {
         _mareMetrics = mareMetrics;
         _fileServiceClient = fileServiceClient;
         _systemInfoService = systemInfoService;
-        var config = configuration.Value;
-        _cdnFullUri = config.CdnFullUrl;
-        _shardName = config.ShardName;
-        _maxExistingGroupsByUser = config.MaxExistingGroupsByUser;
-        _maxJoinedGroupsByUser = config.MaxJoinedGroupsByUser;
-        _maxGroupUserCount = config.MaxGroupUserCount;
+        _cdnFullUri = configuration.GetValue<Uri>(nameof(ServerConfiguration.CdnFullUrl));
+        _shardName = configuration.GetValue<string>(nameof(ServerConfiguration.ShardName));
+        _maxExistingGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxExistingGroupsByUser), 3);
+        _maxJoinedGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxJoinedGroupsByUser), 6);
+        _maxGroupUserCount = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxGroupUserCount), 100);
         _contextAccessor = contextAccessor;
         _clientIdentService = clientIdentService;
         _logger = new MareHubLogger(this, logger);
@@ -109,14 +106,14 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
     }
 
     [Authorize(Policy = "Authenticated")]
-    public async Task<bool> CheckClientHealth()
+    public Task<bool> CheckClientHealth()
     {
         var needsReconnect = !_clientIdentService.IsOnCurrentServer(AuthenticatedUserId);
         if (needsReconnect)
         {
             _logger.LogCallWarning(MareHubLogger.Args(needsReconnect));
         }
-        return needsReconnect;
+        return Task.FromResult(needsReconnect);
     }
 
     public override async Task OnConnectedAsync()

MareSynchronosServer/MareSynchronosServer/Hubs/SignalRLimitFilter.cs

@@ -1,15 +1,10 @@
-using System;
-using System.Linq;
-using System.Security.Claims;
-using System.Threading;
-using System.Threading.Tasks;
+using System.Security.Claims;
 using AspNetCoreRateLimit;
-using Microsoft.AspNetCore.Http;
+using MareSynchronosShared;
 using Microsoft.AspNetCore.SignalR;
-using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
-namespace MareSynchronosServer.Utils;
+namespace MareSynchronosServer.Hubs;
 public class SignalRLimitFilter : IHubFilter
 {
     private readonly IRateLimitProcessor _processor;

MareSynchronosServer/MareSynchronosServer/Identity/IdentityHandler.cs

@@ -1,16 +1,12 @@
 using MareSynchronosShared.Protos;
-using Microsoft.Extensions.Logging;
 using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
 
-namespace MareSynchronosServices.Identity;
+namespace MareSynchronosServer.Identity;
 
 public class IdentityHandler
 {
-    private readonly ConcurrentDictionary<string, ServerIdentity> cachedIdentities = new();
-    private readonly ConcurrentDictionary<string, ConcurrentQueue<IdentChange>> identChanges = new();
+    private readonly ConcurrentDictionary<string, ServerIdentity> _cachedIdentities = new(StringComparer.Ordinal);
+    private readonly ConcurrentDictionary<string, ConcurrentQueue<IdentChange>> _identChanges = new(StringComparer.Ordinal);
     private readonly ILogger<IdentityHandler> _logger;
 
     public IdentityHandler(ILogger<IdentityHandler> logger)
@@ -18,16 +14,9 @@ public class IdentityHandler
         _logger = logger;
     }
 
-    internal Task<string> GetUidForCharacterIdent(string ident, string serverId)
+    internal Task<ServerIdentity> GetIdentForUid(string uid)
     {
-        var exists = cachedIdentities.Any(f => f.Value.CharacterIdent == ident && f.Value.ServerId == serverId);
-        return Task.FromResult(exists ? cachedIdentities.FirstOrDefault(f => f.Value.CharacterIdent == ident && f.Value.ServerId == serverId).Key : string.Empty);
-    }
-
-    internal Task<ServerIdentity> GetIdentForuid(string uid)
-    {
-        ServerIdentity result;
-        if (!cachedIdentities.TryGetValue(uid, out result))
+        if (!_cachedIdentities.TryGetValue(uid, out ServerIdentity result))
         {
             result = new ServerIdentity();
         }
@@ -37,40 +26,40 @@ public class IdentityHandler
 
     internal void SetIdent(string uid, string serverId, string ident)
     {
-        cachedIdentities[uid] = new ServerIdentity() { ServerId = serverId, CharacterIdent = ident };
+        _cachedIdentities[uid] = new ServerIdentity() { ServerId = serverId, CharacterIdent = ident };
     }
 
     internal void RemoveIdent(string uid, string serverId)
     {
-        if (cachedIdentities.ContainsKey(uid) && cachedIdentities[uid].ServerId == serverId)
+        if (_cachedIdentities.ContainsKey(uid) && string.Equals(_cachedIdentities[uid].ServerId, serverId, StringComparison.Ordinal))
         {
-            cachedIdentities.TryRemove(uid, out _);
+            _cachedIdentities.TryRemove(uid, out _);
         }
     }
 
     internal int GetOnlineUsers(string serverId)
     {
         if (string.IsNullOrEmpty(serverId))
-            return cachedIdentities.Count;
-        return cachedIdentities.Count(c => c.Value.ServerId == serverId);
+            return _cachedIdentities.Count;
+        return _cachedIdentities.Count(c => string.Equals(c.Value.ServerId, serverId, StringComparison.Ordinal));
     }
 
     internal Dictionary<string, ServerIdentity> GetIdentsForAllExcept(string serverId)
     {
-        return cachedIdentities.Where(k => k.Value.ServerId != serverId).ToDictionary(k => k.Key, k => k.Value);
+        return _cachedIdentities.Where(k => !string.Equals(k.Value.ServerId, serverId, StringComparison.Ordinal)).ToDictionary(k => k.Key, k => k.Value, StringComparer.Ordinal);
     }
 
     internal Dictionary<string, ServerIdentity> GetIdentsForServer(string serverId)
     {
-        return cachedIdentities.Where(k => k.Value.ServerId == serverId).ToDictionary(k => k.Key, k => k.Value);
+        return _cachedIdentities.Where(k => string.Equals(k.Value.ServerId, serverId, StringComparison.Ordinal)).ToDictionary(k => k.Key, k => k.Value, StringComparer.Ordinal);
     }
 
     internal void ClearIdentsForServer(string serverId)
     {
-        var serverIdentities = cachedIdentities.Where(i => i.Value.ServerId == serverId);
+        var serverIdentities = _cachedIdentities.Where(i => string.Equals(i.Value.ServerId, serverId, StringComparison.Ordinal));
         foreach (var identity in serverIdentities)
         {
-            cachedIdentities.TryRemove(identity.Key, out _);
+            _cachedIdentities.TryRemove(identity.Key, out _);
         }
     }
 
@@ -78,16 +67,16 @@ public class IdentityHandler
     {
         _logger.LogInformation("Enqueued " + identchange.UidWithIdent.Uid.Uid + ":" + identchange.IsOnline + " from " + identchange.UidWithIdent.Ident.ServerId);
 
-        foreach (var k in identChanges.Keys)
+        foreach (var k in _identChanges.Keys)
         {
-            if (string.Equals(k, identchange.UidWithIdent.Ident.ServerId, System.StringComparison.Ordinal)) continue;
-            identChanges[k].Enqueue(identchange);
+            if (string.Equals(k, identchange.UidWithIdent.Ident.ServerId, StringComparison.Ordinal)) continue;
+            _identChanges[k].Enqueue(identchange);
         }
     }
 
-    internal bool DequeueIdentChange(string server, out IdentChange? cur)
+    internal bool DequeueIdentChange(string server, out IdentChange cur)
     {
-        if (!(identChanges.ContainsKey(server) && identChanges[server].TryDequeue(out cur)))
+        if (!(_identChanges.ContainsKey(server) && _identChanges[server].TryDequeue(out cur)))
         {
             cur = null;
             return false;
@@ -98,7 +87,7 @@ public class IdentityHandler
 
     internal void RegisterServerForQueue(string serverId)
     {
-        identChanges[serverId] = new ConcurrentQueue<IdentChange>();
+        _identChanges[serverId] = new ConcurrentQueue<IdentChange>();
     }
 
     internal record ServerIdentity

MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj

@@ -4,6 +4,7 @@
     <TargetFramework>net7.0</TargetFramework>
     <UserSecretsId>aspnet-MareSynchronosServer-BA82A12A-0B30-463C-801D-B7E81318CD50</UserSecretsId>
     <AssemblyVersion>1.1.0.0</AssemblyVersion>
+    <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
 
   <ItemGroup>

MareSynchronosServer/MareSynchronosServer/Program.cs

@@ -1,14 +1,8 @@
-using System;
-using Microsoft.AspNetCore.Hosting;
-using Microsoft.Extensions.Hosting;
-using System.Linq;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
-using Microsoft.Extensions.Options;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils;
 
 namespace MareSynchronosServer;
 
@@ -22,9 +16,12 @@ public class Program
         {
             var services = scope.ServiceProvider;
             using var context = services.GetRequiredService<MareDbContext>();
+            var options = services.GetRequiredService<IConfigurationService<ServerConfiguration>>();
+            var logger = host.Services.GetRequiredService<ILogger<Program>>();
+            logger.LogInformation("Loaded MareSynchronos Server Configuration (IsMain: {isMain})", options.IsMain);
+            logger.LogInformation(options.ToString());
 
-            var secondaryServer = Environment.GetEnvironmentVariable("SECONDARY_SERVER");
-            if (string.IsNullOrEmpty(secondaryServer) || string.Equals(secondaryServer, "0", StringComparison.Ordinal))
+            if (options.IsMain)
             {
                 context.Database.Migrate();
                 context.SaveChanges();
@@ -42,15 +39,18 @@ public class Program
             metrics.SetGaugeTo(MetricsAPI.GaugePairs, context.ClientPairs.Count());
             metrics.SetGaugeTo(MetricsAPI.GaugePairsPaused, context.ClientPairs.Count(p => p.IsPaused));
 
-            var options = host.Services.GetService<IOptions<ServerConfiguration>>();
-            var logger = host.Services.GetService<ILogger<Program>>();
-            logger.LogInformation("Loaded MareSynchronos Server Configuration");
-            logger.LogInformation(options.Value.ToString());
         }
 
         if (args.Length == 0 || !string.Equals(args[0], "dry", StringComparison.Ordinal))
         {
-            host.Run();
+            try
+            {
+                host.Run();
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine(ex);
+            }
         }
     }
 

MareSynchronosServer/MareSynchronosServer/RequirementHandlers/UserRequirementHandler.cs

@@ -1,23 +1,19 @@
-using System.Threading.Tasks;
-using System;
-using System.Linq;
-using System.Security.Claims;
+using System.Security.Claims;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.SignalR;
 using MareSynchronosShared.Data;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.Logging;
 using MareSynchronosServer.Services;
 
 namespace MareSynchronosServer.RequirementHandlers;
 
 public class UserRequirementHandler : AuthorizationHandler<UserRequirement, HubInvocationContext>
 {
-    private readonly GrpcClientIdentificationService identClient;
+    private readonly IClientIdentificationService identClient;
     private readonly MareDbContext dbContext;
     private readonly ILogger<UserRequirementHandler> logger;
 
-    public UserRequirementHandler(GrpcClientIdentificationService identClient, MareDbContext dbContext, ILogger<UserRequirementHandler> logger)
+    public UserRequirementHandler(IClientIdentificationService identClient, MareDbContext dbContext, ILogger<UserRequirementHandler> logger)
     {
         this.identClient = identClient;
         this.dbContext = dbContext;

MareSynchronosServer/MareSynchronosServer/Services/GrpcClientIdentificationService.cs

@@ -2,17 +2,12 @@
 using MareSynchronosShared.Metrics;
 using MareSynchronosShared.Protos;
 using MareSynchronosShared.Services;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-using System;
+using MareSynchronosShared.Utils;
 using System.Collections.Concurrent;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
 
 namespace MareSynchronosServer.Services;
 
-public class GrpcClientIdentificationService : GrpcBaseService
+public class GrpcClientIdentificationService : GrpcBaseService, IClientIdentificationService
 {
     private readonly string _shardName;
     private readonly ILogger<GrpcClientIdentificationService> _logger;
@@ -22,13 +17,15 @@ public class GrpcClientIdentificationService : GrpcBaseService
     private readonly MareMetrics _metrics;
     protected readonly ConcurrentDictionary<string, UidWithIdent> OnlineClients = new(StringComparer.Ordinal);
     private readonly ConcurrentDictionary<string, UidWithIdent> RemoteCachedIdents = new(StringComparer.Ordinal);
-    private ConcurrentQueue<IdentChange> _identChangeQueue = new();
+    private readonly ConcurrentQueue<IdentChange> _identChangeQueue = new();
 
-    public GrpcClientIdentificationService(ILogger<GrpcClientIdentificationService> logger, IdentificationService.IdentificationServiceClient gprcIdentClient,
+    public GrpcClientIdentificationService(ILogger<GrpcClientIdentificationService> logger,
+        IdentificationService.IdentificationServiceClient gprcIdentClient,
         IdentificationService.IdentificationServiceClient gprcIdentClientStreamOut,
-        IdentificationService.IdentificationServiceClient gprcIdentClientStreamIn, MareMetrics metrics, IOptions<ServerConfiguration> configuration) : base(logger)
+        IdentificationService.IdentificationServiceClient gprcIdentClientStreamIn, 
+        MareMetrics metrics, IConfigurationService<ServerConfiguration> configuration) : base(logger)
     {
-        _shardName = configuration.Value.ShardName;
+        _shardName = configuration.GetValueOrDefault(nameof(ServerConfiguration.ShardName), string.Empty);
         _logger = logger;
         _grpcIdentClient = gprcIdentClient;
         _grpcIdentClientStreamOut = gprcIdentClientStreamOut;
@@ -171,7 +168,7 @@ public class GrpcClientIdentificationService : GrpcBaseService
             using var stream = _grpcIdentClientStreamIn.ReceiveStreamIdentStatusChange(new ServerMessage()
             {
                 ServerId = _shardName,
-            });
+            }, cancellationToken: cts);
             _logger.LogInformation("Starting Receive Online Client Data stream");
             await foreach (var cur in stream.ResponseStream.ReadAllAsync(cts).ConfigureAwait(false))
             {
@@ -201,7 +198,7 @@ public class GrpcClientIdentificationService : GrpcBaseService
 
     protected override async Task StopAsyncInternal(CancellationToken cancellationToken)
     {
-        await ExecuteOnGrpc(_grpcIdentClient.ClearIdentsForServerAsync(new ServerMessage() { ServerId = _shardName })).ConfigureAwait(false);
+        await ExecuteOnGrpc(_grpcIdentClient.ClearIdentsForServerAsync(new ServerMessage() { ServerId = _shardName }, cancellationToken: cancellationToken)).ConfigureAwait(false);
     }
 
     protected override async Task OnGrpcRestore()

MareSynchronosServer/MareSynchronosServer/Services/IClientIdentificationService.cs

@@ -0,0 +1,11 @@
+namespace MareSynchronosServer.Services;
+
+public interface IClientIdentificationService : IHostedService
+{
+    string GetCharacterIdentForUid(string uid);
+    Task<long> GetOnlineUsers();
+    string GetServerForUid(string uid);
+    bool IsOnCurrentServer(string uid);
+    void MarkUserOffline(string uid);
+    void MarkUserOnline(string uid, string charaIdent);
+}

MareSynchronosServer/MareSynchronosServer/Services/IdentityService.cs

@@ -1,16 +1,16 @@
 using Grpc.Core;
+using MareSynchronosServer.Identity;
 using MareSynchronosShared.Protos;
-using Microsoft.Extensions.Logging;
-using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authorization;
 
-namespace MareSynchronosServices.Identity;
+namespace MareSynchronosServer.Services;
 
-internal class IdentityService : IdentificationService.IdentificationServiceBase
+internal class GrpcIdentityService : IdentificationService.IdentificationServiceBase
 {
-    private readonly ILogger<IdentityService> _logger;
+    private readonly ILogger<GrpcIdentityService> _logger;
     private readonly IdentityHandler _handler;
 
-    public IdentityService(ILogger<IdentityService> logger, IdentityHandler handler)
+    public GrpcIdentityService(ILogger<GrpcIdentityService> logger, IdentityHandler handler)
     {
         _logger = logger;
         _handler = handler;
@@ -18,7 +18,7 @@ internal class IdentityService : IdentificationService.IdentificationServiceBase
 
     public override async Task<CharacterIdentMessage> GetIdentForUid(UidMessage request, ServerCallContext context)
     {
-        var result = await _handler.GetIdentForuid(request.Uid);
+        var result = await _handler.GetIdentForUid(request.Uid).ConfigureAwait(false);
         return new CharacterIdentMessage()
         {
             Ident = result.CharacterIdent,
@@ -26,6 +26,7 @@ internal class IdentityService : IdentificationService.IdentificationServiceBase
         };
     }
 
+    [AllowAnonymous]
     public override Task<OnlineUserCountResponse> GetOnlineUserCount(ServerMessage request, ServerCallContext context)
     {
         return Task.FromResult(new OnlineUserCountResponse() { Count = _handler.GetOnlineUsers(request.ServerId) });
@@ -66,7 +67,7 @@ internal class IdentityService : IdentificationService.IdentificationServiceBase
 
     public override async Task<Empty> SendStreamIdentStatusChange(IAsyncStreamReader<IdentChangeMessage> requestStream, ServerCallContext context)
     {
-        await requestStream.MoveNext();
+        await requestStream.MoveNext().ConfigureAwait(false);
         var server = requestStream.Current.Server;
         if (server == null) throw new System.Exception("First message needs to be server message");
         _handler.RegisterServerForQueue(server.ServerId);

MareSynchronosServer/MareSynchronosServer/Services/LocalClientIdentificationService.cs

@@ -0,0 +1,60 @@
+using MareSynchronosShared.Protos;
+using System.Collections.Concurrent;
+using MareSynchronosServer.Identity;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils;
+
+namespace MareSynchronosServer.Services;
+
+public class LocalClientIdentificationService : IClientIdentificationService
+{
+    protected readonly ConcurrentDictionary<string, UidWithIdent> OnlineClients = new(StringComparer.Ordinal);
+    private readonly IdentityHandler _identityHandler;
+    private readonly string _shardName;
+
+    public LocalClientIdentificationService(IdentityHandler identityHandler, IConfigurationService<ServerConfiguration> config)
+    {
+        _identityHandler = identityHandler;
+        _shardName = config.GetValueOrDefault(nameof(ServerConfiguration.ShardName), string.Empty);
+    }
+
+    public string GetCharacterIdentForUid(string uid)
+    {
+        return _identityHandler.GetIdentForUid(uid).Result.CharacterIdent;
+    }
+
+    public Task<long> GetOnlineUsers()
+    {
+        return Task.FromResult((long)_identityHandler.GetOnlineUsers(string.Empty));
+    }
+
+    public string GetServerForUid(string uid)
+    {
+        return _identityHandler.GetIdentForUid(uid).Result.ServerId;
+    }
+
+    public bool IsOnCurrentServer(string uid)
+    {
+        return string.Equals(_identityHandler.GetIdentForUid(uid).Result.ServerId, _shardName, StringComparison.Ordinal);
+    }
+
+    public void MarkUserOffline(string uid)
+    {
+        _identityHandler.RemoveIdent(uid, _shardName);
+    }
+
+    public void MarkUserOnline(string uid, string charaIdent)
+    {
+        _identityHandler.SetIdent(uid, _shardName, charaIdent);
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        return Task.CompletedTask;
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        return Task.CompletedTask;
+    }
+}

MareSynchronosServer/MareSynchronosServer/Services/SystemInfoService.cs

@@ -1,32 +1,30 @@
-using System;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using MareSynchronos.API;
+using MareSynchronos.API;
 using MareSynchronosServer.Hubs;
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils;
 using Microsoft.AspNetCore.SignalR;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
 
 namespace MareSynchronosServer.Services;
 
 public class SystemInfoService : IHostedService, IDisposable
 {
     private readonly MareMetrics _mareMetrics;
+    private readonly IConfigurationService<ServerConfiguration> _config;
     private readonly IServiceProvider _services;
-    private readonly GrpcClientIdentificationService _clientIdentService;
+    private readonly IClientIdentificationService _clientIdentService;
     private readonly ILogger<SystemInfoService> _logger;
     private readonly IHubContext<MareHub, IMareHub> _hubContext;
     private Timer _timer;
     public SystemInfoDto SystemInfoDto { get; private set; } = new();
 
-    public SystemInfoService(MareMetrics mareMetrics, IServiceProvider services, GrpcClientIdentificationService clientIdentService, ILogger<SystemInfoService> logger, IHubContext<MareHub, IMareHub> hubContext)
+    public SystemInfoService(MareMetrics mareMetrics, IConfigurationService<ServerConfiguration> configurationService, IServiceProvider services,
+        IClientIdentificationService clientIdentService, ILogger<SystemInfoService> logger, IHubContext<MareHub, IMareHub> hubContext)
     {
         _mareMetrics = mareMetrics;
+        _config = configurationService;
         _services = services;
         _clientIdentService = clientIdentService;
         _logger = logger;
@@ -49,14 +47,16 @@ public class SystemInfoService : IHostedService, IDisposable
         _mareMetrics.SetGaugeTo(MetricsAPI.GaugeAvailableWorkerThreads, workerThreads);
         _mareMetrics.SetGaugeTo(MetricsAPI.GaugeAvailableIOWorkerThreads, ioThreads);
 
-        var secondaryServer = Environment.GetEnvironmentVariable("SECONDARY_SERVER");
-        if (string.IsNullOrEmpty(secondaryServer) || string.Equals(secondaryServer, "0", StringComparison.Ordinal))
+        if (_config.IsMain)
         {
+            var onlineUsers = (int)_clientIdentService.GetOnlineUsers().Result;
             SystemInfoDto = new SystemInfoDto()
             {
-                OnlineUsers = (int)_clientIdentService.GetOnlineUsers().Result,
+                OnlineUsers = onlineUsers,
             };
 
+            _logger.LogInformation("Sending System Info, Online Users: {onlineUsers}", onlineUsers);
+
             _hubContext.Clients.All.Client_UpdateSystemInfo(SystemInfoDto);
 
             using var scope = _services.CreateScope();

MareSynchronosServer/MareSynchronosServer/Services/UserCleanupService.cs

@@ -1,50 +1,114 @@
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
 using MareSynchronosShared.Models;
+using MareSynchronosShared.Services;
 using MareSynchronosShared.Utils;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
 
-namespace MareSynchronosServices;
+namespace MareSynchronosServer.Services;
 
-public class CleanupService : IHostedService, IDisposable
+public class UserCleanupService : IHostedService
 {
     private readonly MareMetrics metrics;
-    private readonly ILogger<CleanupService> _logger;
+    private readonly ILogger<UserCleanupService> _logger;
     private readonly IServiceProvider _services;
-    private readonly ServicesConfiguration _configuration;
-    private Timer? _timer;
+    private readonly IConfigurationService<ServerConfiguration> _configuration;
+    private CancellationTokenSource _cleanupCts;
 
-    public CleanupService(MareMetrics metrics, ILogger<CleanupService> logger, IServiceProvider services, IOptions<ServicesConfiguration> configuration)
+    public UserCleanupService(MareMetrics metrics, ILogger<UserCleanupService> logger, IServiceProvider services, IConfigurationService<ServerConfiguration> configuration)
     {
         this.metrics = metrics;
         _logger = logger;
         _services = services;
-        _configuration = configuration.Value;
+        _configuration = configuration;
     }
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
         _logger.LogInformation("Cleanup Service started");
+        _cleanupCts = new();
 
-        _timer = new Timer(CleanUp, null, TimeSpan.Zero, TimeSpan.FromMinutes(10));
+        _ = CleanUp(_cleanupCts.Token);
 
         return Task.CompletedTask;
     }
 
-    private async void CleanUp(object state)
+    private async Task CleanUp(CancellationToken ct)
     {
-        using var scope = _services.CreateScope();
-        using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
+        while (!ct.IsCancellationRequested)
+        {
+            using var scope = _services.CreateScope();
+            using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
 
+            CleanUpOutdatedLodestoneAuths(dbContext);
+
+            await PurgeUnusedAccounts(dbContext).ConfigureAwait(false);
+
+            await PurgeTempInvites(dbContext).ConfigureAwait(false);
+
+            dbContext.SaveChanges();
+
+            var now = DateTime.Now;
+            TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
+            TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % 10, 0);
+            var span = futureTime.AddMinutes(10) - currentTime;
+
+            _logger.LogInformation("User Cleanup Complete, next run at {date}", now.Add(span));
+            await Task.Delay(span, ct).ConfigureAwait(false);
+        }
+    }
+
+    private async Task PurgeTempInvites(MareDbContext dbContext)
+    {
+        try
+        {
+            var tempInvites = await dbContext.GroupTempInvites.ToListAsync().ConfigureAwait(false);
+            dbContext.RemoveRange(tempInvites.Where(i => i.ExpirationDate < DateTime.UtcNow));
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error during Temp Invite purge");
+        }
+    }
+
+    private async Task PurgeUnusedAccounts(MareDbContext dbContext)
+    {
+        try
+        {
+            if (_configuration.GetValueOrDefault(nameof(ServerConfiguration.PurgeUnusedAccounts), false))
+            {
+                var usersOlderThanDays = _configuration.GetValueOrDefault(nameof(ServerConfiguration.PurgeUnusedAccountsPeriodInDays), 14);
+                var maxGroupsByUser = _configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxGroupUserCount), 3);
+
+                _logger.LogInformation("Cleaning up users older than {usersOlderThanDays} days", usersOlderThanDays);
+
+                var allUsers = dbContext.Users.Where(u => string.IsNullOrEmpty(u.Alias)).ToList();
+                List<User> usersToRemove = new();
+                foreach (var user in allUsers)
+                {
+                    if (user.LastLoggedIn < DateTime.UtcNow - TimeSpan.FromDays(usersOlderThanDays))
+                    {
+                        _logger.LogInformation("User outdated: {userUID}", user.UID);
+                        usersToRemove.Add(user);
+                    }
+                }
+
+                foreach (var user in usersToRemove)
+                {
+                    await SharedDbFunctions.PurgeUser(_logger, user, dbContext, maxGroupsByUser).ConfigureAwait(false);
+                }
+            }
+
+            _logger.LogInformation("Cleaning up unauthorized users");
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error during user purge");
+        }
+    }
+
+    private void CleanUpOutdatedLodestoneAuths(MareDbContext dbContext)
+    {
         try
         {
             _logger.LogInformation($"Cleaning up expired lodestone authentications");
@@ -65,52 +129,6 @@ public class CleanupService : IHostedService, IDisposable
         {
             _logger.LogWarning(ex, "Error during expired auths cleanup");
         }
-
-        try
-        {
-            if (_configuration.PurgeUnusedAccounts)
-            {
-                var usersOlderThanDays = _configuration.PurgeUnusedAccountsPeriodInDays;
-
-                _logger.LogInformation("Cleaning up users older than {usersOlderThanDays} days", usersOlderThanDays);
-
-                var allUsers = dbContext.Users.Where(u => string.IsNullOrEmpty(u.Alias)).ToList();
-                List<User> usersToRemove = new();
-                foreach (var user in allUsers)
-                {
-                    if (user.LastLoggedIn < (DateTime.UtcNow - TimeSpan.FromDays(usersOlderThanDays)))
-                    {
-                        _logger.LogInformation("User outdated: {userUID}", user.UID);
-                        usersToRemove.Add(user);
-                    }
-                }
-
-                foreach (var user in usersToRemove)
-                {
-                    await PurgeUser(user, dbContext);
-                }
-            }
-
-            _logger.LogInformation("Cleaning up unauthorized users");
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning(ex, "Error during user purge");
-        }
-
-        try
-        {
-            var tempInvites = await dbContext.GroupTempInvites.ToListAsync();
-            dbContext.RemoveRange(tempInvites.Where(i => i.ExpirationDate < DateTime.UtcNow));
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning(ex, "Error during Temp Invite purge");
-        }
-
-        _logger.LogInformation($"Cleanup complete");
-
-        dbContext.SaveChanges();
     }
 
     public async Task PurgeUser(User user, MareDbContext dbContext)
@@ -152,13 +170,13 @@ public class CleanupService : IHostedService, IDisposable
                 }
                 else
                 {
-                    _ = await SharedDbFunctions.MigrateOrDeleteGroup(dbContext, userGroupPair.Group, groupPairs, _configuration.MaxExistingGroupsByUser).ConfigureAwait(false);
+                    _ = await SharedDbFunctions.MigrateOrDeleteGroup(dbContext, userGroupPair.Group, groupPairs, _configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxExistingGroupsByUser), 3)).ConfigureAwait(false);
                 }
             }
 
             dbContext.GroupPairs.Remove(userGroupPair);
 
-            await dbContext.SaveChangesAsync();
+            await dbContext.SaveChangesAsync().ConfigureAwait(false);
         }
 
         _logger.LogInformation("User purged: {uid}", user.UID);
@@ -166,20 +184,15 @@ public class CleanupService : IHostedService, IDisposable
         dbContext.Auth.Remove(auth);
         dbContext.Users.Remove(user);
 
-        await dbContext.SaveChangesAsync();
+        await dbContext.SaveChangesAsync().ConfigureAwait(false);
 
         metrics.DecGauge(MetricsAPI.GaugeUsersRegistered, 1);
     }
 
     public Task StopAsync(CancellationToken cancellationToken)
     {
-        _timer?.Change(Timeout.Infinite, 0);
+        _cleanupCts.Cancel();
 
         return Task.CompletedTask;
     }
-
-    public void Dispose()
-    {
-        _timer?.Dispose();
-    }
 }

MareSynchronosServer/MareSynchronosServer/Startup.cs

@@ -1,11 +1,5 @@
-using System;
 using MareSynchronos.API;
-using Microsoft.AspNetCore.Builder;
-using Microsoft.AspNetCore.Hosting;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
 using MareSynchronosServer.Hubs;
 using Microsoft.AspNetCore.Authentication;
 using Microsoft.AspNetCore.Http.Connections;
@@ -16,15 +10,14 @@ using MareSynchronosShared.Authentication;
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Protos;
 using Grpc.Net.Client.Configuration;
-using Prometheus;
 using MareSynchronosShared.Metrics;
-using System.Collections.Generic;
 using MareSynchronosServer.Services;
-using System.Net.Http;
 using MareSynchronosServer.Utils;
 using MareSynchronosServer.RequirementHandlers;
-using Microsoft.Extensions.Logging;
 using MareSynchronosShared.Utils;
+using MareSynchronosServer.Identity;
+using MareSynchronosShared.Services;
+using Prometheus;
 
 namespace MareSynchronosServer;
 
@@ -41,98 +34,91 @@ public class Startup
     {
         services.AddHttpContextAccessor();
 
-        services.Configure<ServerConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
-        services.Configure<MareConfigurationAuthBase>(Configuration.GetRequiredSection("MareSynchronos"));
-        services.Configure<IpRateLimitOptions>(Configuration.GetSection("IpRateLimiting"));
-        services.Configure<IpRateLimitPolicies>(Configuration.GetSection("IpRateLimitPolicies"));
         services.AddTransient(_ => Configuration);
 
-        services.AddMemoryCache();
-        services.AddInMemoryRateLimiting();
-
-        services.AddSingleton<SystemInfoService, SystemInfoService>();
-        services.AddSingleton<IUserIdProvider, IdBasedUserIdProvider>();
-
         var mareConfig = Configuration.GetRequiredSection("MareSynchronos");
 
-        var defaultMethodConfig = new MethodConfig
-        {
-            Names = { MethodName.Default },
-            RetryPolicy = new RetryPolicy
-            {
-                MaxAttempts = 1000,
-                InitialBackoff = TimeSpan.FromSeconds(1),
-                MaxBackoff = TimeSpan.FromSeconds(5),
-                BackoffMultiplier = 1.5,
-                RetryableStatusCodes = { Grpc.Core.StatusCode.Unavailable }
-            }
-        };
+        // configure metrics
+        ConfigureMetrics(services);
 
-        var noRetryConfig = new MethodConfig
-        {
-            Names = { MethodName.Default },
-            RetryPolicy = null
-        };
+        // configure file service grpc connection
+        ConfigureFileServiceGrpcClient(services, mareConfig);
 
-        services.AddSingleton<MareMetrics>(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string>
-        {
-            MetricsAPI.CounterInitializedConnections,
-            MetricsAPI.CounterUserPushData,
-            MetricsAPI.CounterUserPushDataTo,
-            MetricsAPI.CounterUsersRegisteredDeleted,
-            MetricsAPI.CounterAuthenticationCacheHits,
-            MetricsAPI.CounterAuthenticationFailures,
-            MetricsAPI.CounterAuthenticationRequests,
-            MetricsAPI.CounterAuthenticationSuccesses
-        }, new List<string>
-        {
-            MetricsAPI.GaugeAuthorizedConnections,
-            MetricsAPI.GaugeConnections,
-            MetricsAPI.GaugePairs,
-            MetricsAPI.GaugePairsPaused,
-            MetricsAPI.GaugeAvailableIOWorkerThreads,
-            MetricsAPI.GaugeAvailableWorkerThreads,
-            MetricsAPI.GaugeGroups,
-            MetricsAPI.GaugeGroupPairs,
-            MetricsAPI.GaugeGroupPairsPaused
-        }));
+        // configure database
+        ConfigureDatabase(services, mareConfig);
 
-        services.AddGrpcClient<FileService.FileServiceClient>(c =>
+        // configure authentication and authorization
+        ConfigureAuthorization(services);
+
+        // configure rate limiting
+        ConfigureIpRateLimiting(services);
+
+        // configure SignalR
+        ConfigureSignalR(services, mareConfig);
+
+        // configure mare specific services
+        ConfigureMareServices(services, mareConfig);
+    }
+
+    private static void ConfigureMareServices(IServiceCollection services, IConfigurationSection mareConfig)
+    {
+        bool isMainServer = mareConfig.GetValue<Uri>(nameof(ServerConfiguration.MainServerGrpcAddress), defaultValue: null) == null;
+
+        services.Configure<ServerConfiguration>(mareConfig);
+        services.Configure<MareConfigurationBase>(mareConfig);
+        services.Configure<MareConfigurationAuthBase>(mareConfig);
+
+        services.AddSingleton<SystemInfoService>();
+        services.AddSingleton<IUserIdProvider, IdBasedUserIdProvider>();
+        services.AddHostedService(provider => provider.GetService<SystemInfoService>());
+        // configure services based on main server status
+        ConfigureIdentityServices(services, mareConfig, isMainServer);
+
+        if (isMainServer)
         {
-            c.Address = new Uri(mareConfig.GetValue<string>(nameof(ServerConfiguration.StaticFileServiceAddress)));
-        }).ConfigureChannel(c =>
+            services.AddSingleton<UserCleanupService>();
+            services.AddHostedService(provider => provider.GetService<UserCleanupService>());
+        }
+    }
+
+    private static void ConfigureSignalR(IServiceCollection services, IConfigurationSection mareConfig)
+    {
+        var signalRServiceBuilder = services.AddSignalR(hubOptions =>
         {
-            c.ServiceConfig = new ServiceConfig { MethodConfigs = { defaultMethodConfig } };
-        });
-        services.AddGrpcClient<IdentificationService.IdentificationServiceClient>(c =>
-        {
-            c.Address = new Uri(mareConfig.GetValue<string>(nameof(ServerConfiguration.ServiceAddress)));
-        }).ConfigureChannel(c =>
-        {
-            c.ServiceConfig = new ServiceConfig { MethodConfigs = { noRetryConfig } };
-            c.HttpHandler = new SocketsHttpHandler()
-            {
-                EnableMultipleHttp2Connections = true
-            };
+            hubOptions.MaximumReceiveMessageSize = long.MaxValue;
+            hubOptions.EnableDetailedErrors = true;
+            hubOptions.MaximumParallelInvocationsPerClient = 10;
+            hubOptions.StreamBufferCapacity = 200;
+
+            hubOptions.AddFilter<SignalRLimitFilter>();
         });
 
+        // configure redis for SignalR
+        var redis = mareConfig.GetValue(nameof(ServerConfiguration.RedisConnectionString), string.Empty);
+        if (!string.IsNullOrEmpty(redis))
+        {
+            signalRServiceBuilder.AddStackExchangeRedis(redis, options =>
+            {
+                options.Configuration.ChannelPrefix = "MareSynchronos";
+            });
+        }
+    }
+
+    private void ConfigureIpRateLimiting(IServiceCollection services)
+    {
+        services.Configure<IpRateLimitOptions>(Configuration.GetSection("IpRateLimiting"));
+        services.Configure<IpRateLimitPolicies>(Configuration.GetSection("IpRateLimitPolicies"));
+        services.AddSingleton<IRateLimitConfiguration, RateLimitConfiguration>();
+        services.AddMemoryCache();
+        services.AddInMemoryRateLimiting();
+    }
+
+    private static void ConfigureAuthorization(IServiceCollection services)
+    {
         services.AddSingleton<SecretKeyAuthenticatorService>();
-        services.AddSingleton<GrpcClientIdentificationService>();
         services.AddTransient<IAuthorizationHandler, UserRequirementHandler>();
-        services.AddHostedService(p => p.GetService<GrpcClientIdentificationService>());
-
-        services.AddDbContextPool<MareDbContext>(options =>
-        {
-            options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder =>
-            {
-                builder.MigrationsHistoryTable("_efmigrationshistory", "public");
-                builder.MigrationsAssembly("MareSynchronosShared");
-            }).UseSnakeCaseNamingConvention();
-            options.EnableThreadSafetyChecks(false);
-        }, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
-
         services.AddAuthentication(SecretKeyAuthenticationHandler.AuthScheme)
-            .AddScheme<AuthenticationSchemeOptions, SecretKeyAuthenticationHandler>(SecretKeyAuthenticationHandler.AuthScheme, options => { options.Validate(); });
+           .AddScheme<AuthenticationSchemeOptions, SecretKeyAuthenticationHandler>(SecretKeyAuthenticationHandler.AuthScheme, options => { options.Validate(); });
 
         services.AddAuthorization(options =>
         {
@@ -157,44 +143,127 @@ public class Startup
                 policy.AddRequirements(new UserRequirement(UserRequirements.Identified | UserRequirements.Moderator | UserRequirements.Administrator));
             });
         });
-
-        services.AddSingleton<IRateLimitConfiguration, RateLimitConfiguration>();
-
-        var signalRServiceBuilder = services.AddSignalR(hubOptions =>
-        {
-            hubOptions.MaximumReceiveMessageSize = long.MaxValue;
-            hubOptions.EnableDetailedErrors = true;
-            hubOptions.MaximumParallelInvocationsPerClient = 10;
-            hubOptions.StreamBufferCapacity = 200;
-
-            hubOptions.AddFilter<SignalRLimitFilter>();
-        });
-
-        // add redis related options
-        var redis = mareConfig.GetValue(nameof(ServerConfiguration.RedisConnectionString), string.Empty);
-        if (!string.IsNullOrEmpty(redis))
-        {
-            signalRServiceBuilder.AddStackExchangeRedis(redis, options =>
-            {
-                options.Configuration.ChannelPrefix = "MareSynchronos";
-            });
-        }
-
-        services.AddHostedService(provider => provider.GetService<SystemInfoService>());
     }
 
-    public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
+    private void ConfigureDatabase(IServiceCollection services, IConfigurationSection mareConfig)
     {
-        if (env.IsDevelopment())
+        services.AddDbContextPool<MareDbContext>(options =>
         {
-            app.UseDeveloperExceptionPage();
-            app.UseMigrationsEndPoint();
+            options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder =>
+            {
+                builder.MigrationsHistoryTable("_efmigrationshistory", "public");
+                builder.MigrationsAssembly("MareSynchronosShared");
+            }).UseSnakeCaseNamingConvention();
+            options.EnableThreadSafetyChecks(false);
+        }, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
+    }
+
+    private static void ConfigureMetrics(IServiceCollection services)
+    {
+        services.AddSingleton<MareMetrics>(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string>
+        {
+            MetricsAPI.CounterInitializedConnections,
+            MetricsAPI.CounterUserPushData,
+            MetricsAPI.CounterUserPushDataTo,
+            MetricsAPI.CounterUsersRegisteredDeleted,
+            MetricsAPI.CounterAuthenticationCacheHits,
+            MetricsAPI.CounterAuthenticationFailures,
+            MetricsAPI.CounterAuthenticationRequests,
+            MetricsAPI.CounterAuthenticationSuccesses,
+        }, new List<string>
+        {
+            MetricsAPI.GaugeAuthorizedConnections,
+            MetricsAPI.GaugeConnections,
+            MetricsAPI.GaugePairs,
+            MetricsAPI.GaugePairsPaused,
+            MetricsAPI.GaugeAvailableIOWorkerThreads,
+            MetricsAPI.GaugeAvailableWorkerThreads,
+            MetricsAPI.GaugeGroups,
+            MetricsAPI.GaugeGroupPairs,
+            MetricsAPI.GaugeGroupPairsPaused,
+            MetricsAPI.GaugeUsersRegistered
+        }));
+    }
+
+    private static void ConfigureIdentityServices(IServiceCollection services, IConfigurationSection mareConfig, bool isMainServer)
+    {
+        if (!isMainServer)
+        {
+            var noRetryConfig = new MethodConfig
+            {
+                Names = { MethodName.Default },
+                RetryPolicy = null
+            };
+
+            services.AddGrpcClient<IdentificationService.IdentificationServiceClient>(c =>
+            {
+                c.Address = new Uri(mareConfig.GetValue<string>(nameof(ServerConfiguration.MainServerGrpcAddress)));
+            }).ConfigureChannel(c =>
+            {
+                c.ServiceConfig = new ServiceConfig { MethodConfigs = { noRetryConfig } };
+                c.HttpHandler = new SocketsHttpHandler()
+                {
+                    EnableMultipleHttp2Connections = true
+                };
+            });
+
+            services.AddGrpcClient<ConfigurationService.ConfigurationServiceClient>(c =>
+            {
+                c.Address = new Uri(mareConfig.GetValue<string>(nameof(ServerConfiguration.MainServerGrpcAddress)));
+            }).ConfigureChannel(c =>
+            {
+                c.ServiceConfig = new ServiceConfig { MethodConfigs = { noRetryConfig } };
+                c.HttpHandler = new SocketsHttpHandler()
+                {
+                    EnableMultipleHttp2Connections = true
+                };
+            });
+
+            services.AddSingleton<IClientIdentificationService, GrpcClientIdentificationService>();
+            services.AddHostedService(p => p.GetService<IClientIdentificationService>());
+            services.AddSingleton<IConfigurationService<ServerConfiguration>, MareConfigurationServiceClient<ServerConfiguration>>();
+            services.AddSingleton<IConfigurationService<MareConfigurationAuthBase>, MareConfigurationServiceClient<MareConfigurationAuthBase>>();
         }
         else
         {
-            app.UseExceptionHandler("/Error");
-            app.UseHsts();
+            services.AddSingleton<IdentityHandler>();
+            services.AddSingleton<IClientIdentificationService, LocalClientIdentificationService>();
+            services.AddSingleton<IConfigurationService<ServerConfiguration>, MareConfigurationServiceServer<ServerConfiguration>>();
+            services.AddSingleton<IConfigurationService<MareConfigurationAuthBase>, MareConfigurationServiceServer<MareConfigurationAuthBase>>();
+
+            services.AddGrpc();
         }
+    }
+
+    private static void ConfigureFileServiceGrpcClient(IServiceCollection services, IConfigurationSection mareConfig)
+    {
+        var defaultMethodConfig = new MethodConfig
+        {
+            Names = { MethodName.Default },
+            RetryPolicy = new RetryPolicy
+            {
+                MaxAttempts = 1000,
+                InitialBackoff = TimeSpan.FromSeconds(1),
+                MaxBackoff = TimeSpan.FromSeconds(5),
+                BackoffMultiplier = 1.5,
+                RetryableStatusCodes = { Grpc.Core.StatusCode.Unavailable }
+            }
+        };
+        services.AddGrpcClient<FileService.FileServiceClient>((serviceProvider, c) =>
+        {
+            c.Address = serviceProvider.GetRequiredService<IConfigurationService<ServerConfiguration>>()
+                .GetValue<Uri>(nameof(ServerConfiguration.StaticFileServiceAddress));
+        }).ConfigureChannel(c =>
+        {
+            c.ServiceConfig = new ServiceConfig { MethodConfigs = { defaultMethodConfig } };
+        });
+    }
+
+    public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILogger<Startup> logger)
+    {
+        logger.LogInformation("Running Configure");
+
+        var config = app.ApplicationServices.GetRequiredService<IConfigurationService<MareConfigurationAuthBase>>();
 
         app.UseIpRateLimiting();
 
@@ -202,7 +271,7 @@ public class Startup
 
         app.UseWebSockets();
 
-        var metricServer = new KestrelMetricServer(4980);
+        var metricServer = new KestrelMetricServer(config.GetValueOrDefault<int>(nameof(MareConfigurationBase.MetricsPort), 4981));
         metricServer.Start();
 
         app.UseAuthentication();
@@ -216,6 +285,12 @@ public class Startup
                 options.TransportMaxBufferSize = 5242880;
                 options.Transports = HttpTransportType.WebSockets | HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling;
             });
+
+            if (config.IsMain)
+            {
+                endpoints.MapGrpcService<GrpcIdentityService>().AllowAnonymous();
+                endpoints.MapGrpcService<GrpcConfigurationService<ServerConfiguration>>().AllowAnonymous();
+            }
         });
     }
 }

MareSynchronosServer/MareSynchronosServer/Utils/IdBasedUserIdProvider.cs

@@ -1,5 +1,4 @@
-using System.Linq;
-using System.Security.Claims;
+using System.Security.Claims;
 using Microsoft.AspNetCore.SignalR;
 
 namespace MareSynchronosServer.Utils;
@@ -8,6 +7,6 @@ public class IdBasedUserIdProvider : IUserIdProvider
 {
     public string GetUserId(HubConnectionContext context)
     {
-        return context.User!.Claims.SingleOrDefault(c => string.Equals(c.Type, ClaimTypes.NameIdentifier, System.StringComparison.Ordinal))?.Value;
+        return context.User!.Claims.SingleOrDefault(c => string.Equals(c.Type, ClaimTypes.NameIdentifier, StringComparison.Ordinal))?.Value;
     }
 }

MareSynchronosServer/MareSynchronosServer/Utils/MareHubLogger.cs

@@ -1,5 +1,4 @@
 using MareSynchronosServer.Hubs;
-using Microsoft.Extensions.Logging;
 using System.Runtime.CompilerServices;
 
 namespace MareSynchronosServer.Utils;
@@ -14,6 +13,7 @@ public class MareHubLogger
         _hub = hub;
         _logger = logger;
     }
+
     public static object[] Args(params object[] args)
     {
         return args;

MareSynchronosServer/MareSynchronosServer/Utils/PausedEntry.cs

@@ -1,8 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-namespace MareSynchronosServer.Utils;
+namespace MareSynchronosServer.Utils;
 
 public record PausedEntry
 {

MareSynchronosServer/MareSynchronosServices/Discord/DiscordBot.cs

@@ -1,41 +1,34 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using Discord;
+using Discord;
 using Discord.Interactions;
 using Discord.Rest;
 using Discord.WebSocket;
-using MareSynchronosServices.Identity;
 using MareSynchronosShared.Data;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
+using static MareSynchronosShared.Protos.IdentificationService;
 
 namespace MareSynchronosServices.Discord;
 
 internal class DiscordBot : IHostedService
 {
     private readonly DiscordBotServices _botServices;
-    private readonly IdentityHandler _identityHandler;
     private readonly IServiceProvider _services;
-    private readonly ServicesConfiguration _configuration;
+    private readonly IConfigurationService<ServicesConfiguration> _configurationService;
     private readonly ILogger<DiscordBot> _logger;
+    private readonly IdentificationServiceClient _identificationServiceClient;
     private readonly DiscordSocketClient _discordClient;
     private CancellationTokenSource? _updateStatusCts;
     private CancellationTokenSource? _vanityUpdateCts;
 
-    public DiscordBot(DiscordBotServices botServices, IdentityHandler identityHandler, IServiceProvider services, IOptions<ServicesConfiguration> configuration, ILogger<DiscordBot> logger)
+    public DiscordBot(DiscordBotServices botServices, IServiceProvider services, IConfigurationService<ServicesConfiguration> configuration,
+        ILogger<DiscordBot> logger, IdentificationServiceClient identificationServiceClient)
     {
         _botServices = botServices;
-        _identityHandler = identityHandler;
         _services = services;
-        _configuration = configuration.Value;
+        _configurationService = configuration;
         _logger = logger;
-
+        this._identificationServiceClient = identificationServiceClient;
         _discordClient = new(new DiscordSocketConfig()
         {
             DefaultRetryMode = RetryMode.AlwaysRetry
@@ -72,7 +65,8 @@ internal class DiscordBot : IHostedService
         _vanityUpdateCts = new();
         var guild = (await _discordClient.Rest.GetGuildsAsync()).First();
         var commands = await guild.GetApplicationCommandsAsync();
-        var vanityCommandId = commands.First(c => c.Name == "setvanityuid").Id;
+        var appId = await _discordClient.GetApplicationInfoAsync().ConfigureAwait(false);
+        var vanityCommandId = commands.First(c => c.ApplicationId == appId.Id && c.Name == "setvanityuid").Id;
 
         while (!_vanityUpdateCts.IsCancellationRequested)
         {
@@ -176,18 +170,19 @@ internal class DiscordBot : IHostedService
         _updateStatusCts = new();
         while (!_updateStatusCts.IsCancellationRequested)
         {
-            var onlineUsers = _identityHandler.GetOnlineUsers(string.Empty);
-            _logger.LogInformation("Users online: " + onlineUsers);
-            await _discordClient.SetActivityAsync(new Game("Mare for " + onlineUsers + " Users")).ConfigureAwait(false);
+            var onlineUsers = await _identificationServiceClient.GetOnlineUserCountAsync(new MareSynchronosShared.Protos.ServerMessage());
+            _logger.LogInformation("Users online: " + onlineUsers.Count);
+            await _discordClient.SetActivityAsync(new Game("Mare for " + onlineUsers.Count + " Users")).ConfigureAwait(false);
             await Task.Delay(TimeSpan.FromSeconds(15)).ConfigureAwait(false);
         }
     }
 
     public async Task StartAsync(CancellationToken cancellationToken)
     {
-        if (!string.IsNullOrEmpty(_configuration.DiscordBotToken))
+        var token = _configurationService.GetValueOrDefault(nameof(ServicesConfiguration.DiscordBotToken), string.Empty);
+        if (!string.IsNullOrEmpty(token))
         {
-            await _discordClient.LoginAsync(TokenType.Bot, _configuration.DiscordBotToken).ConfigureAwait(false);
+            await _discordClient.LoginAsync(TokenType.Bot, token).ConfigureAwait(false);
             await _discordClient.StartAsync().ConfigureAwait(false);
 
             _discordClient.Ready += DiscordClient_Ready;
@@ -199,7 +194,7 @@ internal class DiscordBot : IHostedService
 
     public async Task StopAsync(CancellationToken cancellationToken)
     {
-        if (!string.IsNullOrEmpty(_configuration.DiscordBotToken))
+        if (!string.IsNullOrEmpty(_configurationService.GetValueOrDefault(nameof(ServicesConfiguration.DiscordBotToken), string.Empty)))
         {
             await _botServices.Stop();
             _updateStatusCts?.Cancel();

MareSynchronosServer/MareSynchronosServices/Discord/DiscordBotServices.cs

@@ -1,17 +1,11 @@
-using System;
-using System.Threading.Tasks;
-using System.Collections.Generic;
-using System.Collections.Concurrent;
+using System.Collections.Concurrent;
 using MareSynchronosShared.Metrics;
-using Microsoft.Extensions.Logging;
-using System.Threading;
-using Microsoft.Extensions.Options;
 
 namespace MareSynchronosServices.Discord;
 
 public class DiscordBotServices
 {
-    public readonly ConcurrentQueue<KeyValuePair<ulong, Action<IServiceProvider>>> verificationQueue = new();
+    public ConcurrentQueue<KeyValuePair<ulong, Action<IServiceProvider>>> VerificationQueue { get; } = new();
     public ConcurrentDictionary<ulong, DateTime> LastVanityChange = new();
     public ConcurrentDictionary<string, DateTime> LastVanityGidChange = new();
     public ConcurrentDictionary<ulong, string> DiscordLodestoneMapping = new();
@@ -19,29 +13,27 @@ public class DiscordBotServices
     public readonly string[] LodestoneServers = new[] { "eu", "na", "jp", "fr", "de" };
     private readonly IServiceProvider _serviceProvider;
 
-    public ServicesConfiguration Configuration { get; init; }
     public ILogger<DiscordBotServices> Logger { get; init; }
     public MareMetrics Metrics { get; init; }
-    public Random Random { get; init; }
     private CancellationTokenSource? verificationTaskCts;
 
-    public DiscordBotServices(IOptions<ServicesConfiguration> configuration, IServiceProvider serviceProvider, ILogger<DiscordBotServices> logger, MareMetrics metrics)
+    public DiscordBotServices(IServiceProvider serviceProvider, ILogger<DiscordBotServices> logger, MareMetrics metrics)
     {
-        Configuration = configuration.Value;
         _serviceProvider = serviceProvider;
         Logger = logger;
         Metrics = metrics;
-        Random = new();
     }
 
-    public async Task Start()
+    public Task Start()
     {
         _ = ProcessVerificationQueue();
+        return Task.CompletedTask;
     }
 
-    public async Task Stop()
+    public Task Stop()
     {
         verificationTaskCts?.Cancel();
+        return Task.CompletedTask;
     }
 
     private async Task ProcessVerificationQueue()
@@ -49,7 +41,7 @@ public class DiscordBotServices
         verificationTaskCts = new CancellationTokenSource();
         while (!verificationTaskCts.IsCancellationRequested)
         {
-            if (verificationQueue.TryDequeue(out var queueitem))
+            if (VerificationQueue.TryDequeue(out var queueitem))
             {
                 try
                 {
@@ -61,8 +53,8 @@ public class DiscordBotServices
                 {
                     Logger.LogError(e, "Error during queue work");
                 }
-
             }
+
             await Task.Delay(TimeSpan.FromSeconds(2), verificationTaskCts.Token).ConfigureAwait(false);
         }
     }

MareSynchronosServer/MareSynchronosServices/Discord/MareModule.cs

@@ -2,20 +2,15 @@
 using Discord.Interactions;
 using MareSynchronosShared.Data;
 using System.Text.RegularExpressions;
-using System;
-using System.Threading.Tasks;
-using Microsoft.Extensions.DependencyInjection;
 using Microsoft.EntityFrameworkCore;
 using Discord.WebSocket;
-using System.Linq;
 using Prometheus;
 using MareSynchronosShared.Models;
-using MareSynchronosServices.Identity;
 using MareSynchronosShared.Metrics;
-using System.Net.Http;
 using MareSynchronosShared.Utils;
-using System.Collections.Generic;
-using Microsoft.Extensions.Logging;
+using MareSynchronosShared.Services;
+using static MareSynchronosShared.Protos.IdentificationService;
+using static System.Formats.Asn1.AsnWriter;
 
 namespace MareSynchronosServices.Discord;
 
@@ -30,22 +25,30 @@ public class LodestoneModal : IModal
 
 public class MareModule : InteractionModuleBase
 {
+    private readonly ILogger<MareModule> _logger;
     private readonly IServiceProvider _services;
     private readonly DiscordBotServices _botServices;
-    private readonly IdentityHandler _identityHandler;
-    private readonly CleanupService _cleanupService;
+    private readonly IdentificationServiceClient _identificationServiceClient;
+    private readonly IConfigurationService<ServerConfiguration> _mareClientConfigurationService;
+    private Random random = new();
 
-    public MareModule(IServiceProvider services, DiscordBotServices botServices, IdentityHandler identityHandler, CleanupService cleanupService)
+    public MareModule(ILogger<MareModule> logger, IServiceProvider services, DiscordBotServices botServices,
+        IdentificationServiceClient identificationServiceClient, IConfigurationService<ServerConfiguration> mareClientConfigurationService)
     {
+        _logger = logger;
         _services = services;
         _botServices = botServices;
-        _identityHandler = identityHandler;
-        _cleanupService = cleanupService;
+        _identificationServiceClient = identificationServiceClient;
+        _mareClientConfigurationService = mareClientConfigurationService;
     }
 
     [SlashCommand("register", "Starts the registration process for the Mare Synchronos server of this Discord")]
     public async Task Register([Summary("overwrite", "Overwrites your old account")] bool overwrite = false)
     {
+        _logger.LogInformation("SlashCommand:{userId}:{Method}:{params}",
+            Context.Client.CurrentUser.Id, nameof(Register),
+            string.Join(",", new[] { $"{nameof(overwrite)}:{overwrite}" }));
+
         await TryRespondAsync(async () =>
         {
             if (overwrite)
@@ -60,6 +63,10 @@ public class MareModule : InteractionModuleBase
     [SlashCommand("setvanityuid", "Sets your Vanity UID.")]
     public async Task SetVanityUid([Summary("vanity_uid", "Desired Vanity UID")] string vanityUid)
     {
+        _logger.LogInformation("SlashCommand:{userId}:{Method}:{params}",
+            Context.Client.CurrentUser.Id, nameof(SetVanityUid),
+            string.Join(",", new[] { $"{nameof(vanityUid)}:{vanityUid}" }));
+
         await TryRespondAsync(async () =>
         {
             EmbedBuilder eb = new();
@@ -75,6 +82,10 @@ public class MareModule : InteractionModuleBase
         [Summary("syncshell_id", "Syncshell ID")] string syncshellId,
         [Summary("vanity_syncshell_id", "Desired Vanity Syncshell ID")] string vanityId)
     {
+        _logger.LogInformation("SlashCommand:{userId}:{Method}:{params}",
+            Context.Client.CurrentUser.Id, nameof(SetSyncshellVanityId),
+            string.Join(",", new[] { $"{nameof(syncshellId)}:{syncshellId}", $"{nameof(vanityId)}:{vanityId}" }));
+
         await TryRespondAsync(async () =>
         {
             EmbedBuilder eb = new();
@@ -88,10 +99,12 @@ public class MareModule : InteractionModuleBase
     [SlashCommand("verify", "Finishes the registration process for the Mare Synchronos server of this Discord")]
     public async Task Verify()
     {
+        _logger.LogInformation("SlashCommand:{userId}:{Method}",
+            Context.Client.CurrentUser.Id, nameof(Verify));
         await TryRespondAsync(async () =>
         {
             EmbedBuilder eb = new();
-            if (_botServices.verificationQueue.Any(u => u.Key == Context.User.Id))
+            if (_botServices.VerificationQueue.Any(u => u.Key == Context.User.Id))
             {
                 eb.WithTitle("Already queued for verfication");
                 eb.WithDescription("You are already queued for verification. Please wait.");
@@ -106,7 +119,7 @@ public class MareModule : InteractionModuleBase
             else
             {
                 await DeferAsync(ephemeral: true).ConfigureAwait(false);
-                _botServices.verificationQueue.Enqueue(new KeyValuePair<ulong, Action<IServiceProvider>>(Context.User.Id, async (sp) => await HandleVerifyAsync((SocketSlashCommand)Context.Interaction, sp)));
+                _botServices.VerificationQueue.Enqueue(new KeyValuePair<ulong, Action<IServiceProvider>>(Context.User.Id, async (sp) => await HandleVerifyAsync((SocketSlashCommand)Context.Interaction, sp)));
             }
         });
     }
@@ -114,10 +127,12 @@ public class MareModule : InteractionModuleBase
     [SlashCommand("verify_relink", "Finishes the relink process for your user on the Mare Synchronos server of this Discord")]
     public async Task VerifyRelink()
     {
+        _logger.LogInformation("SlashCommand:{userId}:{Method}",
+            Context.Client.CurrentUser.Id, nameof(VerifyRelink));
         await TryRespondAsync(async () =>
         {
             EmbedBuilder eb = new();
-            if (_botServices.verificationQueue.Any(u => u.Key == Context.User.Id))
+            if (_botServices.VerificationQueue.Any(u => u.Key == Context.User.Id))
             {
                 eb.WithTitle("Already queued for verfication");
                 eb.WithDescription("You are already queued for verification. Please wait.");
@@ -132,7 +147,7 @@ public class MareModule : InteractionModuleBase
             else
             {
                 await DeferAsync(ephemeral: true).ConfigureAwait(false);
-                _botServices.verificationQueue.Enqueue(new KeyValuePair<ulong, Action<IServiceProvider>>(Context.User.Id, async (sp) => await HandleVerifyRelinkAsync((SocketSlashCommand)Context.Interaction, sp)));
+                _botServices.VerificationQueue.Enqueue(new KeyValuePair<ulong, Action<IServiceProvider>>(Context.User.Id, async (sp) => await HandleVerifyRelinkAsync((SocketSlashCommand)Context.Interaction, sp)));
             }
         });
     }
@@ -140,6 +155,8 @@ public class MareModule : InteractionModuleBase
     [SlashCommand("recover", "Allows you to recover your account by generating a new secret key")]
     public async Task Recover()
     {
+        _logger.LogInformation("SlashCommand:{userId}:{Method}",
+            Context.Client.CurrentUser.Id, nameof(Recover));
         await RespondWithModalAsync<LodestoneModal>("recover_modal").ConfigureAwait(false);
     }
 
@@ -148,6 +165,9 @@ public class MareModule : InteractionModuleBase
         [Summary("discord_user", "ADMIN ONLY: Discord User to check for")] IUser? discordUser = null,
         [Summary("uid", "ADMIN ONLY: UID to check for")] string? uid = null)
     {
+        _logger.LogInformation("SlashCommand:{userId}:{Method}",
+            Context.Client.CurrentUser.Id, nameof(UserInfo));
+
         await TryRespondAsync(async () =>
         {
             EmbedBuilder eb = new();
@@ -161,12 +181,32 @@ public class MareModule : InteractionModuleBase
     [SlashCommand("relink", "Allows you to link a new Discord account to an existing Mare account")]
     public async Task Relink()
     {
+        _logger.LogInformation("SlashCommand:{userId}:{Method}",
+            Context.Client.CurrentUser.Id, nameof(Relink));
         await RespondWithModalAsync<LodestoneModal>("relink_modal").ConfigureAwait(false);
     }
 
+    [SlashCommand("useradd", "ADMIN ONLY: add a user unconditionally to the Database")]
+    public async Task UserAdd([Summary("desired_uid", "Desired UID")] string desiredUid)
+    {
+        _logger.LogInformation("SlashCommand:{userId}:{Method}:{params}",
+            Context.Client.CurrentUser.Id, nameof(UserAdd),
+            string.Join(",", new[] { $"{nameof(desiredUid)}:{desiredUid}" }));
+
+        await TryRespondAsync(async () =>
+        {
+            var embed = await HandleUserAdd(desiredUid, Context.User.Id);
+
+            await RespondAsync(embeds: new[] { embed }, ephemeral: true).ConfigureAwait(false);
+        });
+    }
+
     [ModalInteraction("recover_modal")]
     public async Task RecoverModal(LodestoneModal modal)
     {
+        _logger.LogInformation("Modal:{userId}:{Method}",
+            Context.Client.CurrentUser.Id, nameof(RecoverModal));
+
         await TryRespondAsync(async () =>
         {
             var embed = await HandleRecoverModalAsync(modal, Context.User.Id).ConfigureAwait(false);
@@ -177,6 +217,9 @@ public class MareModule : InteractionModuleBase
     [ModalInteraction("register_modal")]
     public async Task RegisterModal(LodestoneModal modal)
     {
+        _logger.LogInformation("Modal:{userId}:{Method}",
+            Context.Client.CurrentUser.Id, nameof(RegisterModal));
+
         await TryRespondAsync(async () =>
         {
             var embed = await HandleRegisterModalAsync(modal, Context.User.Id).ConfigureAwait(false);
@@ -187,6 +230,9 @@ public class MareModule : InteractionModuleBase
     [ModalInteraction("relink_modal")]
     public async Task RelinkModal(LodestoneModal modal)
     {
+        _logger.LogInformation("Modal:{userId}:{Method}",
+            Context.Client.CurrentUser.Id, nameof(RelinkModal));
+
         await TryRespondAsync(async () =>
         {
             var embed = await HandleRelinkModalAsync(modal, Context.User.Id).ConfigureAwait(false);
@@ -194,6 +240,51 @@ public class MareModule : InteractionModuleBase
         });
     }
 
+    public async Task<Embed> HandleUserAdd(string desiredUid, ulong discordUserId)
+    {
+        var embed = new EmbedBuilder();
+
+        using var scope = _services.CreateScope();
+        using var db = scope.ServiceProvider.GetService<MareDbContext>();
+        if (!(await db.LodeStoneAuth.Include(u => u.User).SingleOrDefaultAsync(a => a.DiscordId == discordUserId))?.User?.IsAdmin ?? true)
+        {
+            embed.WithTitle("Failed to add user");
+            embed.WithDescription("No permission");
+        }
+        else if (db.Users.Any(u => u.UID == desiredUid || u.Alias == desiredUid))
+        {
+            embed.WithTitle("Failed to add user");
+            embed.WithDescription("Already in Database");
+        }
+        else
+        {
+            User newUser = new()
+            {
+                IsAdmin = false,
+                IsModerator = false,
+                LastLoggedIn = DateTime.UtcNow,
+                UID = desiredUid,
+            };
+
+            var computedHash = StringUtils.Sha256String(StringUtils.GenerateRandomString(64) + DateTime.UtcNow.ToString());
+            var auth = new Auth()
+            {
+                HashedKey = StringUtils.Sha256String(computedHash),
+                User = newUser,
+            };
+
+            await db.Users.AddAsync(newUser);
+            await db.Auth.AddAsync(auth);
+
+            await db.SaveChangesAsync();
+
+            embed.WithTitle("Successfully added " + desiredUid);
+            embed.WithDescription("Secret Key: " + computedHash);
+        }
+
+        return embed.Build();
+    }
+
     private async Task TryRespondAsync(Action act)
     {
         try
@@ -258,7 +349,7 @@ public class MareModule : InteractionModuleBase
         var lodestoneUser = await db.LodeStoneAuth.Include(u => u.User).SingleOrDefaultAsync(u => u.DiscordId == userToCheckForDiscordId).ConfigureAwait(false);
         var dbUser = lodestoneUser.User;
         var auth = await db.Auth.SingleOrDefaultAsync(u => u.UserUID == dbUser.UID).ConfigureAwait(false);
-        var identity = await _identityHandler.GetIdentForuid(dbUser.UID).ConfigureAwait(false);
+        var identity = await _identificationServiceClient.GetIdentForUidAsync(new MareSynchronosShared.Protos.UidMessage { Uid = dbUser.UID });
         var groups = await db.Groups.Where(g => g.OwnerUID == dbUser.UID).ToListAsync().ConfigureAwait(false);
         var groupsJoined = await db.GroupPairs.Where(g => g.GroupUserUID == dbUser.UID).ToListAsync().ConfigureAwait(false);
 
@@ -271,7 +362,7 @@ public class MareModule : InteractionModuleBase
             eb.AddField("Vanity UID", dbUser.Alias);
         }
         eb.AddField("Last Online (UTC)", dbUser.LastLoggedIn.ToString("U"));
-        eb.AddField("Currently online: ", !string.IsNullOrEmpty(identity.CharacterIdent));
+        eb.AddField("Currently online: ", !string.IsNullOrEmpty(identity.Ident));
         eb.AddField("Hashed Secret Key", auth.HashedKey);
         eb.AddField("Joined Syncshells", groupsJoined.Count);
         eb.AddField("Owned Syncshells", groups.Count);
@@ -285,9 +376,9 @@ public class MareModule : InteractionModuleBase
             eb.AddField("Owned Syncshell " + group.GID + " User Count", syncShellUserCount);
         }
 
-        if (isAdminCall && !string.IsNullOrEmpty(identity.CharacterIdent))
+        if (isAdminCall && !string.IsNullOrEmpty(identity.Ident))
         {
-            eb.AddField("Character Ident", identity.CharacterIdent);
+            eb.AddField("Character Ident", identity.Ident);
         }
 
         return eb;
@@ -635,7 +726,9 @@ public class MareModule : InteractionModuleBase
         {
             if (discordAuthedUser.User != null)
             {
-                await _cleanupService.PurgeUser(discordAuthedUser.User, db);
+                var maxGroupsByUser = _mareClientConfigurationService.GetValueOrDefault(nameof(ServerConfiguration.MaxGroupUserCount), 3);
+
+                await SharedDbFunctions.PurgeUser(_logger, discordAuthedUser.User, db, maxGroupsByUser);
             }
             else
             {
@@ -657,7 +750,7 @@ public class MareModule : InteractionModuleBase
         var lodestoneAuth = db.LodeStoneAuth.SingleOrDefault(u => u.DiscordId == cmd.User.Id);
         if (lodestoneAuth != null && _botServices.DiscordRelinkLodestoneMapping.ContainsKey(cmd.User.Id))
         {
-            var randomServer = _botServices.LodestoneServers[_botServices.Random.Next(_botServices.LodestoneServers.Length)];
+            var randomServer = _botServices.LodestoneServers[random.Next(_botServices.LodestoneServers.Length)];
             var response = await req.GetAsync($"https://{randomServer}.finalfantasyxiv.com/lodestone/character/{_botServices.DiscordRelinkLodestoneMapping[cmd.User.Id]}").ConfigureAwait(false);
             if (response.IsSuccessStatusCode)
             {
@@ -731,7 +824,7 @@ public class MareModule : InteractionModuleBase
         var lodestoneAuth = db.LodeStoneAuth.SingleOrDefault(u => u.DiscordId == cmd.User.Id);
         if (lodestoneAuth != null && _botServices.DiscordLodestoneMapping.ContainsKey(cmd.User.Id))
         {
-            var randomServer = _botServices.LodestoneServers[_botServices.Random.Next(_botServices.LodestoneServers.Length)];
+            var randomServer = _botServices.LodestoneServers[random.Next(_botServices.LodestoneServers.Length)];
             var response = await req.GetAsync($"https://{randomServer}.finalfantasyxiv.com/lodestone/character/{_botServices.DiscordLodestoneMapping[cmd.User.Id]}").ConfigureAwait(false);
             if (response.IsSuccessStatusCode)
             {
@@ -757,11 +850,7 @@ public class MareModule : InteractionModuleBase
                         user.IsAdmin = true;
                     }
 
-                    if (_botServices.Configuration.PurgeUnusedAccounts)
-                    {
-                        var purgedDays = _botServices.Configuration.PurgeUnusedAccountsPeriodInDays;
-                        user.LastLoggedIn = DateTime.UtcNow - TimeSpan.FromDays(purgedDays) + TimeSpan.FromDays(1);
-                    }
+                    user.LastLoggedIn = DateTime.UtcNow;
 
                     var computedHash = StringUtils.Sha256String(StringUtils.GenerateRandomString(64) + DateTime.UtcNow.ToString());
                     var auth = new Auth()

MareSynchronosServer/MareSynchronosServices/MareSynchronosServices.csproj

@@ -2,6 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
 
   <ItemGroup>

MareSynchronosServer/MareSynchronosServices/Program.cs

@@ -1,13 +1,8 @@
 using MareSynchronosServices;
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
-using Microsoft.AspNetCore.Hosting;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-using System;
-using System.Linq;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils;
 
 public class Program
 {
@@ -24,10 +19,13 @@ public class Program
 
             metrics.SetGaugeTo(MetricsAPI.GaugeUsersRegistered, dbContext.Users.Count());
 
-            var options = host.Services.GetService<IOptions<ServicesConfiguration>>();
+            var options = host.Services.GetService<IConfigurationService<ServicesConfiguration>>();
+            var optionsServer = host.Services.GetService<IConfigurationService<ServerConfiguration>>();
             var logger = host.Services.GetService<ILogger<Program>>();
-            logger.LogInformation("Loaded MareSynchronos Services Configuration");
-            logger.LogInformation(options.Value.ToString());
+            logger.LogInformation("Loaded MareSynchronos Services Configuration (IsMain: {isMain})", options.IsMain);
+            logger.LogInformation(options.ToString());
+            logger.LogInformation("Loaded MareSynchronos Server Configuration (IsMain: {isMain})", optionsServer.IsMain);
+            logger.LogInformation(optionsServer.ToString());
         }
 
         host.Run();

MareSynchronosServer/MareSynchronosServices/ServicesConfiguration.cs

@@ -1,23 +0,0 @@
-using MareSynchronosShared.Utils;
-using System.Text;
-
-namespace MareSynchronosServices;
-
-public class ServicesConfiguration : MareConfigurationBase
-{
-    public string DiscordBotToken { get; set; } = string.Empty;
-    public bool PurgeUnusedAccounts { get; set; } = false;
-    public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14;
-    public int MaxExistingGroupsByUser { get; set; } = 3;
-
-    public override string ToString()
-    {
-        StringBuilder sb = new();
-        sb.AppendLine(base.ToString());
-        sb.AppendLine($"{nameof(DiscordBotToken)} => {DiscordBotToken}");
-        sb.AppendLine($"{nameof(PurgeUnusedAccounts)} => {PurgeUnusedAccounts}");
-        sb.AppendLine($"{nameof(PurgeUnusedAccountsPeriodInDays)} => {PurgeUnusedAccountsPeriodInDays}");
-        sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");
-        return sb.ToString();
-    }
-}

MareSynchronosServer/MareSynchronosServices/Startup.cs

@@ -1,16 +1,12 @@
 using MareSynchronosServices.Discord;
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
-using Microsoft.AspNetCore.Builder;
-using Microsoft.AspNetCore.Hosting;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.DependencyInjection;
 using Prometheus;
-using System.Collections.Generic;
-using MareSynchronosServices.Identity;
-using Microsoft.Extensions.Logging;
 using MareSynchronosShared.Utils;
+using Grpc.Net.Client.Configuration;
+using MareSynchronosShared.Protos;
+using MareSynchronosShared.Services;
 
 namespace MareSynchronosServices;
 
@@ -25,6 +21,8 @@ public class Startup
 
     public void ConfigureServices(IServiceCollection services)
     {
+        var mareConfig = Configuration.GetSection("MareSynchronos");
+
         services.AddDbContextPool<MareDbContext>(options =>
         {
             options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder =>
@@ -34,32 +32,58 @@ public class Startup
             options.EnableThreadSafetyChecks(false);
         }, Configuration.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
 
-        services.AddSingleton(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string> {
-        }, new List<string> 
+        services.AddSingleton(m => new MareMetrics(m.GetService<ILogger<MareMetrics>>(), new List<string> { },
+        new List<string>
         {
             MetricsAPI.GaugeUsersRegistered
         }));
 
+        var noRetryConfig = new MethodConfig
+        {
+            Names = { MethodName.Default },
+            RetryPolicy = null
+        };
+
+        services.AddGrpcClient<IdentificationService.IdentificationServiceClient>(c =>
+        {
+            c.Address = new Uri(mareConfig.GetValue<string>(nameof(ServicesConfiguration.MainServerGrpcAddress)));
+        }).ConfigureChannel(c =>
+        {
+            c.ServiceConfig = new ServiceConfig { MethodConfigs = { noRetryConfig } };
+            c.HttpHandler = new SocketsHttpHandler()
+            {
+                EnableMultipleHttp2Connections = true
+            };
+        });
+
+        services.AddGrpcClient<ConfigurationService.ConfigurationServiceClient>(c =>
+        {
+            c.Address = new Uri(mareConfig.GetValue<string>(nameof(ServicesConfiguration.MainServerGrpcAddress)));
+        }).ConfigureChannel(c =>
+        {
+            c.ServiceConfig = new ServiceConfig { MethodConfigs = { noRetryConfig } };
+            c.HttpHandler = new SocketsHttpHandler()
+            {
+                EnableMultipleHttp2Connections = true
+            };
+        });
+
         services.Configure<ServicesConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
+        services.Configure<ServerConfiguration>(Configuration.GetRequiredSection("MareSynchronos"));
+        services.Configure<MareConfigurationAuthBase>(Configuration.GetRequiredSection("MareSynchronos"));
         services.AddSingleton(Configuration);
         services.AddSingleton<DiscordBotServices>();
-        services.AddSingleton<IdentityHandler>();
-        services.AddSingleton<CleanupService>();
-        services.AddHostedService(provider => provider.GetService<CleanupService>());
         services.AddHostedService<DiscordBot>();
-        services.AddGrpc();
+        services.AddSingleton<IConfigurationService<ServicesConfiguration>, MareConfigurationServiceServer<ServicesConfiguration>>();
+        services.AddSingleton<IConfigurationService<ServerConfiguration>, MareConfigurationServiceClient<ServerConfiguration>>();
+        services.AddSingleton<IConfigurationService<MareConfigurationAuthBase>, MareConfigurationServiceClient<MareConfigurationAuthBase>>();
     }
 
     public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
     {
-        app.UseRouting();
+        var config = app.ApplicationServices.GetRequiredService<IConfigurationService<MareConfigurationAuthBase>>();
 
-        var metricServer = new KestrelMetricServer(4982);
+        var metricServer = new KestrelMetricServer(config.GetValueOrDefault<int>(nameof(MareConfigurationBase.MetricsPort), 4982));
         metricServer.Start();
-
-        app.UseEndpoints(endpoints =>
-        {
-            endpoints.MapGrpcService<IdentityService>();
-        });
     }
 }

MareSynchronosServer/MareSynchronosShared/Authentication/SecretKeyAuthenticationHandler.cs

@@ -1,8 +1,7 @@
 using System.Security.Claims;
 using System.Text.Encodings.Web;
-using MareSynchronosServer;
-using MareSynchronosShared.Data;
 using Microsoft.AspNetCore.Authentication;
+using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
@@ -13,7 +12,6 @@ public class SecretKeyAuthenticationHandler : AuthenticationHandler<Authenticati
 {
     public const string AuthScheme = "SecretKeyGrpcAuth";
 
-    private readonly MareDbContext _mareDbContext;
     private readonly IHttpContextAccessor _accessor;
     private readonly SecretKeyAuthenticatorService secretKeyAuthenticatorService;
 
@@ -26,6 +24,12 @@ public class SecretKeyAuthenticationHandler : AuthenticationHandler<Authenticati
 
     protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
     {
+        var endpoint = Context.GetEndpoint();
+        if (endpoint?.Metadata?.GetMetadata<IAllowAnonymous>() != null)
+        {
+            return AuthenticateResult.NoResult();
+        }
+
         if (!Request.Headers.TryGetValue("Authorization", out var authHeader))
         {
             authHeader = string.Empty;

MareSynchronosServer/MareSynchronosShared/Authentication/SecretKeyAuthenticatorService.cs

@@ -1,6 +1,7 @@
 using System.Collections.Concurrent;
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Services;
 using MareSynchronosShared.Utils;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.DependencyInjection;
@@ -13,25 +14,15 @@ public class SecretKeyAuthenticatorService
 {
     private readonly MareMetrics _metrics;
     private readonly IServiceScopeFactory _serviceScopeFactory;
+    private readonly IConfigurationService<MareConfigurationAuthBase> _configurationService;
     private readonly ILogger<SecretKeyAuthenticatorService> _logger;
     private readonly ConcurrentDictionary<string, SecretKeyAuthReply> _cachedPositiveResponses = new(StringComparer.Ordinal);
     private readonly ConcurrentDictionary<string, SecretKeyFailedAuthorization?> _failedAuthorizations = new(StringComparer.Ordinal);
-    private readonly int _failedAttemptsForTempBan;
-    private readonly int _tempBanMinutes;
-    private readonly List<string> _whitelistedIps;
 
-    public SecretKeyAuthenticatorService(MareMetrics metrics, IServiceScopeFactory serviceScopeFactory, IOptions<MareConfigurationAuthBase> configuration, ILogger<SecretKeyAuthenticatorService> logger)
+    public SecretKeyAuthenticatorService(MareMetrics metrics, IServiceScopeFactory serviceScopeFactory, IConfigurationService<MareConfigurationAuthBase> configuration, ILogger<SecretKeyAuthenticatorService> logger)
     {
         _logger = logger;
-        var config = configuration.Value;
-        _failedAttemptsForTempBan = config.FailedAuthForTempBan;
-        _tempBanMinutes = config.TempBanDurationInMinutes;
-        _whitelistedIps = config.WhitelistedIps;
-        foreach (var ip in _whitelistedIps)
-        {
-            logger.LogInformation("Whitelisted IP: " + ip);
-        }
-
+        _configurationService = configuration;
         _metrics = metrics;
         _serviceScopeFactory = serviceScopeFactory;
     }
@@ -46,7 +37,8 @@ public class SecretKeyAuthenticatorService
             return cachedPositiveResponse;
         }
 
-        if (_failedAuthorizations.TryGetValue(ip, out var existingFailedAuthorization) && existingFailedAuthorization.FailedAttempts > _failedAttemptsForTempBan)
+        if (_failedAuthorizations.TryGetValue(ip, out var existingFailedAuthorization)
+            && existingFailedAuthorization.FailedAttempts > _configurationService.GetValueOrDefault(nameof(MareConfigurationAuthBase.FailedAuthForTempBan), 5))
         {
             if (existingFailedAuthorization.ResetTask == null)
             {
@@ -54,7 +46,7 @@ public class SecretKeyAuthenticatorService
 
                 existingFailedAuthorization.ResetTask = Task.Run(async () =>
                 {
-                    await Task.Delay(TimeSpan.FromMinutes(_tempBanMinutes)).ConfigureAwait(false);
+                    await Task.Delay(TimeSpan.FromMinutes(_configurationService.GetValueOrDefault(nameof(MareConfigurationAuthBase.TempBanDurationInMinutes), 5))).ConfigureAwait(false);
 
                 }).ContinueWith((t) =>
                 {
@@ -96,7 +88,8 @@ public class SecretKeyAuthenticatorService
         _metrics.IncCounter(MetricsAPI.CounterAuthenticationFailures);
 
         _logger.LogWarning("Failed authorization from {ip}", ip);
-        if (!_whitelistedIps.Any(w => ip.Contains(w, StringComparison.OrdinalIgnoreCase)))
+        var whitelisted = _configurationService.GetValueOrDefault(nameof(MareConfigurationAuthBase.WhitelistedIps), new List<string>());
+        if (!whitelisted.Any(w => ip.Contains(w, StringComparison.OrdinalIgnoreCase)))
         {
             if (_failedAuthorizations.TryGetValue(ip, out var auth))
             {

MareSynchronosServer/MareSynchronosShared/Extensions.cs

@@ -1,6 +1,6 @@
 using Microsoft.AspNetCore.Http;
 
-namespace MareSynchronosServer;
+namespace MareSynchronosShared;
 
 public static class Extensions
 {

MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto

@@ -20,6 +20,19 @@ service IdentificationService {
 	rpc ReceiveStreamIdentStatusChange (ServerMessage) returns (stream IdentChange);
 }
 
+service ConfigurationService {
+	rpc GetConfigurationEntry (KeyMessage) returns (ValueMessage);
+}
+
+message KeyMessage {
+	string key = 1;
+	string default = 2;
+}
+
+message ValueMessage {
+	string value = 1;
+}
+
 message Empty { }
 
 message MultiUidMessage {

MareSynchronosServer/MareSynchronosShared/Services/GrpcConfigurationService.cs

@@ -0,0 +1,30 @@
+using Grpc.Core;
+using MareSynchronosShared.Protos;
+using MareSynchronosShared.Utils;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace MareSynchronosShared.Services;
+
+[Authorize]
+[AllowAnonymous]
+public class GrpcConfigurationService<T> : ConfigurationService.ConfigurationServiceBase where T : class, IMareConfiguration
+{
+    private readonly T _config;
+    private readonly ILogger<GrpcConfigurationService<T>> logger;
+
+    public GrpcConfigurationService(IOptions<T> config, ILogger<GrpcConfigurationService<T>> logger)
+    {
+        _config = config.Value;
+        this.logger = logger;
+    }
+
+    [AllowAnonymous]
+    public override Task<ValueMessage> GetConfigurationEntry(KeyMessage request, ServerCallContext context)
+    {
+        logger.LogInformation("Remote requested {key}", request.Key);
+        var returnVal = _config.SerializeValue(request.Key, request.Default);
+        return Task.FromResult(new ValueMessage() { Value = returnVal });
+    }
+}

MareSynchronosServer/MareSynchronosShared/Services/IConfigurationService.cs

@@ -0,0 +1,11 @@
+using MareSynchronosShared.Utils;
+
+namespace MareSynchronosShared.Services;
+
+public interface IConfigurationService<T> where T : class, IMareConfiguration
+{
+    bool IsMain { get; }
+    T1 GetValue<T1>(string key);
+    T1 GetValueOrDefault<T1>(string key, T1 defaultValue);
+    string ToString();
+}

MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceClient.cs

@@ -0,0 +1,126 @@
+using Grpc.Net.ClientFactory;
+using MareSynchronosShared.Protos;
+using MareSynchronosShared.Utils;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using System.Collections.Concurrent;
+using System.Globalization;
+using System.Text;
+using System.Text.Json;
+using static MareSynchronosShared.Protos.ConfigurationService;
+
+namespace MareSynchronosShared.Services;
+
+public class MareConfigurationServiceClient<T> : IConfigurationService<T> where T : class, IMareConfiguration
+{
+    internal record RemoteCachedEntry(object Value, DateTime Inserted);
+
+    private readonly T _config;
+    private readonly ConcurrentDictionary<string, RemoteCachedEntry> _cachedRemoteProperties = new(StringComparer.Ordinal);
+    private readonly ILogger<MareConfigurationServiceClient<T>> _logger;
+    private readonly ConfigurationServiceClient _configurationServiceClient;
+
+    public MareConfigurationServiceClient(ILogger<MareConfigurationServiceClient<T>> logger, IOptions<T> config, ConfigurationServiceClient configurationServiceClient)
+    {
+        _config = config.Value;
+        _logger = logger;
+        _configurationServiceClient = configurationServiceClient;
+    }
+
+    public MareConfigurationServiceClient(ILogger<MareConfigurationServiceClient<T>> logger, IOptions<T> config, GrpcClientFactory grpcClientFactory, string grpcClientName)
+    {
+        _config = config.Value;
+        _logger = logger;
+        _configurationServiceClient = grpcClientFactory.CreateClient<ConfigurationServiceClient>(grpcClientName);
+    }
+
+    public bool IsMain => false;
+
+    public T1 GetValueOrDefault<T1>(string key, T1 defaultValue)
+    {
+        var prop = _config.GetType().GetProperty(key);
+        if (prop == null) return defaultValue;
+        if (prop.PropertyType != typeof(T1)) throw new InvalidCastException($"Invalid Cast: Property {key} is {prop.PropertyType}, wanted: {typeof(T1)}");
+        bool isRemote = prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), inherit: true).Any();
+        if (isRemote)
+        {
+            bool isCurrent = false;
+            if (_cachedRemoteProperties.TryGetValue(key, out var existingEntry) && existingEntry.Inserted > DateTime.Now - TimeSpan.FromMinutes(30))
+            {
+                isCurrent = true;
+            }
+
+            if (!isCurrent)
+            {
+                var result = GetValueFromGrpc(key, defaultValue, prop.PropertyType);
+                if (result == null) return defaultValue;
+                _cachedRemoteProperties[key] = result;
+                return (T1)_cachedRemoteProperties[key].Value;
+            }
+        }
+
+        var value = prop.GetValue(_config);
+        return (T1)value;
+    }
+
+    private RemoteCachedEntry? GetValueFromGrpc(string key, object defaultValue, Type t)
+    {
+        // grab stuff from grpc
+        try
+        {
+            _logger.LogInformation("Getting {key} from Grpc", key);
+            var response = _configurationServiceClient.GetConfigurationEntry(new KeyMessage { Key = key, Default = Convert.ToString(defaultValue, CultureInfo.InvariantCulture) });
+            _logger.LogInformation("Grpc Response for {key} = {value}", key, response.Value);
+            return new RemoteCachedEntry(JsonSerializer.Deserialize(response.Value, t), DateTime.Now);
+        }
+        catch
+        {
+            return null;
+        }
+    }
+
+    public T1 GetValue<T1>(string key)
+    {
+        var prop = _config.GetType().GetProperty(key);
+        if (prop == null) throw new KeyNotFoundException(key);
+        if (prop.PropertyType != typeof(T1)) throw new InvalidCastException($"Invalid Cast: Property {key} is {prop.PropertyType}, wanted: {typeof(T1)}");
+        bool isRemote = prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), inherit: true).Any();
+        if (isRemote)
+        {
+            bool isCurrent = false;
+            if (_cachedRemoteProperties.TryGetValue(key, out var existingEntry) && existingEntry.Inserted > DateTime.Now - TimeSpan.FromMinutes(30))
+            {
+                isCurrent = true;
+            }
+
+            if (!isCurrent)
+            {
+                var result = GetValueFromGrpc(key, null, prop.PropertyType);
+                if (result == null) throw new KeyNotFoundException(key);
+                _cachedRemoteProperties[key] = result;
+            }
+
+            if (!_cachedRemoteProperties.ContainsKey(key)) throw new KeyNotFoundException(key);
+
+            return (T1)_cachedRemoteProperties[key].Value;
+        }
+
+        var value = prop.GetValue(_config);
+        return (T1)value;
+    }
+
+    public override string ToString()
+    {
+        var props = _config.GetType().GetProperties();
+        StringBuilder sb = new();
+        foreach (var prop in props)
+        {
+            var isRemote = prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any();
+            var mi = GetType().GetMethod(nameof(GetValue)).MakeGenericMethod(prop.PropertyType);
+            var val = mi.Invoke(this, new[] { prop.Name });
+            var value = isRemote ? val : prop.GetValue(_config);
+            sb.AppendLine($"{prop.Name} (IsRemote: {isRemote}) => {value}");
+        }
+        return sb.ToString();
+    }
+}

MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceServer.cs

@@ -0,0 +1,37 @@
+using MareSynchronosShared.Utils;
+using Microsoft.Extensions.Options;
+using System.Text;
+
+namespace MareSynchronosShared.Services;
+
+public class MareConfigurationServiceServer<T> : IConfigurationService<T> where T : class, IMareConfiguration
+{
+    private readonly T _config;
+    public bool IsMain => true;
+
+    public MareConfigurationServiceServer(IOptions<T> config)
+    {
+        _config = config.Value;
+    }
+
+    public T1 GetValueOrDefault<T1>(string key, T1 defaultValue)
+    {
+        return _config.GetValueOrDefault<T1>(key, defaultValue);
+    }
+
+    public T1 GetValue<T1>(string key)
+    {
+        return _config.GetValue<T1>(key);
+    }
+
+    public override string ToString()
+    {
+        var props = _config.GetType().GetProperties();
+        StringBuilder sb = new();
+        foreach (var prop in props)
+        {
+            sb.AppendLine($"{prop.Name} (IsRemote: {prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any()}) => {prop.GetValue(_config)}");
+        }
+        return sb.ToString();
+    }
+}

MareSynchronosServer/MareSynchronosShared/Utils/IMareConfiguration.cs

@@ -0,0 +1,8 @@
+namespace MareSynchronosShared.Utils;
+
+public interface IMareConfiguration
+{
+    T GetValueOrDefault<T>(string key, T defaultValue);
+    T GetValue<T>(string key);
+    string SerializeValue(string key, string defaultValue);
+}

MareSynchronosServer/MareSynchronosShared/Utils/MareConfigurationAuthBase.cs

@@ -0,0 +1,25 @@
+using System.Text;
+
+namespace MareSynchronosShared.Utils;
+
+public class MareConfigurationAuthBase : MareConfigurationBase
+{
+    public Uri MainServerGrpcAddress { get; set; } = null;
+    [RemoteConfiguration]
+    public int FailedAuthForTempBan { get; set; } = 5;
+    [RemoteConfiguration]
+    public int TempBanDurationInMinutes { get; set; } = 5;
+    [RemoteConfiguration]
+    public List<string> WhitelistedIps { get; set; } = new();
+
+    public override string ToString()
+    {
+        StringBuilder sb = new();
+        sb.AppendLine(base.ToString());
+        sb.AppendLine($"{nameof(MainServerGrpcAddress)} => {MainServerGrpcAddress}");
+        sb.AppendLine($"{nameof(FailedAuthForTempBan)} => {FailedAuthForTempBan}");
+        sb.AppendLine($"{nameof(TempBanDurationInMinutes)} => {TempBanDurationInMinutes}");
+        sb.AppendLine($"{nameof(WhitelistedIps)} => {string.Join(", ", WhitelistedIps)}");
+        return sb.ToString();
+    }
+}

MareSynchronosServer/MareSynchronosShared/Utils/MareConfigurationBase.cs

@@ -1,32 +1,46 @@
-using System.Text;
+using System.Globalization;
+using System.Reflection;
+using System.Text;
+using System.Text.Json;
 
 namespace MareSynchronosShared.Utils;
 
-public class MareConfigurationBase
+public class MareConfigurationBase : IMareConfiguration
 {
     public int DbContextPoolSize { get; set; } = 100;
+    public string ShardName { get; set; } = string.Empty;
+    public int MetricsPort { get; set; } = 4981;
+
+    public T GetValue<T>(string key)
+    {
+        var prop = GetType().GetProperty(key);
+        if (prop == null) throw new KeyNotFoundException(key);
+        if (prop.PropertyType != typeof(T)) throw new ArgumentException($"Requested {key} with T:{typeof(T)}, where {key} is {prop.PropertyType}");
+        return (T)prop.GetValue(this);
+    }
+
+    public T GetValueOrDefault<T>(string key, T defaultValue)
+    {
+        var prop = GetType().GetProperty(key);
+        if (prop.PropertyType != typeof(T)) throw new ArgumentException($"Requested {key} with T:{typeof(T)}, where {key} is {prop.PropertyType}");
+        if (prop == null) return defaultValue;
+        return (T)prop.GetValue(this);
+    }
+
+    public string SerializeValue(string key, string defaultValue)
+    {
+        var prop = GetType().GetProperty(key);
+        if (prop == null) return defaultValue;
+        if (prop.GetCustomAttribute<RemoteConfigurationAttribute>() == null) return defaultValue;
+        return JsonSerializer.Serialize(prop.GetValue(this), prop.PropertyType);
+    }
 
     public override string ToString()
     {
         StringBuilder sb = new();
+        sb.AppendLine(base.ToString());
+        sb.AppendLine($"{nameof(ShardName)} => {ShardName}");
         sb.AppendLine($"{nameof(DbContextPoolSize)} => {DbContextPoolSize}");
         return sb.ToString();
     }
-}
-
-public class MareConfigurationAuthBase : MareConfigurationBase
-{
-    public int DbContextPoolSize { get; set; } = 100;
-    public int FailedAuthForTempBan { get; set; } = 5;
-    public int TempBanDurationInMinutes { get; set; } = 5;
-    public List<string> WhitelistedIps { get; set; } = new();
-
-    public override string ToString()
-    {
-        StringBuilder sb = new();
-        sb.AppendLine($"{nameof(FailedAuthForTempBan)} => {FailedAuthForTempBan}");
-        sb.AppendLine($"{nameof(TempBanDurationInMinutes)} => {TempBanDurationInMinutes}");
-        sb.AppendLine($"{nameof(WhitelistedIps)} => {string.Join(", ", WhitelistedIps)}");
-        return sb.ToString();
-    }
 }

MareSynchronosServer/MareSynchronosShared/Utils/RemoteConfigurationAttribute.cs

@@ -0,0 +1,4 @@
+namespace MareSynchronosShared.Utils;
+
+[AttributeUsage(AttributeTargets.Property)]
+public class RemoteConfigurationAttribute : Attribute { }

MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs

@@ -1,32 +1,39 @@
-using MareSynchronosShared.Utils;
-using System;
-using System.Text;
+using System.Text;
 
-namespace MareSynchronosServer;
+namespace MareSynchronosShared.Utils;
 
 public class ServerConfiguration : MareConfigurationAuthBase
 {
-    public Uri CdnFullUrl { get; set; } = null;
-    public Uri ServiceAddress { get; set; } = null;
-    public Uri StaticFileServiceAddress { get; set; } = null;
     public string RedisConnectionString { get; set; } = string.Empty;
+
+    [RemoteConfiguration]
+    public Uri CdnFullUrl { get; set; } = null;
+    [RemoteConfiguration]
+    public Uri StaticFileServiceAddress { get; set; } = null;
+    [RemoteConfiguration]
     public int MaxExistingGroupsByUser { get; set; } = 3;
+    [RemoteConfiguration]
     public int MaxJoinedGroupsByUser { get; set; } = 6;
+    [RemoteConfiguration]
     public int MaxGroupUserCount { get; set; } = 100;
-    public string ShardName { get; set; } = string.Empty;
+    [RemoteConfiguration]
+    public bool PurgeUnusedAccounts { get; set; } = false;
+    [RemoteConfiguration]
+    public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14;
 
     public override string ToString()
     {
         StringBuilder sb = new();
         sb.AppendLine(base.ToString());
-        sb.AppendLine($"{nameof(ShardName)} => {ShardName}");
+        sb.AppendLine($"{nameof(MainServerGrpcAddress)} => {MainServerGrpcAddress}");
         sb.AppendLine($"{nameof(CdnFullUrl)} => {CdnFullUrl}");
-        sb.AppendLine($"{nameof(ServiceAddress)} => {ServiceAddress}");
         sb.AppendLine($"{nameof(StaticFileServiceAddress)} => {StaticFileServiceAddress}");
         sb.AppendLine($"{nameof(RedisConnectionString)} => {RedisConnectionString}");
         sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");
         sb.AppendLine($"{nameof(MaxJoinedGroupsByUser)} => {MaxJoinedGroupsByUser}");
         sb.AppendLine($"{nameof(MaxGroupUserCount)} => {MaxGroupUserCount}");
+        sb.AppendLine($"{nameof(PurgeUnusedAccounts)} => {PurgeUnusedAccounts}");
+        sb.AppendLine($"{nameof(PurgeUnusedAccountsPeriodInDays)} => {PurgeUnusedAccountsPeriodInDays}");
         return sb.ToString();
     }
 }

MareSynchronosServer/MareSynchronosShared/Utils/ServicesConfiguration.cs

@@ -0,0 +1,18 @@
+using System.Text;
+
+namespace MareSynchronosShared.Utils;
+
+public class ServicesConfiguration : MareConfigurationBase
+{
+    public string DiscordBotToken { get; set; } = string.Empty;
+    public Uri MainServerGrpcAddress { get; set; } = null;
+
+    public override string ToString()
+    {
+        StringBuilder sb = new();
+        sb.AppendLine(base.ToString());
+        sb.AppendLine($"{nameof(DiscordBotToken)} => {DiscordBotToken}");
+        sb.AppendLine($"{nameof(MainServerGrpcAddress)} => {MainServerGrpcAddress}");
+        return sb.ToString();
+    }
+}

MareSynchronosServer/MareSynchronosShared/Utils/SharedDbFunctions.cs

@@ -1,11 +1,68 @@
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Models;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
 
 namespace MareSynchronosShared.Utils;
 
 public static class SharedDbFunctions
 {
+    public static async Task PurgeUser(ILogger _logger, User user, MareDbContext dbContext, int maxGroupsByUser)
+    {
+        _logger.LogInformation("Purging user: {uid}", user.UID);
+
+        var lodestone = dbContext.LodeStoneAuth.SingleOrDefault(a => a.User.UID == user.UID);
+
+        if (lodestone != null)
+        {
+            dbContext.Remove(lodestone);
+        }
+
+        var auth = dbContext.Auth.Single(a => a.UserUID == user.UID);
+
+        var userFiles = dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == user.UID).ToList();
+        dbContext.Files.RemoveRange(userFiles);
+
+        var ownPairData = dbContext.ClientPairs.Where(u => u.User.UID == user.UID).ToList();
+        dbContext.ClientPairs.RemoveRange(ownPairData);
+        var otherPairData = dbContext.ClientPairs.Include(u => u.User)
+            .Where(u => u.OtherUser.UID == user.UID).ToList();
+        dbContext.ClientPairs.RemoveRange(otherPairData);
+
+        var userJoinedGroups = await dbContext.GroupPairs.Include(g => g.Group).Where(u => u.GroupUserUID == user.UID).ToListAsync().ConfigureAwait(false);
+
+        foreach (var userGroupPair in userJoinedGroups)
+        {
+            bool ownerHasLeft = string.Equals(userGroupPair.Group.OwnerUID, user.UID, StringComparison.Ordinal);
+
+            if (ownerHasLeft)
+            {
+                var groupPairs = await dbContext.GroupPairs.Where(g => g.GroupGID == userGroupPair.GroupGID && g.GroupUserUID != user.UID).ToListAsync().ConfigureAwait(false);
+
+                if (!groupPairs.Any())
+                {
+                    _logger.LogInformation("Group {gid} has no new owner, deleting", userGroupPair.GroupGID);
+                    dbContext.Groups.Remove(userGroupPair.Group);
+                }
+                else
+                {
+                    _ = await MigrateOrDeleteGroup(dbContext, userGroupPair.Group, groupPairs, maxGroupsByUser).ConfigureAwait(false);
+                }
+            }
+
+            dbContext.GroupPairs.Remove(userGroupPair);
+
+            await dbContext.SaveChangesAsync().ConfigureAwait(false);
+        }
+
+        _logger.LogInformation("User purged: {uid}", user.UID);
+
+        dbContext.Auth.Remove(auth);
+        dbContext.Users.Remove(user);
+
+        await dbContext.SaveChangesAsync().ConfigureAwait(false);
+    }
+
     public static async Task<(bool, string)> MigrateOrDeleteGroup(MareDbContext context, Group group, List<GroupPair> groupPairs, int maxGroupsByUser)
     {
         bool groupHasMigrated = false;

MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs

@@ -3,18 +3,21 @@ using System.Text;
 
 namespace MareSynchronosStaticFilesServer;
 
-public class StaticFilesServerConfiguration : MareConfigurationAuthBase
+public class StaticFilesServerConfiguration : MareConfigurationBase
 {
+    public Uri FileServerGrpcAddress { get; set; } = null;
     public int ForcedDeletionOfFilesAfterHours { get; set; } = -1;
     public double CacheSizeHardLimitInGiB { get; set; } = -1;
-    public int UnusedFileRetentionPeriodInDays { get; set; } = -1;
+    public int UnusedFileRetentionPeriodInDays { get; set; } = 14;
     public string CacheDirectory { get; set; }
     public Uri? RemoteCacheSourceUri { get; set; } = null;
-
+    public Uri MainServerGrpcAddress { get; set; } = null;
     public override string ToString()
     {
         StringBuilder sb = new();
         sb.AppendLine(base.ToString());
+        sb.AppendLine($"{nameof(FileServerGrpcAddress)} => {FileServerGrpcAddress}");
+        sb.AppendLine($"{nameof(MainServerGrpcAddress)} => {MainServerGrpcAddress}");
         sb.AppendLine($"{nameof(ForcedDeletionOfFilesAfterHours)} => {ForcedDeletionOfFilesAfterHours}");
         sb.AppendLine($"{nameof(CacheSizeHardLimitInGiB)} => {CacheSizeHardLimitInGiB}");
         sb.AppendLine($"{nameof(UnusedFileRetentionPeriodInDays)} => {UnusedFileRetentionPeriodInDays}");

MareSynchronosServer/MareSynchronosStaticFilesServer/CachedFileProvider.cs

@@ -1,4 +1,5 @@
 using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Services;
 using Microsoft.Extensions.Options;
 using System.Collections.Concurrent;
 
@@ -14,13 +15,13 @@ public class CachedFileProvider
     private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
     private bool IsMainServer => _remoteCacheSourceUri == null;
 
-    public CachedFileProvider(IOptions<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics)
+    public CachedFileProvider(IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService, MareMetrics metrics)
     {
         _logger = logger;
         _fileStatisticsService = fileStatisticsService;
         _metrics = metrics;
-        _remoteCacheSourceUri = configuration.Value.RemoteCacheSourceUri;
-        _basePath = configuration.Value.CacheDirectory;
+        _remoteCacheSourceUri = configuration.GetValueOrDefault<Uri>(nameof(StaticFilesServerConfiguration.RemoteCacheSourceUri), null);
+        _basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
     }
 
     public async Task<FileStream?> GetFileStream(string hash, string auth)
@@ -79,6 +80,21 @@ public class CachedFileProvider
 
         _fileStatisticsService.LogFile(hash, fi.Length);
 
-        return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read);
+        int attempts = 0;
+        while (attempts < 5)
+        {
+            try
+            {
+                return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read);
+            }
+            catch (Exception ex)
+            {
+                attempts++;
+                _logger.LogWarning(ex, "Error opening file, retrying");
+                await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
+            }
+        }
+
+        throw new IOException("Could not open file " + fi.FullName);
     }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/FileCleanupService.cs

@@ -2,6 +2,7 @@
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
 using MareSynchronosShared.Models;
+using MareSynchronosShared.Services;
 using Microsoft.Extensions.Options;
 
 namespace MareSynchronosStaticFilesServer;
@@ -11,19 +12,19 @@ public class FileCleanupService : IHostedService
     private readonly MareMetrics _metrics;
     private readonly ILogger<FileCleanupService> _logger;
     private readonly IServiceProvider _services;
-    private readonly StaticFilesServerConfiguration _configuration;
+    private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
     private readonly bool _isMainServer;
     private readonly string _cacheDir;
     private CancellationTokenSource _cleanupCts;
 
-    public FileCleanupService(MareMetrics metrics, ILogger<FileCleanupService> logger, IServiceProvider services, IOptions<StaticFilesServerConfiguration> configuration)
+    public FileCleanupService(MareMetrics metrics, ILogger<FileCleanupService> logger, IServiceProvider services, IConfigurationService<StaticFilesServerConfiguration> configuration)
     {
         _metrics = metrics;
         _logger = logger;
         _services = services;
-        _configuration = configuration.Value;
-        _isMainServer = _configuration.RemoteCacheSourceUri == null;
-        _cacheDir = _configuration.CacheDirectory;
+        _configuration = configuration;
+        _isMainServer = configuration.IsMain;
+        _cacheDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
     }
 
     public Task StartAsync(CancellationToken cancellationToken)
@@ -60,26 +61,32 @@ public class FileCleanupService : IHostedService
                 await dbContext.SaveChangesAsync(ct).ConfigureAwait(false);
             }
 
-            _logger.LogInformation("File Cleanup Complete, next run at {date}", DateTime.Now.Add(TimeSpan.FromMinutes(10)));
-            await Task.Delay(TimeSpan.FromMinutes(10), ct).ConfigureAwait(false);
+            var now = DateTime.Now;
+            TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
+            TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % 10, 0);
+            var span = futureTime.AddMinutes(10) - currentTime;
+
+            _logger.LogInformation("File Cleanup Complete, next run at {date}", now.Add(span));
+            await Task.Delay(span, ct).ConfigureAwait(false);
         }
     }
 
     private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct)
     {
-        if (_configuration.CacheSizeHardLimitInGiB <= 0)
+        var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);
+        if (sizeLimit <= 0)
         {
             return;
         }
 
         try
         {
-            _logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", _configuration.CacheSizeHardLimitInGiB);
+            _logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", sizeLimit);
             var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories)
                 .Select(f => new FileInfo(f)).ToList()
                 .OrderBy(f => f.LastAccessTimeUtc).ToList();
             var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
-            long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(_configuration.CacheSizeHardLimitInGiB).Bytes;
+            long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(sizeLimit).Bytes;
             while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested)
             {
                 var oldestFile = allLocalFiles[0];
@@ -106,15 +113,18 @@ public class FileCleanupService : IHostedService
     {
         try
         {
-            _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", _configuration.UnusedFileRetentionPeriodInDays);
-            if (_configuration.ForcedDeletionOfFilesAfterHours > 0)
+            var unusedRetention = _configuration.GetValueOrDefault<int>(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
+            var forcedDeletionAfterHours = _configuration.GetValueOrDefault<int>(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
+
+            _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", unusedRetention);
+            if (forcedDeletionAfterHours > 0)
             {
-                _logger.LogInformation("Cleaning up files written to longer than {hours}h ago", _configuration.ForcedDeletionOfFilesAfterHours);
+                _logger.LogInformation("Cleaning up files written to longer than {hours}h ago", forcedDeletionAfterHours);
             }
 
             // clean up files in DB but not on disk or last access is expired
-            var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(_configuration.UnusedFileRetentionPeriodInDays));
-            var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(_configuration.ForcedDeletionOfFilesAfterHours));
+            var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention));
+            var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
             var allFiles = dbContext.Files.ToList();
             foreach (var fileCache in allFiles.Where(f => f.Uploaded))
             {
@@ -133,7 +143,7 @@ public class FileCleanupService : IHostedService
                     if (_isMainServer)
                         dbContext.Files.Remove(fileCache);
                 }
-                else if (file != null && _configuration.ForcedDeletionOfFilesAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
+                else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
                 {
                     _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
                     _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
@@ -147,24 +157,7 @@ public class FileCleanupService : IHostedService
             }
 
             // clean up files that are on disk but not in DB for some reason
-            if (_isMainServer)
-            {
-                var allFilesHashes = new HashSet<string>(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal);
-                DirectoryInfo dir = new(_cacheDir);
-                var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories);
-                foreach (var file in allFilesInDir)
-                {
-                    if (!allFilesHashes.Contains(file.Name.ToUpperInvariant()))
-                    {
-                        _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
-                        _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
-                        file.Delete();
-                        _logger.LogInformation("File not in DB, deleting: {fileName}", file.Name);
-                    }
-
-                    ct.ThrowIfCancellationRequested();
-                }
-            }
+            CleanUpOrphanedFiles(allFiles, ct);
         }
         catch (Exception ex)
         {
@@ -172,6 +165,28 @@ public class FileCleanupService : IHostedService
         }
     }
 
+    private void CleanUpOrphanedFiles(List<FileCache> allFiles, CancellationToken ct)
+    {
+        if (_isMainServer)
+        {
+            var allFilesHashes = new HashSet<string>(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal);
+            DirectoryInfo dir = new(_cacheDir);
+            var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories);
+            foreach (var file in allFilesInDir)
+            {
+                if (!allFilesHashes.Contains(file.Name.ToUpperInvariant()))
+                {
+                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
+                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
+                    file.Delete();
+                    _logger.LogInformation("File not in DB, deleting: {fileName}", file.Name);
+                }
+
+                ct.ThrowIfCancellationRequested();
+            }
+        }
+    }
+
     public Task StopAsync(CancellationToken cancellationToken)
     {
         _cleanupCts.Cancel();

MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs

@@ -51,7 +51,13 @@ public class FileStatisticsService : IHostedService
             _pastHourFiles = new(StringComparer.Ordinal);
             _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHour, 0);
             _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastHourSize, 0);
-            await Task.Delay(TimeSpan.FromHours(1), _resetCancellationTokenSource.Token).ConfigureAwait(false);
+
+            var now = DateTime.UtcNow;
+            TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
+            TimeOnly futureTime = new(now.Hour, 0, 0);
+            var span = futureTime.AddHours(1) - currentTime;
+
+            await Task.Delay(span, _resetCancellationTokenSource.Token).ConfigureAwait(false);
         }
     }
 
@@ -64,7 +70,13 @@ public class FileStatisticsService : IHostedService
             _pastDayFiles = new(StringComparer.Ordinal);
             _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDay, 0);
             _metrics.SetGaugeTo(MetricsAPI.GaugeFilesUniquePastDaySize, 0);
-            await Task.Delay(TimeSpan.FromDays(1), _resetCancellationTokenSource.Token).ConfigureAwait(false);
+
+            var now = DateTime.UtcNow;
+            TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
+            TimeOnly futureTime = new(0, 0, 0);
+            var span = futureTime - currentTime;
+
+            await Task.Delay(span, _resetCancellationTokenSource.Token).ConfigureAwait(false);
         }
     }
 

MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs

@@ -18,7 +18,7 @@ public class FilesController : Controller
     [HttpGet("{fileId}")]
     public async Task<IActionResult> GetFile(string fileId)
     {
-        var authedUser = HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, ClaimTypes.NameIdentifier, System.StringComparison.Ordinal))?.Value ?? "Unknown";
+        var authedUser = HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, ClaimTypes.NameIdentifier, StringComparison.Ordinal))?.Value ?? "Unknown";
         _logger.LogInformation($"GetFile:{authedUser}:{fileId}");
 
         var fs = await _cachedFileProvider.GetFileStream(fileId, Request.Headers["Authorization"]);

MareSynchronosServer/MareSynchronosStaticFilesServer/GrpcFileService.cs

@@ -2,8 +2,8 @@
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
 using MareSynchronosShared.Protos;
+using MareSynchronosShared.Services;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.Options;
 
 namespace MareSynchronosStaticFilesServer;
 
@@ -14,9 +14,9 @@ public class GrpcFileService : FileService.FileServiceBase
     private readonly ILogger<GrpcFileService> _logger;
     private readonly MareMetrics _metricsClient;
 
-    public GrpcFileService(MareDbContext mareDbContext, IOptions<StaticFilesServerConfiguration> configuration, ILogger<GrpcFileService> logger, MareMetrics metricsClient)
+    public GrpcFileService(MareDbContext mareDbContext, IConfigurationService<StaticFilesServerConfiguration> configuration, ILogger<GrpcFileService> logger, MareMetrics metricsClient)
     {
-        _basePath = configuration.Value.CacheDirectory;
+        _basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
         _mareDbContext = mareDbContext;
         _logger = logger;
         _metricsClient = metricsClient;

MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs

@@ -1,3 +1,5 @@
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils;
 using Microsoft.Extensions.Options;
 
 namespace MareSynchronosStaticFilesServer;
@@ -11,10 +13,13 @@ public class Program
 
         using (var scope = host.Services.CreateScope())
         {
-            var options = host.Services.GetService<IOptions<StaticFilesServerConfiguration>>();
+            var options = host.Services.GetService<IConfigurationService<StaticFilesServerConfiguration>>();
+            var optionsServer = host.Services.GetService<IConfigurationService<MareConfigurationAuthBase>>();
             var logger = host.Services.GetService<ILogger<Program>>();
-            logger.LogInformation("Loaded MareSynchronos Static Files Server Configuration");
-            logger.LogInformation(options.Value.ToString());
+            logger.LogInformation("Loaded MareSynchronos Static Files Server Configuration (IsMain: {isMain})", options.IsMain);
+            logger.LogInformation(options.ToString());
+            logger.LogInformation("Loaded MareSynchronos Server Auth Configuration (IsMain: {isMain})", optionsServer.IsMain);
+            logger.LogInformation(optionsServer.ToString());
         }
 
         host.Run();