From b6404a9c1d8314626e4c1915946d5ab3b5df650e Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Sat, 31 Dec 2022 14:28:24 +0100 Subject: [PATCH] add horizontal file sharding based on filename matches --- Docker/Readme.md | 7 ++++--- Docker/run/compose/mare-sharded.yml | 9 ++++++++- Docker/run/config/sharded/haproxy-shards.cfg | 9 --------- .../run/config/sharded/server-shard-main.json | 12 +++++++++++- .../MareSynchronosServer/Hubs/MareHub.Files.cs | 9 +++++++-- .../MareSynchronosServer/Hubs/MareHub.cs | 6 ++++-- .../MareSynchronosServer/Program.cs | 2 ++ .../Services/MareConfigurationServiceClient.cs | 16 +++++++++++++--- .../Services/MareConfigurationServiceServer.cs | 3 +++ .../Utils/CdnShardConfiguration.cs | 17 +++++++++++++++++ .../Utils/ServerConfiguration.cs | 3 +++ 11 files changed, 72 insertions(+), 21 deletions(-) create mode 100644 MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs diff --git a/Docker/Readme.md b/Docker/Readme.md index 60e9a6c..99b3600 100644 --- a/Docker/Readme.md +++ b/Docker/Readme.md @@ -12,18 +12,19 @@ It is possible to build all required images at once by running `docker-build.bat ## 2. Configure ports + token You should set up 2 environment variables that hold server specific configuration and open up ports. -The default ports used through the provided configuration are `6000` for the main server and `6200` for the files downloads. +The default ports used through the provided configuration are `6000` for the main server and `6200` as well as `6201` for the files downloads. Both ports should be open to your computer through your router if you wish to test this with clients. Furthermore there are two environment variables `DEV_MARE_CDNURL` and `DEV_MARE_DISCORDTOKEN` which you are required to set. -`DEV_MARE_CDNURL` should point to `http://:6200/cache/` and `DEV_MARE_DISCORDTOKEN` is an oauth token from a bot you need to create through the Discord bot portal. +`DEV_MARE_CDNURL` should point to `http://:6200/cache/` and `DEV_MARE_DISCORDTOKEN` is an oauth token from a bot you need to create through the Discord bot portal. +You should also set `DEV_MARE_CDNURL2` to `http://:6201/cache/` It is enough to set them as User variables. The compose files refer to those environment variables to overwrite configuration settings for the Server and Services to set those respective values. It is also possible to set those values in the configuration.json files themselves. Without a valid Discord bot you will not be able to register accounts without fumbling around in the PostgreSQL database. ## 3. Run Mare Server The run folder contains two major Mare configurations which is `standalone` and `sharded`. -Both configurations default to port `6000` for the main server connection and `6200` for the files downloads. No HTTPS. +Both configurations default to port `6000` for the main server connection and `6200` for the files downloads. Sharded configuration additionally uses `6201` for downloads. No HTTPS. All `appsettings.json` configurations provided are extensive at the point of writing, note the differences between the shard configurations and the main servers respectively. They can be used as examples if you want to spin up your own servers otherwise. diff --git a/Docker/run/compose/mare-sharded.yml b/Docker/run/compose/mare-sharded.yml index 4d2954b..2b7c55c 100644 --- a/Docker/run/compose/mare-sharded.yml +++ b/Docker/run/compose/mare-sharded.yml @@ -20,7 +20,6 @@ services: restart: always ports: - 6000:6000/tcp - - 6200:6200/tcp volumes: - ../config/sharded/haproxy-shards.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro @@ -35,6 +34,10 @@ services: restart: on-failure environment: MareSynchronos__CdnFullUrl: "${DEV_MARE_CDNURL}" + MareSynchronos__CdnShardConfiguration__0__CdnFullUrl: "${DEV_MARE_CDNURL}" + MareSynchronos__CdnShardConfiguration__0__FileMatch: "^[01234567]" + MareSynchronos__CdnShardConfiguration__1__CdnFullUrl: "${DEV_MARE_CDNURL2}" + MareSynchronos__CdnShardConfiguration__1__FileMatch: "^[89ABCDEF]" volumes: - ../config/sharded/server-shard-main.json:/opt/MareSynchronosServer/appsettings.json - ../log/server-shard-main/:/opt/MareSynchronosServer/logs/:rw @@ -98,6 +101,8 @@ services: - ../log/files-shard-1/:/opt/MareSynchronosStaticFilesServer/logs/:rw - postgres_socket:/var/run/postgresql/:rw - ../data/files-shard-1/:/marecache/:rw + ports: + - 6200:6200/tcp depends_on: - "postgres" - "mare-files" @@ -110,6 +115,8 @@ services: - ../log/files-shard-2/:/opt/MareSynchronosStaticFilesServer/logs/:rw - postgres_socket:/var/run/postgresql/:rw - ../data/files-shard-2/:/marecache/:rw + ports: + - 6201:6200/tcp depends_on: - "postgres" - "mare-files" diff --git a/Docker/run/config/sharded/haproxy-shards.cfg b/Docker/run/config/sharded/haproxy-shards.cfg index 58f6711..2075e1a 100644 --- a/Docker/run/config/sharded/haproxy-shards.cfg +++ b/Docker/run/config/sharded/haproxy-shards.cfg @@ -23,17 +23,8 @@ frontend mare bind :6000 default_backend mare-servers -frontend mare-files - bind :6200 - default_backend mare-files - backend mare-servers balance leastconn cookie SERVER insert indirect nocache server mare1 mare-shard-1:6000 cookie mare1 server mare2 mare-shard-2:6000 cookie mare2 - -backend mare-files - balance roundrobin - server files1 mare-files-shard-1:6200 - server files2 mare-files-shard-2:6200 diff --git a/Docker/run/config/sharded/server-shard-main.json b/Docker/run/config/sharded/server-shard-main.json index 6157142..f21b471 100644 --- a/Docker/run/config/sharded/server-shard-main.json +++ b/Docker/run/config/sharded/server-shard-main.json @@ -41,7 +41,17 @@ "MaxJoinedGroupsByUser": 6, "MaxGroupUserCount": 100, "PurgeUnusedAccounts": false, - "PurgeUnusedAccountsPeriodInDays": 14 + "PurgeUnusedAccountsPeriodInDays": 14, + "CdnShardConfiguration": [ + { + "FileMatch": "^[01234567]", + "CdnFullUrl": "" + }, + { + "FileMatch": "^[89ABCDEF]", + "CdnFullUrl": "" + } + ] }, "AllowedHosts": "*", "Kestrel": { diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs index 092614d..6d0655b 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs @@ -6,6 +6,7 @@ using MareSynchronos.API; using MareSynchronosServer.Utils; using MareSynchronosShared.Models; using MareSynchronosShared.Protos; +using MareSynchronosShared.Utils; using Microsoft.AspNetCore.Authorization; using Microsoft.EntityFrameworkCore; @@ -52,10 +53,14 @@ public partial class MareHub var cacheFile = await _dbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false); + var shardConfig = _configurationService.GetValueOrDefault(nameof(ServerConfiguration.CdnShardConfiguration), new List()); + foreach (var file in cacheFile) { var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase)); - var downloadFile = allFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase)); + + var matchedShardConfig = shardConfig.Find(f => f.FileMatchRegex.Match(file.Hash).Success); + var baseUrl = matchedShardConfig?.CdnFullUrl ?? _mainCdnFullUrl; response.Add(new DownloadFileDto { @@ -64,7 +69,7 @@ public partial class MareHub IsForbidden = forbiddenFile != null, Hash = file.Hash, Size = file.Size, - Url = new Uri(_cdnFullUri, file.Hash.ToUpperInvariant()).ToString() + Url = new Uri(baseUrl, file.Hash.ToUpperInvariant()).ToString() }); } diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs index fbaa55c..6d1cdf8 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs @@ -23,11 +23,12 @@ public partial class MareHub : Hub, IMareHub private readonly IClientIdentificationService _clientIdentService; private readonly MareHubLogger _logger; private readonly MareDbContext _dbContext; - private readonly Uri _cdnFullUri; + private readonly Uri _mainCdnFullUrl; private readonly string _shardName; private readonly int _maxExistingGroupsByUser; private readonly int _maxJoinedGroupsByUser; private readonly int _maxGroupUserCount; + private IConfigurationService _configurationService; public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient, MareDbContext mareDbContext, ILogger logger, SystemInfoService systemInfoService, @@ -37,7 +38,8 @@ public partial class MareHub : Hub, IMareHub _mareMetrics = mareMetrics; _fileServiceClient = fileServiceClient; _systemInfoService = systemInfoService; - _cdnFullUri = configuration.GetValue(nameof(ServerConfiguration.CdnFullUrl)); + _configurationService = configuration; + _mainCdnFullUrl = configuration.GetValue(nameof(ServerConfiguration.CdnFullUrl)); _shardName = configuration.GetValue(nameof(ServerConfiguration.ShardName)); _maxExistingGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxExistingGroupsByUser), 3); _maxJoinedGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxJoinedGroupsByUser), 6); diff --git a/MareSynchronosServer/MareSynchronosServer/Program.cs b/MareSynchronosServer/MareSynchronosServer/Program.cs index dfe857b..6d32264 100644 --- a/MareSynchronosServer/MareSynchronosServer/Program.cs +++ b/MareSynchronosServer/MareSynchronosServer/Program.cs @@ -30,6 +30,8 @@ public class Program context.RemoveRange(unfinishedRegistrations); context.RemoveRange(looseFiles); context.SaveChanges(); + + logger.LogInformation(options.ToString()); } var metrics = services.GetRequiredService(); diff --git a/MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceClient.cs b/MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceClient.cs index cd00b64..dfe9ed7 100644 --- a/MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceClient.cs +++ b/MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceClient.cs @@ -4,11 +4,13 @@ using MareSynchronosShared.Utils; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using System.Collections; using System.Collections.Concurrent; using System.Globalization; using System.Reflection; using System.Text; using System.Text.Json; +using System.Text.RegularExpressions; using static MareSynchronosShared.Protos.ConfigurationService; namespace MareSynchronosShared.Services; @@ -75,9 +77,17 @@ public class MareConfigurationServiceClient : IHostedService, IConfigurationS foreach (var prop in props) { var isRemote = prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any(); - var mi = GetType().GetMethod(nameof(GetValue)).MakeGenericMethod(prop.PropertyType); - var val = mi.Invoke(this, new[] { prop.Name }); - var value = isRemote ? val : prop.GetValue(_config); + var getValueMethod = GetType().GetMethod(nameof(GetValue)).MakeGenericMethod(prop.PropertyType); + var value = isRemote ? getValueMethod.Invoke(this, new[] { prop.Name }) : prop.GetValue(_config); + if (typeof(IEnumerable).IsAssignableFrom(prop.PropertyType) && !typeof(string).IsAssignableFrom(prop.PropertyType)) + { + var enumVal = (IEnumerable)value; + value = string.Empty; + foreach (var listVal in enumVal) + { + value += listVal.ToString() + ", "; + } + } sb.AppendLine($"{prop.Name} (IsRemote: {isRemote}) => {value}"); } return sb.ToString(); diff --git a/MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceServer.cs b/MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceServer.cs index d2b6fc1..2f15060 100644 --- a/MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceServer.cs +++ b/MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceServer.cs @@ -32,6 +32,9 @@ public class MareConfigurationServiceServer : IConfigurationService where { sb.AppendLine($"{prop.Name} (IsRemote: {prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any()}) => {prop.GetValue(_config)}"); } + + sb.AppendLine(_config.ToString()); + return sb.ToString(); } } diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs b/MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs new file mode 100644 index 0000000..601c4a0 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs @@ -0,0 +1,17 @@ +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; + +namespace MareSynchronosShared.Utils; + +public class CdnShardConfiguration +{ + public string FileMatch { get; set; } + [JsonIgnore] + public Regex FileMatchRegex => new Regex(FileMatch); + public Uri CdnFullUrl { get; set; } + + public override string ToString() + { + return CdnFullUrl.ToString() + " == " + FileMatch; + } +} \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs b/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs index e5cb530..4ca0e43 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs @@ -20,6 +20,8 @@ public class ServerConfiguration : MareConfigurationAuthBase public bool PurgeUnusedAccounts { get; set; } = false; [RemoteConfiguration] public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14; + [RemoteConfiguration] + public List CdnShardConfiguration { get; set; } = new(); public override string ToString() { @@ -27,6 +29,7 @@ public class ServerConfiguration : MareConfigurationAuthBase sb.AppendLine(base.ToString()); sb.AppendLine($"{nameof(MainServerGrpcAddress)} => {MainServerGrpcAddress}"); sb.AppendLine($"{nameof(CdnFullUrl)} => {CdnFullUrl}"); + sb.AppendLine($"{nameof(CdnShardConfiguration)} => {string.Join(", ", CdnShardConfiguration.Select(c => c.ToString()))}"); sb.AppendLine($"{nameof(StaticFileServiceAddress)} => {StaticFileServiceAddress}"); sb.AppendLine($"{nameof(RedisConnectionString)} => {RedisConnectionString}"); sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");