t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add horizontal file sharding based on filename matches

Browse Source
This commit is contained in:
rootdarkarchon
2022-12-31 14:28:24 +01:00
parent 2a5e505130
commit b6404a9c1d
11 changed files with 72 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
Docker/Readme.md
View File

@@ -12,18 +12,19 @@ It is possible to build all required images at once by running `docker-build.bat
## 2. Configure ports + token
You should set up 2 environment variables that hold server specific configuration and open up ports.
The default ports used through the provided configuration are `6000` for the main server and `6200` for the files downloads.
The default ports used through the provided configuration are `6000` for the main server and `6200` as well as `6201` for the files downloads.
Both ports should be open to your computer through your router if you wish to test this with clients.
Furthermore there are two environment variables `DEV_MARE_CDNURL` and `DEV_MARE_DISCORDTOKEN` which you are required to set.
`DEV_MARE_CDNURL` should point to `http://<yourip or dyndns>:6200/cache/` and `DEV_MARE_DISCORDTOKEN` is an oauth token from a bot you need to create through the Discord bot portal.
You should also set `DEV_MARE_CDNURL2` to `http://<yourip or dyndns>:6201/cache/`
It is enough to set them as User variables. The compose files refer to those environment variables to overwrite configuration settings for the Server and Services to set those respective values.
It is also possible to set those values in the configuration.json files themselves.
Without a valid Discord bot you will not be able to register accounts without fumbling around in the PostgreSQL database.
## 3. Run Mare Server
The run folder contains two major Mare configurations which is `standalone` and `sharded`.
Both configurations default to port `6000` for the main server connection and `6200` for the files downloads. No HTTPS.
Both configurations default to port `6000` for the main server connection and `6200` for the files downloads. Sharded configuration additionally uses `6201` for downloads. No HTTPS.
All `appsettings.json` configurations provided are extensive at the point of writing, note the differences between the shard configurations and the main servers respectively.
They can be used as examples if you want to spin up your own servers otherwise.

9
Docker/run/compose/mare-sharded.yml
View File

@@ -20,7 +20,6 @@ services:
restart: always
ports:
- 6000:6000/tcp
- 6200:6200/tcp
volumes:
- ../config/sharded/haproxy-shards.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
@@ -35,6 +34,10 @@ services:
restart: on-failure
environment:
MareSynchronos__CdnFullUrl: "${DEV_MARE_CDNURL}"
MareSynchronos__CdnShardConfiguration__0__CdnFullUrl: "${DEV_MARE_CDNURL}"
MareSynchronos__CdnShardConfiguration__0__FileMatch: "^[01234567]"
MareSynchronos__CdnShardConfiguration__1__CdnFullUrl: "${DEV_MARE_CDNURL2}"
MareSynchronos__CdnShardConfiguration__1__FileMatch: "^[89ABCDEF]"
volumes:
- ../config/sharded/server-shard-main.json:/opt/MareSynchronosServer/appsettings.json
- ../log/server-shard-main/:/opt/MareSynchronosServer/logs/:rw
@@ -98,6 +101,8 @@ services:
- ../log/files-shard-1/:/opt/MareSynchronosStaticFilesServer/logs/:rw
- postgres_socket:/var/run/postgresql/:rw
- ../data/files-shard-1/:/marecache/:rw
ports:
- 6200:6200/tcp
depends_on:
- "postgres"
- "mare-files"
@@ -110,6 +115,8 @@ services:
- ../log/files-shard-2/:/opt/MareSynchronosStaticFilesServer/logs/:rw
- postgres_socket:/var/run/postgresql/:rw
- ../data/files-shard-2/:/marecache/:rw
ports:
- 6201:6200/tcp
depends_on:
- "postgres"
- "mare-files"

9
Docker/run/config/sharded/haproxy-shards.cfg
View File

@@ -23,17 +23,8 @@ frontend mare
bind :6000
default_backend mare-servers
frontend mare-files
bind :6200
default_backend mare-files
backend mare-servers
balance leastconn
cookie SERVER insert indirect nocache
server mare1 mare-shard-1:6000 cookie mare1
server mare2 mare-shard-2:6000 cookie mare2
backend mare-files
balance roundrobin
server files1 mare-files-shard-1:6200
server files2 mare-files-shard-2:6200

12
Docker/run/config/sharded/server-shard-main.json
View File

@@ -41,7 +41,17 @@
"MaxJoinedGroupsByUser": 6,
"MaxGroupUserCount": 100,
"PurgeUnusedAccounts": false,
"PurgeUnusedAccountsPeriodInDays": 14
"PurgeUnusedAccountsPeriodInDays": 14,
"CdnShardConfiguration": [
{
"FileMatch": "^[01234567]",
"CdnFullUrl": ""
},
{
"FileMatch": "^[89ABCDEF]",
"CdnFullUrl": ""
}
]
},
"AllowedHosts": "*",
"Kestrel": {

9
MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs
View File

@@ -6,6 +6,7 @@ using MareSynchronos.API;
using MareSynchronosServer.Utils;
using MareSynchronosShared.Models;
using MareSynchronosShared.Protos;
using MareSynchronosShared.Utils;
using Microsoft.AspNetCore.Authorization;
using Microsoft.EntityFrameworkCore;
@@ -52,10 +53,14 @@ public partial class MareHub
var cacheFile = await _dbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false);
var shardConfig = _configurationService.GetValueOrDefault(nameof(ServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>());
foreach (var file in cacheFile)
{
var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
var downloadFile = allFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
var matchedShardConfig = shardConfig.Find(f => f.FileMatchRegex.Match(file.Hash).Success);
var baseUrl = matchedShardConfig?.CdnFullUrl ?? _mainCdnFullUrl;
response.Add(new DownloadFileDto
{
@@ -64,7 +69,7 @@ public partial class MareHub
IsForbidden = forbiddenFile != null,
Hash = file.Hash,
Size = file.Size,
Url = new Uri(_cdnFullUri, file.Hash.ToUpperInvariant()).ToString()
Url = new Uri(baseUrl, file.Hash.ToUpperInvariant()).ToString()
});
}

6
MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs
View File

@@ -23,11 +23,12 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
private readonly IClientIdentificationService _clientIdentService;
private readonly MareHubLogger _logger;
private readonly MareDbContext _dbContext;
private readonly Uri _cdnFullUri;
private readonly Uri _mainCdnFullUrl;
private readonly string _shardName;
private readonly int _maxExistingGroupsByUser;
private readonly int _maxJoinedGroupsByUser;
private readonly int _maxGroupUserCount;
private IConfigurationService<ServerConfiguration> _configurationService;
public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient,
MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService,
@@ -37,7 +38,8 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
_mareMetrics = mareMetrics;
_fileServiceClient = fileServiceClient;
_systemInfoService = systemInfoService;
_cdnFullUri = configuration.GetValue<Uri>(nameof(ServerConfiguration.CdnFullUrl));
_configurationService = configuration;
_mainCdnFullUrl = configuration.GetValue<Uri>(nameof(ServerConfiguration.CdnFullUrl));
_shardName = configuration.GetValue<string>(nameof(ServerConfiguration.ShardName));
_maxExistingGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxExistingGroupsByUser), 3);
_maxJoinedGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxJoinedGroupsByUser), 6);

2
MareSynchronosServer/MareSynchronosServer/Program.cs
View File

@@ -30,6 +30,8 @@ public class Program
context.RemoveRange(unfinishedRegistrations);
context.RemoveRange(looseFiles);
context.SaveChanges();
logger.LogInformation(options.ToString());
}
var metrics = services.GetRequiredService<MareMetrics>();

16
MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceClient.cs
View File

@@ -4,11 +4,13 @@ using MareSynchronosShared.Utils;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System.Collections;
using System.Collections.Concurrent;
using System.Globalization;
using System.Reflection;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using static MareSynchronosShared.Protos.ConfigurationService;
namespace MareSynchronosShared.Services;
@@ -75,9 +77,17 @@ public class MareConfigurationServiceClient<T> : IHostedService, IConfigurationS
foreach (var prop in props)
{
var isRemote = prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any();
var mi = GetType().GetMethod(nameof(GetValue)).MakeGenericMethod(prop.PropertyType);
var val = mi.Invoke(this, new[] { prop.Name });
var value = isRemote ? val : prop.GetValue(_config);
var getValueMethod = GetType().GetMethod(nameof(GetValue)).MakeGenericMethod(prop.PropertyType);
var value = isRemote ? getValueMethod.Invoke(this, new[] { prop.Name }) : prop.GetValue(_config);
if (typeof(IEnumerable).IsAssignableFrom(prop.PropertyType) && !typeof(string).IsAssignableFrom(prop.PropertyType))
{
var enumVal = (IEnumerable)value;
value = string.Empty;
foreach (var listVal in enumVal)
{
value += listVal.ToString() + ", ";
}
}
sb.AppendLine($"{prop.Name} (IsRemote: {isRemote}) => {value}");
}
return sb.ToString();

3
MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceServer.cs
View File

@@ -32,6 +32,9 @@ public class MareConfigurationServiceServer<T> : IConfigurationService<T> where
{
sb.AppendLine($"{prop.Name} (IsRemote: {prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any()}) => {prop.GetValue(_config)}");
}
sb.AppendLine(_config.ToString());
return sb.ToString();
}
}

17
MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs Normal file
View File

@@ -0,0 +1,17 @@
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
namespace MareSynchronosShared.Utils;
public class CdnShardConfiguration
{
public string FileMatch { get; set; }
[JsonIgnore]
public Regex FileMatchRegex => new Regex(FileMatch);
public Uri CdnFullUrl { get; set; }
public override string ToString()
{
return CdnFullUrl.ToString() + " == " + FileMatch;
}
}

3
MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs
View File

@@ -20,6 +20,8 @@ public class ServerConfiguration : MareConfigurationAuthBase
public bool PurgeUnusedAccounts { get; set; } = false;
[RemoteConfiguration]
public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14;
[RemoteConfiguration]
public List<CdnShardConfiguration> CdnShardConfiguration { get; set; } = new();
public override string ToString()
{
@@ -27,6 +29,7 @@ public class ServerConfiguration : MareConfigurationAuthBase
sb.AppendLine(base.ToString());
sb.AppendLine($"{nameof(MainServerGrpcAddress)} => {MainServerGrpcAddress}");
sb.AppendLine($"{nameof(CdnFullUrl)} => {CdnFullUrl}");
sb.AppendLine($"{nameof(CdnShardConfiguration)} => {string.Join(", ", CdnShardConfiguration.Select(c => c.ToString()))}");
sb.AppendLine($"{nameof(StaticFileServiceAddress)} => {StaticFileServiceAddress}");
sb.AppendLine($"{nameof(RedisConnectionString)} => {RedisConnectionString}");
sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");