add horizontal file sharding based on filename matches

Docker/Readme.md

@@ -12,18 +12,19 @@ It is possible to build all required images at once by running `docker-build.bat
 
 ## 2. Configure ports + token
 You should set up 2 environment variables that hold server specific configuration and open up ports.  
-The default ports used through the provided configuration are `6000` for the main server and `6200` for the files downloads.  
+The default ports used through the provided configuration are `6000` for the main server and `6200` as well as `6201` for the files downloads.
 Both ports should be open to your computer through your router if you wish to test this with clients.
 
 Furthermore there are two environment variables `DEV_MARE_CDNURL` and `DEV_MARE_DISCORDTOKEN` which you are required to set.  
 `DEV_MARE_CDNURL` should point to `http://<yourip or dyndns>:6200/cache/` and `DEV_MARE_DISCORDTOKEN` is an oauth token from a bot you need to create through the Discord bot portal. 
+You should also set `DEV_MARE_CDNURL2` to `http://<yourip or dyndns>:6201/cache/`
 It is enough to set them as User variables. The compose files refer to those environment variables to overwrite configuration settings for the Server and Services to set those respective values.  
 It is also possible to set those values in the configuration.json files themselves.  
 Without a valid Discord bot you will not be able to register accounts without fumbling around in the PostgreSQL database.
 
 ## 3. Run Mare Server
 The run folder contains two major Mare configurations which is `standalone` and `sharded`.  
-Both configurations default to port `6000` for the main server connection and `6200` for the files downloads. No HTTPS.  
+Both configurations default to port `6000` for the main server connection and `6200` for the files downloads. Sharded configuration additionally uses `6201` for downloads. No HTTPS.  
 All `appsettings.json` configurations provided are extensive at the point of writing, note the differences between the shard configurations and the main servers respectively.  
 They can be used as examples if you want to spin up your own servers otherwise.
 

Docker/run/compose/mare-sharded.yml

@@ -20,7 +20,6 @@ services:
     restart: always
     ports: 
       - 6000:6000/tcp
-      - 6200:6200/tcp
     volumes:
       - ../config/sharded/haproxy-shards.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
 
@@ -35,6 +34,10 @@ services:
     restart: on-failure
     environment:
       MareSynchronos__CdnFullUrl: "${DEV_MARE_CDNURL}"
+      MareSynchronos__CdnShardConfiguration__0__CdnFullUrl: "${DEV_MARE_CDNURL}"
+      MareSynchronos__CdnShardConfiguration__0__FileMatch: "^[01234567]"
+      MareSynchronos__CdnShardConfiguration__1__CdnFullUrl: "${DEV_MARE_CDNURL2}"
+      MareSynchronos__CdnShardConfiguration__1__FileMatch: "^[89ABCDEF]"
     volumes:
       - ../config/sharded/server-shard-main.json:/opt/MareSynchronosServer/appsettings.json
       - ../log/server-shard-main/:/opt/MareSynchronosServer/logs/:rw
@@ -98,6 +101,8 @@ services:
       - ../log/files-shard-1/:/opt/MareSynchronosStaticFilesServer/logs/:rw
       - postgres_socket:/var/run/postgresql/:rw
       - ../data/files-shard-1/:/marecache/:rw
+    ports: 
+      - 6200:6200/tcp
     depends_on:
       - "postgres"
       - "mare-files"
@@ -110,6 +115,8 @@ services:
       - ../log/files-shard-2/:/opt/MareSynchronosStaticFilesServer/logs/:rw
       - postgres_socket:/var/run/postgresql/:rw
       - ../data/files-shard-2/:/marecache/:rw
+    ports: 
+      - 6201:6200/tcp
     depends_on:
       - "postgres"
       - "mare-files"

Docker/run/config/sharded/haproxy-shards.cfg

@@ -23,17 +23,8 @@ frontend mare
         bind :6000
         default_backend mare-servers
 
-frontend mare-files
-        bind :6200
-        default_backend mare-files
-
 backend mare-servers
         balance leastconn
         cookie SERVER insert indirect nocache
         server mare1 mare-shard-1:6000 cookie mare1
         server mare2 mare-shard-2:6000 cookie mare2
-
-backend mare-files
-        balance roundrobin
-        server files1 mare-files-shard-1:6200
-        server files2 mare-files-shard-2:6200

Docker/run/config/sharded/server-shard-main.json

@@ -41,7 +41,17 @@
         "MaxJoinedGroupsByUser": 6,
         "MaxGroupUserCount": 100,
         "PurgeUnusedAccounts": false,
-        "PurgeUnusedAccountsPeriodInDays": 14
+        "PurgeUnusedAccountsPeriodInDays": 14,
+        "CdnShardConfiguration": [
+            {
+                "FileMatch": "^[01234567]",
+                "CdnFullUrl": ""
+            },
+            {
+                "FileMatch": "^[89ABCDEF]",
+                "CdnFullUrl": ""
+            }
+        ]
     },
     "AllowedHosts": "*",
     "Kestrel": {

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs

@@ -6,6 +6,7 @@ using MareSynchronos.API;
 using MareSynchronosServer.Utils;
 using MareSynchronosShared.Models;
 using MareSynchronosShared.Protos;
+using MareSynchronosShared.Utils;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.EntityFrameworkCore;
 
@@ -52,10 +53,14 @@ public partial class MareHub
 
         var cacheFile = await _dbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false);
 
+        var shardConfig = _configurationService.GetValueOrDefault(nameof(ServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>());
+
         foreach (var file in cacheFile)
         {
             var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
-            var downloadFile = allFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
+
+            var matchedShardConfig = shardConfig.Find(f => f.FileMatchRegex.Match(file.Hash).Success);
+            var baseUrl = matchedShardConfig?.CdnFullUrl ?? _mainCdnFullUrl;
 
             response.Add(new DownloadFileDto
             {
@@ -64,7 +69,7 @@ public partial class MareHub
                 IsForbidden = forbiddenFile != null,
                 Hash = file.Hash,
                 Size = file.Size,
-                Url = new Uri(_cdnFullUri, file.Hash.ToUpperInvariant()).ToString()
+                Url = new Uri(baseUrl, file.Hash.ToUpperInvariant()).ToString()
             });
         }
 

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs

@@ -23,11 +23,12 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
     private readonly IClientIdentificationService _clientIdentService;
     private readonly MareHubLogger _logger;
     private readonly MareDbContext _dbContext;
-    private readonly Uri _cdnFullUri;
+    private readonly Uri _mainCdnFullUrl;
     private readonly string _shardName;
     private readonly int _maxExistingGroupsByUser;
     private readonly int _maxJoinedGroupsByUser;
     private readonly int _maxGroupUserCount;
+    private IConfigurationService<ServerConfiguration> _configurationService;
 
     public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient,
         MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService,
@@ -37,7 +38,8 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
         _mareMetrics = mareMetrics;
         _fileServiceClient = fileServiceClient;
         _systemInfoService = systemInfoService;
-        _cdnFullUri = configuration.GetValue<Uri>(nameof(ServerConfiguration.CdnFullUrl));
+        _configurationService = configuration;
+        _mainCdnFullUrl = configuration.GetValue<Uri>(nameof(ServerConfiguration.CdnFullUrl));
         _shardName = configuration.GetValue<string>(nameof(ServerConfiguration.ShardName));
         _maxExistingGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxExistingGroupsByUser), 3);
         _maxJoinedGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxJoinedGroupsByUser), 6);

MareSynchronosServer/MareSynchronosServer/Program.cs

@@ -30,6 +30,8 @@ public class Program
                 context.RemoveRange(unfinishedRegistrations);
                 context.RemoveRange(looseFiles);
                 context.SaveChanges();
+
+                logger.LogInformation(options.ToString());
             }
 
             var metrics = services.GetRequiredService<MareMetrics>();

MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceClient.cs

@@ -4,11 +4,13 @@ using MareSynchronosShared.Utils;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using System.Collections;
 using System.Collections.Concurrent;
 using System.Globalization;
 using System.Reflection;
 using System.Text;
 using System.Text.Json;
+using System.Text.RegularExpressions;
 using static MareSynchronosShared.Protos.ConfigurationService;
 
 namespace MareSynchronosShared.Services;
@@ -75,9 +77,17 @@ public class MareConfigurationServiceClient<T> : IHostedService, IConfigurationS
         foreach (var prop in props)
         {
             var isRemote = prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any();
-            var mi = GetType().GetMethod(nameof(GetValue)).MakeGenericMethod(prop.PropertyType);
+            var getValueMethod = GetType().GetMethod(nameof(GetValue)).MakeGenericMethod(prop.PropertyType);
-            var val = mi.Invoke(this, new[] { prop.Name });
+            var value = isRemote ? getValueMethod.Invoke(this, new[] { prop.Name }) : prop.GetValue(_config);
-            var value = isRemote ? val : prop.GetValue(_config);
+            if (typeof(IEnumerable).IsAssignableFrom(prop.PropertyType) && !typeof(string).IsAssignableFrom(prop.PropertyType))
+            {
+                var enumVal = (IEnumerable)value;
+                value = string.Empty;
+                foreach (var listVal in enumVal)
+                {
+                    value += listVal.ToString() + ", ";
+                }
+            }
             sb.AppendLine($"{prop.Name} (IsRemote: {isRemote}) => {value}");
         }
         return sb.ToString();

MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceServer.cs

@@ -32,6 +32,9 @@ public class MareConfigurationServiceServer<T> : IConfigurationService<T> where
         {
             sb.AppendLine($"{prop.Name} (IsRemote: {prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any()}) => {prop.GetValue(_config)}");
         }
+
+        sb.AppendLine(_config.ToString());
+
         return sb.ToString();
     }
 }

MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs

@@ -0,0 +1,17 @@
+using System.Text.Json.Serialization;
+using System.Text.RegularExpressions;
+
+namespace MareSynchronosShared.Utils;
+
+public class CdnShardConfiguration
+{
+    public string FileMatch { get; set; }
+    [JsonIgnore]
+    public Regex FileMatchRegex => new Regex(FileMatch);
+    public Uri CdnFullUrl { get; set; }
+
+    public override string ToString()
+    {
+        return CdnFullUrl.ToString() + " == " + FileMatch;
+    }
+}

MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs

@@ -20,6 +20,8 @@ public class ServerConfiguration : MareConfigurationAuthBase
     public bool PurgeUnusedAccounts { get; set; } = false;
     [RemoteConfiguration]
     public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14;
+    [RemoteConfiguration]
+    public List<CdnShardConfiguration> CdnShardConfiguration { get; set; } = new();
 
     public override string ToString()
     {
@@ -27,6 +29,7 @@ public class ServerConfiguration : MareConfigurationAuthBase
         sb.AppendLine(base.ToString());
         sb.AppendLine($"{nameof(MainServerGrpcAddress)} => {MainServerGrpcAddress}");
         sb.AppendLine($"{nameof(CdnFullUrl)} => {CdnFullUrl}");
+        sb.AppendLine($"{nameof(CdnShardConfiguration)} => {string.Join(", ", CdnShardConfiguration.Select(c => c.ToString()))}");
         sb.AppendLine($"{nameof(StaticFileServiceAddress)} => {StaticFileServiceAddress}");
         sb.AppendLine($"{nameof(RedisConnectionString)} => {RedisConnectionString}");
         sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");