add geoip service for file shard matching

fix bug

more logging

fix logging wtf is going on even

handle lists in config log output

do not set "*" as default continent

do not rely on "*" being present in configuration when picking file shard

MareSynchronosServer/MareSynchronosServer/Controllers/JwtController.cs

@@ -1,5 +1,6 @@
 using MareSynchronos.API.Routes;
 using MareSynchronosServer.Authentication;
+using MareSynchronosServer.Services;
 using MareSynchronosShared;
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Models;
@@ -23,6 +24,7 @@ public class JwtController : Controller
     private readonly IHttpContextAccessor _accessor;
     private readonly IRedisDatabase _redis;
     private readonly MareDbContext _mareDbContext;
+    private readonly GeoIPService _geoIPProvider;
     private readonly SecretKeyAuthenticatorService _secretKeyAuthenticatorService;
     private readonly AccountRegistrationService _accountRegistrationService;
     private readonly IConfigurationService<MareConfigurationAuthBase> _configuration;
@@ -31,10 +33,11 @@ public class JwtController : Controller
         SecretKeyAuthenticatorService secretKeyAuthenticatorService,
         AccountRegistrationService accountRegistrationService,
         IConfigurationService<MareConfigurationAuthBase> configuration,
-        IRedisDatabase redisDb)
+        IRedisDatabase redisDb, GeoIPService geoIPProvider)
     {
         _accessor = accessor;
         _redis = redisDb;
+        _geoIPProvider = geoIPProvider;
         _mareDbContext = mareDbContext;
         _secretKeyAuthenticatorService = secretKeyAuthenticatorService;
         _accountRegistrationService = accountRegistrationService;
@@ -112,6 +115,7 @@ public class JwtController : Controller
         {
             new Claim(MareClaimTypes.Uid, authResult.Uid),
             new Claim(MareClaimTypes.CharaIdent, charaIdent),
+            new Claim(MareClaimTypes.Continent, await _geoIPProvider.GetCountryFromIP(_accessor)),
         });
 
         return Content(token.RawData);
@@ -140,3 +144,4 @@ public class JwtController : Controller
         return handler.CreateJwtSecurityToken(token);
     }
 }
+

MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj

@@ -29,6 +29,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
+    <PackageReference Include="MaxMind.GeoIP2" Version="5.2.0" />
     <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="8.0.0" />
     <PackageReference Include="Microsoft.IdentityModel.Tokens" Version="7.5.1" />
     <PackageReference Include="SixLabors.ImageSharp" Version="3.1.4" />

MareSynchronosServer/MareSynchronosServer/Services/GeoIPService.cs

@@ -0,0 +1,123 @@
+using MareSynchronosShared;
+using MareSynchronosShared.Services;
+using MareSynchronosShared.Utils;
+using MaxMind.GeoIP2;
+
+namespace MareSynchronosServer.Services;
+
+public class GeoIPService : IHostedService
+{
+    private readonly ILogger<GeoIPService> _logger;
+    private readonly IConfigurationService<ServerConfiguration> _mareConfiguration;
+    private bool _useGeoIP = false;
+    private string _countryFile = string.Empty;
+    private DatabaseReader? _dbReader;
+    private DateTime _dbLastWriteTime = DateTime.Now;
+    private CancellationTokenSource _fileWriteTimeCheckCts = new();
+    private bool _processingReload = false;
+
+    public GeoIPService(ILogger<GeoIPService> logger,
+        IConfigurationService<ServerConfiguration> mareConfiguration)
+    {
+        _logger = logger;
+        _mareConfiguration = mareConfiguration;
+    }
+
+    public async Task<string> GetCountryFromIP(IHttpContextAccessor httpContextAccessor)
+    {
+        if (!_useGeoIP)
+        {
+            return "*";
+        }
+
+        try
+        {
+            var ip = httpContextAccessor.GetIpAddress();
+
+            using CancellationTokenSource waitCts = new();
+            waitCts.CancelAfter(TimeSpan.FromSeconds(5));
+            while (_processingReload) await Task.Delay(100, waitCts.Token).ConfigureAwait(false);
+
+            if (_dbReader.TryCountry(ip, out var response))
+            {
+                return response.Continent.Code;
+            }
+
+            return "*";
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error handling Geo IP country in request");
+            return "*";
+        }
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("GeoIP module starting update task");
+
+        var token = _fileWriteTimeCheckCts.Token;
+        _ = PeriodicReloadTask(token);
+
+        return Task.CompletedTask;
+    }
+
+    private async Task PeriodicReloadTask(CancellationToken token)
+    {
+        while (!token.IsCancellationRequested)
+        {
+            try
+            {
+                _processingReload = true;
+
+                var useGeoIP = _mareConfiguration.GetValueOrDefault(nameof(ServerConfiguration.UseGeoIP), false);
+                var countryFile = _mareConfiguration.GetValueOrDefault(nameof(ServerConfiguration.GeoIPDbCountryFile), string.Empty);
+                var lastWriteTime = new FileInfo(countryFile).LastWriteTimeUtc;
+                if (useGeoIP && (!string.Equals(countryFile, _countryFile, StringComparison.OrdinalIgnoreCase) || lastWriteTime != _dbLastWriteTime))
+                {
+                    _countryFile = countryFile;
+                    if (!File.Exists(_countryFile)) throw new FileNotFoundException($"Could not open GeoIP Country Database, path does not exist: {_countryFile}");
+                    _dbReader?.Dispose();
+                    _dbReader = null;
+                    _dbReader = new DatabaseReader(_countryFile);
+                    _dbLastWriteTime = lastWriteTime;
+
+                    _ = _dbReader.Country("8.8.8.8").Continent;
+
+                    _logger.LogInformation($"Loaded GeoIP country file from {_countryFile}");
+
+                    if (_useGeoIP != useGeoIP)
+                    {
+                        _logger.LogInformation("GeoIP module is now enabled");
+                        _useGeoIP = useGeoIP;
+                    }
+                }
+
+                if (_useGeoIP != useGeoIP && !useGeoIP)
+                {
+                    _logger.LogInformation("GeoIP module is now disabled");
+                    _useGeoIP = useGeoIP;
+                }
+            }
+            catch (Exception e)
+            {
+                _logger.LogWarning(e, "Error during periodic GeoIP module reload task, disabling GeoIP");
+                _useGeoIP = false;
+            }
+            finally
+            {
+                _processingReload = false;
+            }
+
+            await Task.Delay(TimeSpan.FromMinutes(1)).ConfigureAwait(false);
+        }
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        _fileWriteTimeCheckCts.Cancel();
+        _fileWriteTimeCheckCts.Dispose();
+        _dbReader.Dispose();
+        return Task.CompletedTask;
+    }
+}

MareSynchronosServer/MareSynchronosServer/Startup.cs

@@ -96,8 +96,10 @@ public class Startup
 
         if (isMainServer)
         {
+            services.AddSingleton<GeoIPService>();
             services.AddSingleton<UserCleanupService>();
             services.AddHostedService(provider => provider.GetService<UserCleanupService>());
+            services.AddHostedService(provider => provider.GetService<GeoIPService>());
         }
     }
 

MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceServer.cs

@@ -1,5 +1,6 @@
 using MareSynchronosShared.Utils;
 using Microsoft.Extensions.Options;
+using System.Collections;
 using System.Text;
 
 namespace MareSynchronosShared.Services;
@@ -30,11 +31,20 @@ public class MareConfigurationServiceServer<T> : IConfigurationService<T> where
         StringBuilder sb = new();
         foreach (var prop in props)
         {
-            sb.AppendLine($"{prop.Name} (IsRemote: {prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any()}) => {prop.GetValue(_config.CurrentValue)}");
+            var isRemote = prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any();
+            var getValueMethod = GetType().GetMethod(nameof(GetValue)).MakeGenericMethod(prop.PropertyType);
+            var value = isRemote ? getValueMethod.Invoke(this, new[] { prop.Name }) : prop.GetValue(_config.CurrentValue);
+            if (typeof(IEnumerable).IsAssignableFrom(prop.PropertyType) && !typeof(string).IsAssignableFrom(prop.PropertyType))
+            {
+                var enumVal = (IEnumerable)value;
+                value = string.Empty;
+                foreach (var listVal in enumVal)
+                {
+                    value += listVal.ToString() + ", ";
+                }
+            }
+            sb.AppendLine($"{prop.Name} (IsRemote: {isRemote}) => {value}");
         }
-
-        sb.AppendLine(_config.ToString());
-
         return sb.ToString();
     }
 }

MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs

@@ -2,11 +2,12 @@
 
 public class CdnShardConfiguration
 {
+    public List<string> Continents { get; set; }
     public string FileMatch { get; set; }
     public Uri CdnFullUrl { get; set; }
 
     public override string ToString()
     {
-        return CdnFullUrl.ToString() + " == " + FileMatch;
+        return CdnFullUrl.ToString() + "[" + string.Join(',', Continents) + "] == " + FileMatch;
     }
 }

MareSynchronosServer/MareSynchronosShared/Utils/MareClaimTypes.cs

@@ -5,4 +5,5 @@ public static class MareClaimTypes
     public const string Uid = "uid";
     public const string CharaIdent = "character_identification";
     public const string Internal = "internal";
+    public const string Continent = "continent";
 }

MareSynchronosServer/MareSynchronosShared/Utils/MareConfigurationAuthBase.cs

@@ -14,6 +14,8 @@ public class MareConfigurationAuthBase : MareConfigurationBase
     public int RegisterIpDurationInMinutes { get; set; } = 10;
     [RemoteConfiguration]
     public List<string> WhitelistedIps { get; set; } = new();
+    [RemoteConfiguration]
+    public bool UseGeoIP { get; set; } = false;
 
     public override string ToString()
     {
@@ -25,6 +27,7 @@ public class MareConfigurationAuthBase : MareConfigurationBase
         sb.AppendLine($"{nameof(RegisterIpDurationInMinutes)} => {RegisterIpDurationInMinutes}");
         sb.AppendLine($"{nameof(Jwt)} => {Jwt}");
         sb.AppendLine($"{nameof(WhitelistedIps)} => {string.Join(", ", WhitelistedIps)}");
+        sb.AppendLine($"{nameof(UseGeoIP)} => {UseGeoIP}");
         return sb.ToString();
     }
 }

MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs

@@ -24,6 +24,7 @@ public class ServerConfiguration : MareConfigurationAuthBase
 
     [RemoteConfiguration]
     public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14;
+    public string GeoIPDbCountryFile { get; set; } = string.Empty;
 
     public int RedisPool { get; set; } = 50;
 
@@ -40,6 +41,7 @@ public class ServerConfiguration : MareConfigurationAuthBase
         sb.AppendLine($"{nameof(MaxGroupUserCount)} => {MaxGroupUserCount}");
         sb.AppendLine($"{nameof(PurgeUnusedAccounts)} => {PurgeUnusedAccounts}");
         sb.AppendLine($"{nameof(PurgeUnusedAccountsPeriodInDays)} => {PurgeUnusedAccountsPeriodInDays}");
+        sb.AppendLine($"{nameof(GeoIPDbCountryFile)} => {GeoIPDbCountryFile}");
         return sb.ToString();
     }
 }

MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs

@@ -29,6 +29,7 @@ public class StaticFilesServerConfiguration : MareConfigurationBase
         sb.AppendLine($"{nameof(CacheDirectory)} => {CacheDirectory}");
         sb.AppendLine($"{nameof(DownloadQueueSize)} => {DownloadQueueSize}");
         sb.AppendLine($"{nameof(DownloadQueueReleaseSeconds)} => {DownloadQueueReleaseSeconds}");
+        sb.AppendLine($"{nameof(CdnShardConfiguration)} => {string.Join(", ", CdnShardConfiguration)}");
         return sb.ToString();
     }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs

@@ -13,4 +13,5 @@ public class ControllerBase : Controller
     }
 
     protected string MareUser => HttpContext.User.Claims.First(f => string.Equals(f.Type, MareClaimTypes.Uid, StringComparison.Ordinal)).Value;
+    protected string Continent => HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, MareClaimTypes.Continent, StringComparison.Ordinal))?.Value ?? "*";
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs

@@ -36,7 +36,7 @@ public class ServerFilesController : ControllerBase
 
     public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider,
         IConfigurationService<StaticFilesServerConfiguration> configuration,
-        IHubContext<MareSynchronosServer.Hubs.MareHub> hubContext,
+        IHubContext<MareHub> hubContext,
         MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger)
     {
         _basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
@@ -79,14 +79,35 @@ public class ServerFilesController : ControllerBase
 
         var cacheFile = await _mareDbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false);
 
-        var shardConfig = new List<CdnShardConfiguration>(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>()));
+        var allFileShards = new List<CdnShardConfiguration>(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>()));
 
         foreach (var file in cacheFile)
         {
             var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
+            Uri? baseUrl = null;
 
-            var matchedShardConfig = shardConfig.OrderBy(g => Guid.NewGuid()).FirstOrDefault(f => new Regex(f.FileMatch).IsMatch(file.Hash));
+            if (forbiddenFile == null)
-            var baseUrl = matchedShardConfig?.CdnFullUrl ?? _configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.CdnFullUrl));
+            {
+                List<CdnShardConfiguration> selectedShards = new();
+                var matchingShards = allFileShards.Where(f => new Regex(f.FileMatch).IsMatch(file.Hash)).ToList();
+
+                if (string.Equals(Continent, "*", StringComparison.Ordinal))
+                {
+                    selectedShards = matchingShards;
+                }
+                else
+                {
+                    selectedShards = matchingShards.Where(c => c.Continents.Contains(Continent, StringComparer.OrdinalIgnoreCase)).ToList();
+                    if (!selectedShards.Any()) selectedShards = matchingShards;
+                }
+
+                var shard = selectedShards
+                    .OrderBy(s => !s.Continents.Any() ? 0 : 1)
+                    .ThenBy(s => s.Continents.Contains("*", StringComparer.Ordinal) ? 0 : 1)
+                    .ThenBy(g => Guid.NewGuid()).FirstOrDefault();
+
+                baseUrl = shard?.CdnFullUrl ?? _configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.CdnFullUrl));
+            }
 
             response.Add(new DownloadFileDto
             {
@@ -95,7 +116,7 @@ public class ServerFilesController : ControllerBase
                 IsForbidden = forbiddenFile != null,
                 Hash = file.Hash,
                 Size = file.Size,
-                Url = baseUrl.ToString(),
+                Url = baseUrl?.ToString() ?? string.Empty,
             });
         }