t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add geoip service for file shard matching

Browse Source 
fix bug

more logging

fix logging wtf is going on even

handle lists in config log output

do not set "*" as default continent

do not rely on "*" being present in configuration when picking file shard
This commit is contained in:
rootdarkarchon
2024-01-12 13:10:14 +01:00
committed by Loporrit
parent 7667b74734
commit 989046da13
12 changed files with 182 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
MareSynchronosServer/MareSynchronosServer/Controllers/JwtController.cs
View File

@@ -1,5 +1,6 @@
using MareSynchronos.API.Routes;
using MareSynchronosServer.Authentication;
using MareSynchronosServer.Services;
using MareSynchronosShared;
using MareSynchronosShared.Data;
using MareSynchronosShared.Models;
@@ -23,6 +24,7 @@ public class JwtController : Controller
private readonly IHttpContextAccessor _accessor;
private readonly IRedisDatabase _redis;
private readonly MareDbContext _mareDbContext;
private readonly GeoIPService _geoIPProvider;
private readonly SecretKeyAuthenticatorService _secretKeyAuthenticatorService;
private readonly AccountRegistrationService _accountRegistrationService;
private readonly IConfigurationService<MareConfigurationAuthBase> _configuration;
@@ -31,10 +33,11 @@ public class JwtController : Controller
SecretKeyAuthenticatorService secretKeyAuthenticatorService,
AccountRegistrationService accountRegistrationService,
IConfigurationService<MareConfigurationAuthBase> configuration,
IRedisDatabase redisDb)
IRedisDatabase redisDb, GeoIPService geoIPProvider)
{
_accessor = accessor;
_redis = redisDb;
_geoIPProvider = geoIPProvider;
_mareDbContext = mareDbContext;
_secretKeyAuthenticatorService = secretKeyAuthenticatorService;
_accountRegistrationService = accountRegistrationService;
@@ -112,6 +115,7 @@ public class JwtController : Controller
{
new Claim(MareClaimTypes.Uid, authResult.Uid),
new Claim(MareClaimTypes.CharaIdent, charaIdent),
new Claim(MareClaimTypes.Continent, await _geoIPProvider.GetCountryFromIP(_accessor)),
});
return Content(token.RawData);
@@ -140,3 +144,4 @@ public class JwtController : Controller
return handler.CreateJwtSecurityToken(token);
}
}

1
MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj
View File

@@ -29,6 +29,7 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="MaxMind.GeoIP2" Version="5.2.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="8.0.0" />
<PackageReference Include="Microsoft.IdentityModel.Tokens" Version="7.5.1" />
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.4" />

123
MareSynchronosServer/MareSynchronosServer/Services/GeoIPService.cs Normal file
View File

@@ -0,0 +1,123 @@
using MareSynchronosShared;
using MareSynchronosShared.Services;
using MareSynchronosShared.Utils;
using MaxMind.GeoIP2;
namespace MareSynchronosServer.Services;
public class GeoIPService : IHostedService
{
private readonly ILogger<GeoIPService> _logger;
private readonly IConfigurationService<ServerConfiguration> _mareConfiguration;
private bool _useGeoIP = false;
private string _countryFile = string.Empty;
private DatabaseReader? _dbReader;
private DateTime _dbLastWriteTime = DateTime.Now;
private CancellationTokenSource _fileWriteTimeCheckCts = new();
private bool _processingReload = false;
public GeoIPService(ILogger<GeoIPService> logger,
IConfigurationService<ServerConfiguration> mareConfiguration)
{
_logger = logger;
_mareConfiguration = mareConfiguration;
}
public async Task<string> GetCountryFromIP(IHttpContextAccessor httpContextAccessor)
{
if (!_useGeoIP)
{
return "*";
}
try
{
var ip = httpContextAccessor.GetIpAddress();
using CancellationTokenSource waitCts = new();
waitCts.CancelAfter(TimeSpan.FromSeconds(5));
while (_processingReload) await Task.Delay(100, waitCts.Token).ConfigureAwait(false);
if (_dbReader.TryCountry(ip, out var response))
{
return response.Continent.Code;
}
return "*";
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error handling Geo IP country in request");
return "*";
}
}
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("GeoIP module starting update task");
var token = _fileWriteTimeCheckCts.Token;
_ = PeriodicReloadTask(token);
return Task.CompletedTask;
}
private async Task PeriodicReloadTask(CancellationToken token)
{
while (!token.IsCancellationRequested)
{
try
{
_processingReload = true;
var useGeoIP = _mareConfiguration.GetValueOrDefault(nameof(ServerConfiguration.UseGeoIP), false);
var countryFile = _mareConfiguration.GetValueOrDefault(nameof(ServerConfiguration.GeoIPDbCountryFile), string.Empty);
var lastWriteTime = new FileInfo(countryFile).LastWriteTimeUtc;
if (useGeoIP && (!string.Equals(countryFile, _countryFile, StringComparison.OrdinalIgnoreCase) || lastWriteTime != _dbLastWriteTime))
{
_countryFile = countryFile;
if (!File.Exists(_countryFile)) throw new FileNotFoundException($"Could not open GeoIP Country Database, path does not exist: {_countryFile}");
_dbReader?.Dispose();
_dbReader = null;
_dbReader = new DatabaseReader(_countryFile);
_dbLastWriteTime = lastWriteTime;
_ = _dbReader.Country("8.8.8.8").Continent;
_logger.LogInformation($"Loaded GeoIP country file from {_countryFile}");
if (_useGeoIP != useGeoIP)
{
_logger.LogInformation("GeoIP module is now enabled");
_useGeoIP = useGeoIP;
}
}
if (_useGeoIP != useGeoIP && !useGeoIP)
{
_logger.LogInformation("GeoIP module is now disabled");
_useGeoIP = useGeoIP;
}
}
catch (Exception e)
{
_logger.LogWarning(e, "Error during periodic GeoIP module reload task, disabling GeoIP");
_useGeoIP = false;
}
finally
{
_processingReload = false;
}
await Task.Delay(TimeSpan.FromMinutes(1)).ConfigureAwait(false);
}
}
public Task StopAsync(CancellationToken cancellationToken)
{
_fileWriteTimeCheckCts.Cancel();
_fileWriteTimeCheckCts.Dispose();
_dbReader.Dispose();
return Task.CompletedTask;
}
}

2
MareSynchronosServer/MareSynchronosServer/Startup.cs
View File

@@ -96,8 +96,10 @@ public class Startup
if (isMainServer)
{
services.AddSingleton<GeoIPService>();
services.AddSingleton<UserCleanupService>();
services.AddHostedService(provider => provider.GetService<UserCleanupService>());
services.AddHostedService(provider => provider.GetService<GeoIPService>());
}
}

18
MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceServer.cs
View File

@@ -1,5 +1,6 @@
using MareSynchronosShared.Utils;
using Microsoft.Extensions.Options;
using System.Collections;
using System.Text;
namespace MareSynchronosShared.Services;
@@ -30,11 +31,20 @@ public class MareConfigurationServiceServer<T> : IConfigurationService<T> where
StringBuilder sb = new();
foreach (var prop in props)
{
sb.AppendLine($"{prop.Name} (IsRemote: {prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any()}) => {prop.GetValue(_config.CurrentValue)}");
var isRemote = prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any();
var getValueMethod = GetType().GetMethod(nameof(GetValue)).MakeGenericMethod(prop.PropertyType);
var value = isRemote ? getValueMethod.Invoke(this, new[] { prop.Name }) : prop.GetValue(_config.CurrentValue);
if (typeof(IEnumerable).IsAssignableFrom(prop.PropertyType) && !typeof(string).IsAssignableFrom(prop.PropertyType))
{
var enumVal = (IEnumerable)value;
value = string.Empty;
foreach (var listVal in enumVal)
{
value += listVal.ToString() + ", ";
}
}
sb.AppendLine($"{prop.Name} (IsRemote: {isRemote}) => {value}");
}
sb.AppendLine(_config.ToString());
return sb.ToString();
}
}

3
MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs
View File

@@ -2,11 +2,12 @@
public class CdnShardConfiguration
{
public List<string> Continents { get; set; }
public string FileMatch { get; set; }
public Uri CdnFullUrl { get; set; }
public override string ToString()
{
return CdnFullUrl.ToString() + " == " + FileMatch;
return CdnFullUrl.ToString() + "[" + string.Join(',', Continents) + "] == " + FileMatch;
}
}

1
MareSynchronosServer/MareSynchronosShared/Utils/MareClaimTypes.cs
View File

@@ -5,4 +5,5 @@ public static class MareClaimTypes
public const string Uid = "uid";
public const string CharaIdent = "character_identification";
public const string Internal = "internal";
public const string Continent = "continent";
}

3
MareSynchronosServer/MareSynchronosShared/Utils/MareConfigurationAuthBase.cs
View File

@@ -14,6 +14,8 @@ public class MareConfigurationAuthBase : MareConfigurationBase
public int RegisterIpDurationInMinutes { get; set; } = 10;
[RemoteConfiguration]
public List<string> WhitelistedIps { get; set; } = new();
[RemoteConfiguration]
public bool UseGeoIP { get; set; } = false;
public override string ToString()
{
@@ -25,6 +27,7 @@ public class MareConfigurationAuthBase : MareConfigurationBase
sb.AppendLine($"{nameof(RegisterIpDurationInMinutes)} => {RegisterIpDurationInMinutes}");
sb.AppendLine($"{nameof(Jwt)} => {Jwt}");
sb.AppendLine($"{nameof(WhitelistedIps)} => {string.Join(", ", WhitelistedIps)}");
sb.AppendLine($"{nameof(UseGeoIP)} => {UseGeoIP}");
return sb.ToString();
}
}

2
MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs
View File

@@ -24,6 +24,7 @@ public class ServerConfiguration : MareConfigurationAuthBase
[RemoteConfiguration]
public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14;
public string GeoIPDbCountryFile { get; set; } = string.Empty;
public int RedisPool { get; set; } = 50;
@@ -40,6 +41,7 @@ public class ServerConfiguration : MareConfigurationAuthBase
sb.AppendLine($"{nameof(MaxGroupUserCount)} => {MaxGroupUserCount}");
sb.AppendLine($"{nameof(PurgeUnusedAccounts)} => {PurgeUnusedAccounts}");
sb.AppendLine($"{nameof(PurgeUnusedAccountsPeriodInDays)} => {PurgeUnusedAccountsPeriodInDays}");
sb.AppendLine($"{nameof(GeoIPDbCountryFile)} => {GeoIPDbCountryFile}");
return sb.ToString();
}
}

1
MareSynchronosServer/MareSynchronosShared/Utils/StaticFilesServerConfiguration.cs
View File

@@ -29,6 +29,7 @@ public class StaticFilesServerConfiguration : MareConfigurationBase
sb.AppendLine($"{nameof(CacheDirectory)} => {CacheDirectory}");
sb.AppendLine($"{nameof(DownloadQueueSize)} => {DownloadQueueSize}");
sb.AppendLine($"{nameof(DownloadQueueReleaseSeconds)} => {DownloadQueueReleaseSeconds}");
sb.AppendLine($"{nameof(CdnShardConfiguration)} => {string.Join(", ", CdnShardConfiguration)}");
return sb.ToString();
}
}

1
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs
View File

@@ -13,4 +13,5 @@ public class ControllerBase : Controller
}
protected string MareUser => HttpContext.User.Claims.First(f => string.Equals(f.Type, MareClaimTypes.Uid, StringComparison.Ordinal)).Value;
protected string Continent => HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, MareClaimTypes.Continent, StringComparison.Ordinal))?.Value ?? "*";
}

31
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs
View File

@@ -36,7 +36,7 @@ public class ServerFilesController : ControllerBase
public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider,
IConfigurationService<StaticFilesServerConfiguration> configuration,
IHubContext<MareSynchronosServer.Hubs.MareHub> hubContext,
IHubContext<MareHub> hubContext,
MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger)
{
_basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
@@ -79,14 +79,35 @@ public class ServerFilesController : ControllerBase
var cacheFile = await _mareDbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false);
var shardConfig = new List<CdnShardConfiguration>(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>()));
var allFileShards = new List<CdnShardConfiguration>(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>()));
foreach (var file in cacheFile)
{
var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
Uri? baseUrl = null;
var matchedShardConfig = shardConfig.OrderBy(g => Guid.NewGuid()).FirstOrDefault(f => new Regex(f.FileMatch).IsMatch(file.Hash));
var baseUrl = matchedShardConfig?.CdnFullUrl ?? _configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.CdnFullUrl));
if (forbiddenFile == null)
{
List<CdnShardConfiguration> selectedShards = new();
var matchingShards = allFileShards.Where(f => new Regex(f.FileMatch).IsMatch(file.Hash)).ToList();
if (string.Equals(Continent, "*", StringComparison.Ordinal))
{
selectedShards = matchingShards;
}
else
{
selectedShards = matchingShards.Where(c => c.Continents.Contains(Continent, StringComparer.OrdinalIgnoreCase)).ToList();
if (!selectedShards.Any()) selectedShards = matchingShards;
}
var shard = selectedShards
.OrderBy(s => !s.Continents.Any() ? 0 : 1)
.ThenBy(s => s.Continents.Contains("*", StringComparer.Ordinal) ? 0 : 1)
.ThenBy(g => Guid.NewGuid()).FirstOrDefault();
baseUrl = shard?.CdnFullUrl ?? _configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.CdnFullUrl));
}
response.Add(new DownloadFileDto
{
@@ -95,7 +116,7 @@ public class ServerFilesController : ControllerBase
IsForbidden = forbiddenFile != null,
Hash = file.Hash,
Size = file.Size,
Url = baseUrl.ToString(),
Url = baseUrl?.ToString() ?? string.Empty,
});
}