t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add horizontal file sharding based on filename matches

Browse Source
This commit is contained in:
rootdarkarchon
2022-12-31 14:28:24 +01:00
parent 2a5e505130
commit b6404a9c1d
11 changed files with 72 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs
View File

@@ -6,6 +6,7 @@ using MareSynchronos.API;
using MareSynchronosServer.Utils;
using MareSynchronosShared.Models;
using MareSynchronosShared.Protos;
using MareSynchronosShared.Utils;
using Microsoft.AspNetCore.Authorization;
using Microsoft.EntityFrameworkCore;
@@ -52,10 +53,14 @@ public partial class MareHub
var cacheFile = await _dbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false);
var shardConfig = _configurationService.GetValueOrDefault(nameof(ServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>());
foreach (var file in cacheFile)
{
var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
var downloadFile = allFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
var matchedShardConfig = shardConfig.Find(f => f.FileMatchRegex.Match(file.Hash).Success);
var baseUrl = matchedShardConfig?.CdnFullUrl ?? _mainCdnFullUrl;
response.Add(new DownloadFileDto
{
@@ -64,7 +69,7 @@ public partial class MareHub
IsForbidden = forbiddenFile != null,
Hash = file.Hash,
Size = file.Size,
Url = new Uri(_cdnFullUri, file.Hash.ToUpperInvariant()).ToString()
Url = new Uri(baseUrl, file.Hash.ToUpperInvariant()).ToString()
});
}

6
MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs
View File

@@ -23,11 +23,12 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
private readonly IClientIdentificationService _clientIdentService;
private readonly MareHubLogger _logger;
private readonly MareDbContext _dbContext;
private readonly Uri _cdnFullUri;
private readonly Uri _mainCdnFullUrl;
private readonly string _shardName;
private readonly int _maxExistingGroupsByUser;
private readonly int _maxJoinedGroupsByUser;
private readonly int _maxGroupUserCount;
private IConfigurationService<ServerConfiguration> _configurationService;
public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient,
MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService,
@@ -37,7 +38,8 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
_mareMetrics = mareMetrics;
_fileServiceClient = fileServiceClient;
_systemInfoService = systemInfoService;
_cdnFullUri = configuration.GetValue<Uri>(nameof(ServerConfiguration.CdnFullUrl));
_configurationService = configuration;
_mainCdnFullUrl = configuration.GetValue<Uri>(nameof(ServerConfiguration.CdnFullUrl));
_shardName = configuration.GetValue<string>(nameof(ServerConfiguration.ShardName));
_maxExistingGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxExistingGroupsByUser), 3);
_maxJoinedGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxJoinedGroupsByUser), 6);

2
MareSynchronosServer/MareSynchronosServer/Program.cs
View File

@@ -30,6 +30,8 @@ public class Program
context.RemoveRange(unfinishedRegistrations);
context.RemoveRange(looseFiles);
context.SaveChanges();
logger.LogInformation(options.ToString());
}
var metrics = services.GetRequiredService<MareMetrics>();