t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

decouple fileservice to be able to run standalone

Browse Source
This commit is contained in:
Stanley Dimant
2022-08-23 02:28:04 +02:00
parent bdfe51c15c
commit a63174009f
11 changed files with 297 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs
View File

@@ -2,13 +2,11 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
using Google.Protobuf;
using MareSynchronos.API;
using MareSynchronosShared.Authentication;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Models;
using MareSynchronosShared.Protos;
using Microsoft.AspNetCore.Authorization;
@@ -20,8 +18,6 @@ namespace MareSynchronosServer.Hubs
{
public partial class MareHub
{
private string BasePath => _configuration["CacheDirectory"];
[Authorize(AuthenticationSchemes = SecretKeyGrpcAuthenticationHandler.AuthScheme)]
[HubMethodName(Api.SendFileAbortUpload)]
public async Task AbortUpload()
@@ -40,20 +36,9 @@ namespace MareSynchronosServer.Hubs
_logger.LogInformation("User {AuthenticatedUserId} deleted all their files", AuthenticatedUserId);
var ownFiles = await _dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == AuthenticatedUserId).ToListAsync().ConfigureAwait(false);
foreach (var file in ownFiles)
{
var fi = new FileInfo(Path.Combine(BasePath, file.Hash));
if (fi.Exists)
{
await _metricsClient.DecGaugeAsync(new GaugeRequest()
{GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = fi.Length}).ConfigureAwait(false);
await _metricsClient.DecGaugeAsync(new GaugeRequest()
{ GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1}).ConfigureAwait(false);
fi.Delete();
}
}
_dbContext.Files.RemoveRange(ownFiles);
await _dbContext.SaveChangesAsync().ConfigureAwait(false);
var request = new DeleteFilesRequest();
request.Hash.AddRange(ownFiles.Select(f => f.Hash));
_ = await _fileServiceClient.DeleteFilesAsync(request).ConfigureAwait(false);
}
[Authorize(AuthenticationSchemes = SecretKeyGrpcAuthenticationHandler.AuthScheme)]
@@ -64,37 +49,24 @@ namespace MareSynchronosServer.Hubs
var forbiddenFiles = await _dbContext.ForbiddenUploadEntries.
Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false);
List<DownloadFileDto> response = new();
foreach (var hash in hashes)
{
var fileInfo = new FileInfo(Path.Combine(BasePath, hash));
long fileSize = 0;
try
{
fileSize = fileInfo.Length;
}
catch
{
// file doesn't exist anymore
}
var forbiddenFile = forbiddenFiles.SingleOrDefault(f => f.Hash == hash);
var downloadFile = allFiles.SingleOrDefault(f => f.Hash == hash);
FileSizeRequest request = new FileSizeRequest();
var grpcResponse = await _fileServiceClient.GetFileSizesAsync(request).ConfigureAwait(false);
foreach (var hash in grpcResponse.HashToFileSize)
{
var forbiddenFile = forbiddenFiles.SingleOrDefault(f => f.Hash == hash.Key);
var downloadFile = allFiles.SingleOrDefault(f => f.Hash == hash.Key);
response.Add(new DownloadFileDto
{
FileExists = fileInfo.Exists,
FileExists = hash.Value > 0,
ForbiddenBy = forbiddenFile?.ForbiddenBy ?? string.Empty,
IsForbidden = forbiddenFile != null,
Hash = hash,
Size = fileSize,
Url = _configuration["CdnFullUrl"] + hash.ToUpperInvariant()
Hash = hash.Key,
Size = hash.Value,
Url = _configuration["CdnFullUrl"] + hash.Key.ToUpperInvariant()
});
if (!fileInfo.Exists && downloadFile != null)
{
_dbContext.Files.Remove(downloadFile);
await _dbContext.SaveChangesAsync().ConfigureAwait(false);
}
}
return response;
@@ -169,8 +141,8 @@ namespace MareSynchronosServer.Hubs
if (relatedFile == null) return;
var forbiddenFile = _dbContext.ForbiddenUploadEntries.SingleOrDefault(f => f.Hash == hash);
if (forbiddenFile != null) return;
var finalFileName = Path.Combine(BasePath, hash);
var tempFileName = finalFileName + ".tmp";
var tempFileName = Path.GetTempFileName();
using var fileStream = new FileStream(tempFileName, FileMode.OpenOrCreate);
long length = 0;
try
@@ -223,17 +195,10 @@ namespace MareSynchronosServer.Hubs
return;
}
File.Move(tempFileName, finalFileName, true);
relatedFile = _dbContext.Files.Single(f => f.Hash == hash);
relatedFile.Uploaded = true;
await _metricsClient.IncGaugeAsync(new GaugeRequest()
{ GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = length }).ConfigureAwait(false);
await _metricsClient.IncGaugeAsync(new GaugeRequest()
{ GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 }).ConfigureAwait(false);
await _dbContext.SaveChangesAsync().ConfigureAwait(false);
_logger.LogInformation("File {hash} added to DB", hash);
UploadFileRequest req = new();
req.FileData = ByteString.CopyFrom(await File.ReadAllBytesAsync(tempFileName).ConfigureAwait(false));
File.Delete(tempFileName);
_ = await _fileServiceClient.UploadFileAsync(req).ConfigureAwait(false);
}
catch (Exception ex)
{

7
MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs
View File

@@ -1,7 +1,6 @@
using System;
using System.Linq;
using System.Security.Claims;
using System.Security.Cryptography;
using System.Threading.Tasks;
using MareSynchronos.API;
using MareSynchronosShared.Authentication;
@@ -22,22 +21,24 @@ namespace MareSynchronosServer.Hubs
{
private readonly MetricsService.MetricsServiceClient _metricsClient;
private readonly AuthService.AuthServiceClient _authServiceClient;
private readonly FileService.FileServiceClient _fileServiceClient;
private readonly SystemInfoService _systemInfoService;
private readonly IConfiguration _configuration;
private readonly IHttpContextAccessor contextAccessor;
private readonly ILogger<MareHub> _logger;
private readonly MareDbContext _dbContext;
public MareHub(MetricsService.MetricsServiceClient metricsClient, AuthService.AuthServiceClient authServiceClient,
public MareHub(MetricsService.MetricsServiceClient metricsClient, AuthService.AuthServiceClient authServiceClient, FileService.FileServiceClient fileServiceClient,
MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService, IConfiguration configuration, IHttpContextAccessor contextAccessor)
{
_metricsClient = metricsClient;
_authServiceClient = authServiceClient;
_fileServiceClient = fileServiceClient;
_systemInfoService = systemInfoService;
_configuration = configuration;
this.contextAccessor = contextAccessor;
_logger = logger;
_dbContext = mareDbContext;
_staticFileAddress = new Uri(_configuration["StaticFileServiceAddress"]);
}
[HubMethodName(Api.InvokeHeartbeat)]

4
MareSynchronosServer/MareSynchronosServer/Startup.cs
View File

@@ -53,6 +53,10 @@ namespace MareSynchronosServer
{
c.Address = new Uri(Configuration.GetValue<string>("ServiceAddress"));
});
services.AddGrpcClient<FileService.FileServiceClient>(c =>
{
c.Address = new Uri(Configuration.GetValue<string>("StaticFileServiceAddress"));
});
services.AddDbContextPool<MareDbContext>(options =>
{

13
MareSynchronosServer/MareSynchronosServer/appsettings.json
View File

@@ -25,9 +25,8 @@
},
"DbContextPoolSize": 2000,
"CdnFullUrl": "https://<url or ip to your server>/cache/",
"CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored
"ServiceAddress": "http://localhost:5002",
"StaticFileServiceAddress": "http://localhost:5001",
"AllowedHosts": "*",
"Kestrel": {
"Endpoints": {
@@ -51,14 +50,8 @@
"RealIpHeader": "X-Real-IP",
"ClientIdHeader": "X-ClientId",
"HttpStatusCode": 429,
"IpWhitelist": [ "127.0.0.1", "::1/10", "192.168.0.0/24" ],
"GeneralRules": [
{
"Endpoint": "*",
"Period": "1s",
"Limit": 2
}
]
"IpWhitelist": [ ],
"GeneralRules": [ ]
},
"IPRateLimitPolicies": {
"IpRules": []

3
MareSynchronosServer/MareSynchronosServices/Program.cs
View File

@@ -1,7 +1,4 @@
using MareSynchronosServices;
using MareSynchronosServices.Metrics;
using MareSynchronosShared.Data;
using Microsoft.EntityFrameworkCore;
public class Program
{

2
MareSynchronosServer/MareSynchronosServices/appsettings.json
View File

@@ -18,10 +18,8 @@
},
"DbContextPoolSize": 1024,
"DiscordBotToken": "",
"UnusedFileRetentionPeriodInDays": 7,
"PurgeUnusedAccounts": true,
"PurgeUnusedAccountsPeriodInDays": 14,
"CacheSizeHardLimitInGiB": -1,
"CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored
"FailedAuthForTempBan": 5,
"TempBanDurationInMinutes": 30,

24
MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto
View File

@@ -17,8 +17,32 @@ service MetricsService {
rpc IncGauge (GaugeRequest) returns (Empty);
}
service FileService {
rpc UploadFile (UploadFileRequest) returns (Empty);
rpc GetFileSizes (FileSizeRequest) returns (FileSizeResponse);
rpc DeleteFiles (DeleteFilesRequest) returns (Empty);
}
message Empty { }
message UploadFileRequest {
string hash = 1;
string uploader = 2;
bytes fileData = 3;
}
message DeleteFilesRequest {
repeated string hash = 1;
}
message FileSizeRequest {
repeated string hash = 1;
}
message FileSizeResponse {
map<string, int64> hashToFileSize = 1;
}
message GaugeRequest {
string gaugeName = 1;
double value = 2;

120
MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs Normal file
View File

@@ -0,0 +1,120 @@
using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using static MareSynchronosShared.Protos.MetricsService;
namespace MareSynchronosStaticFilesServer;
public class CleanupService : IHostedService, IDisposable
{
private readonly MetricsServiceClient _metrics;
private readonly ILogger<CleanupService> _logger;
private readonly IServiceProvider _services;
private readonly IConfiguration _configuration;
private Timer? _timer;
public CleanupService(MetricsServiceClient metrics, ILogger<CleanupService> logger, IServiceProvider services, IConfiguration configuration)
{
_metrics = metrics;
_logger = logger;
_services = services;
_configuration = configuration;
}
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Cleanup Service started");
_timer = new Timer(CleanUp, null, TimeSpan.Zero, TimeSpan.FromMinutes(10));
return Task.CompletedTask;
}
private void CleanUp(object? state)
{
if (!int.TryParse(_configuration["UnusedFileRetentionPeriodInDays"], out var filesOlderThanDays))
{
filesOlderThanDays = 7;
}
using var scope = _services.CreateScope();
using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
_logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", filesOlderThanDays);
try
{
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays));
var allFiles = dbContext.Files.ToList();
var cachedir = _configuration["CacheDirectory"];
foreach (var file in allFiles.Where(f => f.Uploaded))
{
var fileName = Path.Combine(cachedir, file.Hash);
var fi = new FileInfo(fileName);
if (!fi.Exists)
{
_logger.LogInformation("File does not exist anymore: {fileName}", fileName);
dbContext.Files.Remove(file);
}
else if (fi.LastAccessTime < prevTime)
{
_metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = fi.Length });
_metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 });
_logger.LogInformation("File outdated: {fileName}", fileName);
dbContext.Files.Remove(file);
fi.Delete();
}
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error during file cleanup");
}
var cacheSizeLimitInGiB = _configuration.GetValue<double>("CacheSizeHardLimitInGiB", -1);
try
{
if (cacheSizeLimitInGiB > 0)
{
_logger.LogInformation("Cleaning up files beyond the cache size limit");
var allLocalFiles = Directory.EnumerateFiles(_configuration["CacheDirectory"]).Select(f => new FileInfo(f)).ToList().OrderBy(f => f.LastAccessTimeUtc).ToList();
var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
long cacheSizeLimitInBytes = (long)(cacheSizeLimitInGiB * 1024 * 1024 * 1024);
HashSet<string> removedHashes = new();
while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any())
{
var oldestFile = allLocalFiles.First();
removedHashes.Add(oldestFile.Name.ToLower());
allLocalFiles.Remove(oldestFile);
totalCacheSizeInBytes -= oldestFile.Length;
_metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = oldestFile.Length });
_metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 });
oldestFile.Delete();
}
dbContext.Files.RemoveRange(dbContext.Files.Where(f => removedHashes.Contains(f.Hash.ToLower())));
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error during cache size limit cleanup");
}
_logger.LogInformation($"Cleanup complete");
dbContext.SaveChanges();
}
public Task StopAsync(CancellationToken cancellationToken)
{
_timer?.Change(Timeout.Infinite, 0);
return Task.CompletedTask;
}
public void Dispose()
{
_timer?.Dispose();
}
}

95
MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs Normal file
View File

@@ -0,0 +1,95 @@
using Grpc.Core;
using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Protos;
using Microsoft.EntityFrameworkCore;
using System.Security.Policy;
namespace MareSynchronosStaticFilesServer;
public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBase
{
private readonly string _basePath;
private readonly MareDbContext _mareDbContext;
private readonly ILogger<FileService> _logger;
private readonly MetricsService.MetricsServiceClient _metricsClient;
public FileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger<FileService> logger, MetricsService.MetricsServiceClient metricsClient)
{
_basePath = configuration["CacheDirectory"];
_mareDbContext = mareDbContext;
_logger = logger;
_metricsClient = metricsClient;
}
public override async Task<Empty> UploadFile(UploadFileRequest request, ServerCallContext context)
{
var filePath = Path.Combine(_basePath, request.Hash);
var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == request.Hash && f.UploaderUID == request.Uploader);
if (file != null)
{
var byteData = request.FileData.ToArray();
await File.WriteAllBytesAsync(filePath, byteData);
file.Uploaded = true;
await _metricsClient.IncGaugeAsync(new GaugeRequest()
{ GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = byteData.Length }).ConfigureAwait(false);
await _metricsClient.IncGaugeAsync(new GaugeRequest()
{ GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 }).ConfigureAwait(false);
await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
_logger.LogInformation("User {user} uploaded file {hash}", request.Uploader, request.Hash);
}
return new Empty();
}
public override async Task<Empty> DeleteFiles(DeleteFilesRequest request, ServerCallContext context)
{
foreach (var hash in request.Hash)
{
try
{
FileInfo fi = new FileInfo(Path.Combine(_basePath, hash));
fi.Delete();
var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
if (file != null)
{
_mareDbContext.Files.Remove(file);
await _metricsClient.DecGaugeAsync(new GaugeRequest()
{ GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = fi.Length }).ConfigureAwait(false);
await _metricsClient.DecGaugeAsync(new GaugeRequest()
{ GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 }).ConfigureAwait(false);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Could not delete file for hash {hash}", hash);
}
}
await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
return new Empty();
}
public override Task<FileSizeResponse> GetFileSizes(FileSizeRequest request, ServerCallContext context)
{
FileSizeResponse response = new();
foreach (var hash in request.Hash)
{
FileInfo fi = new(Path.Combine(_basePath, hash));
if (fi.Exists)
{
response.HashToFileSize.Add(hash, fi.Length);
}
else
{
response.HashToFileSize.Add(hash, 0);
}
}
return Task.FromResult(response);
}
}

26
MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs
View File

@@ -1,6 +1,5 @@
using MareSynchronosShared.Authentication;
using MareSynchronosShared.Data;
using MareSynchronosShared.Models;
using MareSynchronosShared.Protos;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authorization;
@@ -28,6 +27,19 @@ public class Startup
{
c.Address = new Uri(Configuration.GetValue<string>("ServiceAddress"));
});
services.AddGrpcClient<MetricsService.MetricsServiceClient>(c =>
{
c.Address = new Uri(Configuration.GetValue<string>("ServiceAddress"));
});
services.AddDbContextPool<MareDbContext>(options =>
{
options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder =>
{
builder.MigrationsHistoryTable("_efmigrationshistory", "public");
}).UseSnakeCaseNamingConvention();
options.EnableThreadSafetyChecks(false);
}, Configuration.GetValue("DbContextPoolSize", 1024));
services.AddAuthentication(options =>
{
@@ -35,6 +47,11 @@ public class Startup
})
.AddScheme<AuthenticationSchemeOptions, SecretKeyGrpcAuthenticationHandler>(SecretKeyGrpcAuthenticationHandler.AuthScheme, options => { });
services.AddAuthorization(options => options.FallbackPolicy = new AuthorizationPolicyBuilder().RequireAuthenticatedUser().Build());
services.AddGrpc(o =>
{
o.MaxReceiveMessageSize = null;
});
}
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
@@ -46,12 +63,17 @@ public class Startup
app.UseAuthentication();
app.UseAuthorization();
app.UseStaticFiles(new StaticFileOptions()
{
FileProvider = new PhysicalFileProvider(Configuration["CacheDirectory"]),
RequestPath = "/cache",
ServeUnknownFileTypes = true
});
app.UseEndpoints(e =>
{
e.MapGrpcService<FileService>();
});
}
}

2
MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json
View File

@@ -12,6 +12,8 @@
}
}
},
"CacheSizeHardLimitInGiB": -1,
"UnusedFileRetentionPeriodInDays": 7,
"AllowedHosts": "*",
"CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored
"ServiceAddress": "http://localhost:5002"