diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs index d4e7396..2c645e3 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs @@ -2,13 +2,11 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Runtime.CompilerServices; using System.Security.Cryptography; -using System.Threading; using System.Threading.Tasks; +using Google.Protobuf; using MareSynchronos.API; using MareSynchronosShared.Authentication; -using MareSynchronosShared.Metrics; using MareSynchronosShared.Models; using MareSynchronosShared.Protos; using Microsoft.AspNetCore.Authorization; @@ -20,8 +18,6 @@ namespace MareSynchronosServer.Hubs { public partial class MareHub { - private string BasePath => _configuration["CacheDirectory"]; - [Authorize(AuthenticationSchemes = SecretKeyGrpcAuthenticationHandler.AuthScheme)] [HubMethodName(Api.SendFileAbortUpload)] public async Task AbortUpload() @@ -40,20 +36,9 @@ namespace MareSynchronosServer.Hubs _logger.LogInformation("User {AuthenticatedUserId} deleted all their files", AuthenticatedUserId); var ownFiles = await _dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == AuthenticatedUserId).ToListAsync().ConfigureAwait(false); - foreach (var file in ownFiles) - { - var fi = new FileInfo(Path.Combine(BasePath, file.Hash)); - if (fi.Exists) - { - await _metricsClient.DecGaugeAsync(new GaugeRequest() - {GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = fi.Length}).ConfigureAwait(false); - await _metricsClient.DecGaugeAsync(new GaugeRequest() - { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1}).ConfigureAwait(false); - fi.Delete(); - } - } - _dbContext.Files.RemoveRange(ownFiles); - await _dbContext.SaveChangesAsync().ConfigureAwait(false); + var request = new DeleteFilesRequest(); + request.Hash.AddRange(ownFiles.Select(f => f.Hash)); + _ = await _fileServiceClient.DeleteFilesAsync(request).ConfigureAwait(false); } [Authorize(AuthenticationSchemes = SecretKeyGrpcAuthenticationHandler.AuthScheme)] @@ -64,37 +49,24 @@ namespace MareSynchronosServer.Hubs var forbiddenFiles = await _dbContext.ForbiddenUploadEntries. Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false); List response = new(); - foreach (var hash in hashes) - { - var fileInfo = new FileInfo(Path.Combine(BasePath, hash)); - long fileSize = 0; - try - { - fileSize = fileInfo.Length; - } - catch - { - // file doesn't exist anymore - } - var forbiddenFile = forbiddenFiles.SingleOrDefault(f => f.Hash == hash); - var downloadFile = allFiles.SingleOrDefault(f => f.Hash == hash); + FileSizeRequest request = new FileSizeRequest(); + var grpcResponse = await _fileServiceClient.GetFileSizesAsync(request).ConfigureAwait(false); + + foreach (var hash in grpcResponse.HashToFileSize) + { + var forbiddenFile = forbiddenFiles.SingleOrDefault(f => f.Hash == hash.Key); + var downloadFile = allFiles.SingleOrDefault(f => f.Hash == hash.Key); response.Add(new DownloadFileDto { - FileExists = fileInfo.Exists, + FileExists = hash.Value > 0, ForbiddenBy = forbiddenFile?.ForbiddenBy ?? string.Empty, IsForbidden = forbiddenFile != null, - Hash = hash, - Size = fileSize, - Url = _configuration["CdnFullUrl"] + hash.ToUpperInvariant() + Hash = hash.Key, + Size = hash.Value, + Url = _configuration["CdnFullUrl"] + hash.Key.ToUpperInvariant() }); - - if (!fileInfo.Exists && downloadFile != null) - { - _dbContext.Files.Remove(downloadFile); - await _dbContext.SaveChangesAsync().ConfigureAwait(false); - } } return response; @@ -169,8 +141,8 @@ namespace MareSynchronosServer.Hubs if (relatedFile == null) return; var forbiddenFile = _dbContext.ForbiddenUploadEntries.SingleOrDefault(f => f.Hash == hash); if (forbiddenFile != null) return; - var finalFileName = Path.Combine(BasePath, hash); - var tempFileName = finalFileName + ".tmp"; + + var tempFileName = Path.GetTempFileName(); using var fileStream = new FileStream(tempFileName, FileMode.OpenOrCreate); long length = 0; try @@ -223,17 +195,10 @@ namespace MareSynchronosServer.Hubs return; } - File.Move(tempFileName, finalFileName, true); - relatedFile = _dbContext.Files.Single(f => f.Hash == hash); - relatedFile.Uploaded = true; - - await _metricsClient.IncGaugeAsync(new GaugeRequest() - { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = length }).ConfigureAwait(false); - await _metricsClient.IncGaugeAsync(new GaugeRequest() - { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 }).ConfigureAwait(false); - - await _dbContext.SaveChangesAsync().ConfigureAwait(false); - _logger.LogInformation("File {hash} added to DB", hash); + UploadFileRequest req = new(); + req.FileData = ByteString.CopyFrom(await File.ReadAllBytesAsync(tempFileName).ConfigureAwait(false)); + File.Delete(tempFileName); + _ = await _fileServiceClient.UploadFileAsync(req).ConfigureAwait(false); } catch (Exception ex) { diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs index 413702c..a094533 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs @@ -1,7 +1,6 @@ using System; using System.Linq; using System.Security.Claims; -using System.Security.Cryptography; using System.Threading.Tasks; using MareSynchronos.API; using MareSynchronosShared.Authentication; @@ -22,22 +21,24 @@ namespace MareSynchronosServer.Hubs { private readonly MetricsService.MetricsServiceClient _metricsClient; private readonly AuthService.AuthServiceClient _authServiceClient; + private readonly FileService.FileServiceClient _fileServiceClient; private readonly SystemInfoService _systemInfoService; private readonly IConfiguration _configuration; private readonly IHttpContextAccessor contextAccessor; private readonly ILogger _logger; private readonly MareDbContext _dbContext; - - public MareHub(MetricsService.MetricsServiceClient metricsClient, AuthService.AuthServiceClient authServiceClient, + public MareHub(MetricsService.MetricsServiceClient metricsClient, AuthService.AuthServiceClient authServiceClient, FileService.FileServiceClient fileServiceClient, MareDbContext mareDbContext, ILogger logger, SystemInfoService systemInfoService, IConfiguration configuration, IHttpContextAccessor contextAccessor) { _metricsClient = metricsClient; _authServiceClient = authServiceClient; + _fileServiceClient = fileServiceClient; _systemInfoService = systemInfoService; _configuration = configuration; this.contextAccessor = contextAccessor; _logger = logger; _dbContext = mareDbContext; + _staticFileAddress = new Uri(_configuration["StaticFileServiceAddress"]); } [HubMethodName(Api.InvokeHeartbeat)] diff --git a/MareSynchronosServer/MareSynchronosServer/Startup.cs b/MareSynchronosServer/MareSynchronosServer/Startup.cs index 57fca87..374f236 100644 --- a/MareSynchronosServer/MareSynchronosServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosServer/Startup.cs @@ -53,6 +53,10 @@ namespace MareSynchronosServer { c.Address = new Uri(Configuration.GetValue("ServiceAddress")); }); + services.AddGrpcClient(c => + { + c.Address = new Uri(Configuration.GetValue("StaticFileServiceAddress")); + }); services.AddDbContextPool(options => { diff --git a/MareSynchronosServer/MareSynchronosServer/appsettings.json b/MareSynchronosServer/MareSynchronosServer/appsettings.json index 20bad5c..1883cef 100644 --- a/MareSynchronosServer/MareSynchronosServer/appsettings.json +++ b/MareSynchronosServer/MareSynchronosServer/appsettings.json @@ -25,9 +25,8 @@ }, "DbContextPoolSize": 2000, "CdnFullUrl": "https:///cache/", - "CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored "ServiceAddress": "http://localhost:5002", - + "StaticFileServiceAddress": "http://localhost:5001", "AllowedHosts": "*", "Kestrel": { "Endpoints": { @@ -51,14 +50,8 @@ "RealIpHeader": "X-Real-IP", "ClientIdHeader": "X-ClientId", "HttpStatusCode": 429, - "IpWhitelist": [ "127.0.0.1", "::1/10", "192.168.0.0/24" ], - "GeneralRules": [ - { - "Endpoint": "*", - "Period": "1s", - "Limit": 2 - } - ] + "IpWhitelist": [ ], + "GeneralRules": [ ] }, "IPRateLimitPolicies": { "IpRules": [] diff --git a/MareSynchronosServer/MareSynchronosServices/Program.cs b/MareSynchronosServer/MareSynchronosServices/Program.cs index 09607b5..8da5729 100644 --- a/MareSynchronosServer/MareSynchronosServices/Program.cs +++ b/MareSynchronosServer/MareSynchronosServices/Program.cs @@ -1,7 +1,4 @@ using MareSynchronosServices; -using MareSynchronosServices.Metrics; -using MareSynchronosShared.Data; -using Microsoft.EntityFrameworkCore; public class Program { diff --git a/MareSynchronosServer/MareSynchronosServices/appsettings.json b/MareSynchronosServer/MareSynchronosServices/appsettings.json index fdc0173..51429f3 100644 --- a/MareSynchronosServer/MareSynchronosServices/appsettings.json +++ b/MareSynchronosServer/MareSynchronosServices/appsettings.json @@ -18,10 +18,8 @@ }, "DbContextPoolSize": 1024, "DiscordBotToken": "", - "UnusedFileRetentionPeriodInDays": 7, "PurgeUnusedAccounts": true, "PurgeUnusedAccountsPeriodInDays": 14, - "CacheSizeHardLimitInGiB": -1, "CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored "FailedAuthForTempBan": 5, "TempBanDurationInMinutes": 30, diff --git a/MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto b/MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto index 96e405d..1aba480 100644 --- a/MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto +++ b/MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto @@ -17,8 +17,32 @@ service MetricsService { rpc IncGauge (GaugeRequest) returns (Empty); } +service FileService { + rpc UploadFile (UploadFileRequest) returns (Empty); + rpc GetFileSizes (FileSizeRequest) returns (FileSizeResponse); + rpc DeleteFiles (DeleteFilesRequest) returns (Empty); +} + message Empty { } +message UploadFileRequest { + string hash = 1; + string uploader = 2; + bytes fileData = 3; +} + +message DeleteFilesRequest { + repeated string hash = 1; +} + +message FileSizeRequest { + repeated string hash = 1; +} + +message FileSizeResponse { + map hashToFileSize = 1; +} + message GaugeRequest { string gaugeName = 1; double value = 2; diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs new file mode 100644 index 0000000..72c6cca --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs @@ -0,0 +1,120 @@ +using MareSynchronosShared.Data; +using MareSynchronosShared.Metrics; +using static MareSynchronosShared.Protos.MetricsService; + +namespace MareSynchronosStaticFilesServer; + +public class CleanupService : IHostedService, IDisposable +{ + private readonly MetricsServiceClient _metrics; + private readonly ILogger _logger; + private readonly IServiceProvider _services; + private readonly IConfiguration _configuration; + private Timer? _timer; + + public CleanupService(MetricsServiceClient metrics, ILogger logger, IServiceProvider services, IConfiguration configuration) + { + _metrics = metrics; + _logger = logger; + _services = services; + _configuration = configuration; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Cleanup Service started"); + + _timer = new Timer(CleanUp, null, TimeSpan.Zero, TimeSpan.FromMinutes(10)); + + return Task.CompletedTask; + } + + private void CleanUp(object? state) + { + if (!int.TryParse(_configuration["UnusedFileRetentionPeriodInDays"], out var filesOlderThanDays)) + { + filesOlderThanDays = 7; + } + + using var scope = _services.CreateScope(); + using var dbContext = scope.ServiceProvider.GetService()!; + + _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", filesOlderThanDays); + + try + { + var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays)); + + var allFiles = dbContext.Files.ToList(); + var cachedir = _configuration["CacheDirectory"]; + foreach (var file in allFiles.Where(f => f.Uploaded)) + { + var fileName = Path.Combine(cachedir, file.Hash); + var fi = new FileInfo(fileName); + if (!fi.Exists) + { + _logger.LogInformation("File does not exist anymore: {fileName}", fileName); + dbContext.Files.Remove(file); + } + else if (fi.LastAccessTime < prevTime) + { + _metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = fi.Length }); + _metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 }); + _logger.LogInformation("File outdated: {fileName}", fileName); + dbContext.Files.Remove(file); + fi.Delete(); + } + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error during file cleanup"); + } + + var cacheSizeLimitInGiB = _configuration.GetValue("CacheSizeHardLimitInGiB", -1); + + try + { + if (cacheSizeLimitInGiB > 0) + { + _logger.LogInformation("Cleaning up files beyond the cache size limit"); + var allLocalFiles = Directory.EnumerateFiles(_configuration["CacheDirectory"]).Select(f => new FileInfo(f)).ToList().OrderBy(f => f.LastAccessTimeUtc).ToList(); + var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length); + long cacheSizeLimitInBytes = (long)(cacheSizeLimitInGiB * 1024 * 1024 * 1024); + HashSet removedHashes = new(); + while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any()) + { + var oldestFile = allLocalFiles.First(); + removedHashes.Add(oldestFile.Name.ToLower()); + allLocalFiles.Remove(oldestFile); + totalCacheSizeInBytes -= oldestFile.Length; + _metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = oldestFile.Length }); + _metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 }); + oldestFile.Delete(); + } + + dbContext.Files.RemoveRange(dbContext.Files.Where(f => removedHashes.Contains(f.Hash.ToLower()))); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error during cache size limit cleanup"); + } + + _logger.LogInformation($"Cleanup complete"); + + dbContext.SaveChanges(); + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _timer?.Change(Timeout.Infinite, 0); + + return Task.CompletedTask; + } + + public void Dispose() + { + _timer?.Dispose(); + } +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs new file mode 100644 index 0000000..a7a02f6 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs @@ -0,0 +1,95 @@ +using Grpc.Core; +using MareSynchronosShared.Data; +using MareSynchronosShared.Metrics; +using MareSynchronosShared.Protos; +using Microsoft.EntityFrameworkCore; +using System.Security.Policy; + +namespace MareSynchronosStaticFilesServer; + +public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBase +{ + private readonly string _basePath; + private readonly MareDbContext _mareDbContext; + private readonly ILogger _logger; + private readonly MetricsService.MetricsServiceClient _metricsClient; + + public FileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger logger, MetricsService.MetricsServiceClient metricsClient) + { + _basePath = configuration["CacheDirectory"]; + _mareDbContext = mareDbContext; + _logger = logger; + _metricsClient = metricsClient; + } + + public override async Task UploadFile(UploadFileRequest request, ServerCallContext context) + { + + var filePath = Path.Combine(_basePath, request.Hash); + var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == request.Hash && f.UploaderUID == request.Uploader); + if (file != null) + { + var byteData = request.FileData.ToArray(); + await File.WriteAllBytesAsync(filePath, byteData); + file.Uploaded = true; + + await _metricsClient.IncGaugeAsync(new GaugeRequest() + { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = byteData.Length }).ConfigureAwait(false); + await _metricsClient.IncGaugeAsync(new GaugeRequest() + { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 }).ConfigureAwait(false); + + await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); + _logger.LogInformation("User {user} uploaded file {hash}", request.Uploader, request.Hash); + } + + return new Empty(); + } + + public override async Task DeleteFiles(DeleteFilesRequest request, ServerCallContext context) + { + foreach (var hash in request.Hash) + { + try + { + FileInfo fi = new FileInfo(Path.Combine(_basePath, hash)); + fi.Delete(); + var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash); + if (file != null) + { + _mareDbContext.Files.Remove(file); + + await _metricsClient.DecGaugeAsync(new GaugeRequest() + { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = fi.Length }).ConfigureAwait(false); + await _metricsClient.DecGaugeAsync(new GaugeRequest() + { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 }).ConfigureAwait(false); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Could not delete file for hash {hash}", hash); + } + } + + await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); + return new Empty(); + } + + public override Task GetFileSizes(FileSizeRequest request, ServerCallContext context) + { + FileSizeResponse response = new(); + foreach (var hash in request.Hash) + { + FileInfo fi = new(Path.Combine(_basePath, hash)); + if (fi.Exists) + { + response.HashToFileSize.Add(hash, fi.Length); + } + else + { + response.HashToFileSize.Add(hash, 0); + } + } + + return Task.FromResult(response); + } +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs index c1f99a9..cb3760c 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs @@ -1,6 +1,5 @@ using MareSynchronosShared.Authentication; using MareSynchronosShared.Data; -using MareSynchronosShared.Models; using MareSynchronosShared.Protos; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authorization; @@ -28,6 +27,19 @@ public class Startup { c.Address = new Uri(Configuration.GetValue("ServiceAddress")); }); + services.AddGrpcClient(c => + { + c.Address = new Uri(Configuration.GetValue("ServiceAddress")); + }); + + services.AddDbContextPool(options => + { + options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder => + { + builder.MigrationsHistoryTable("_efmigrationshistory", "public"); + }).UseSnakeCaseNamingConvention(); + options.EnableThreadSafetyChecks(false); + }, Configuration.GetValue("DbContextPoolSize", 1024)); services.AddAuthentication(options => { @@ -35,6 +47,11 @@ public class Startup }) .AddScheme(SecretKeyGrpcAuthenticationHandler.AuthScheme, options => { }); services.AddAuthorization(options => options.FallbackPolicy = new AuthorizationPolicyBuilder().RequireAuthenticatedUser().Build()); + + services.AddGrpc(o => + { + o.MaxReceiveMessageSize = null; + }); } public void Configure(IApplicationBuilder app, IWebHostEnvironment env) @@ -46,12 +63,17 @@ public class Startup app.UseAuthentication(); app.UseAuthorization(); - + app.UseStaticFiles(new StaticFileOptions() { FileProvider = new PhysicalFileProvider(Configuration["CacheDirectory"]), RequestPath = "/cache", ServeUnknownFileTypes = true }); + + app.UseEndpoints(e => + { + e.MapGrpcService(); + }); } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json b/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json index 7e9426e..2e9aa47 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json @@ -12,6 +12,8 @@ } } }, + "CacheSizeHardLimitInGiB": -1, + "UnusedFileRetentionPeriodInDays": 7, "AllowedHosts": "*", "CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored "ServiceAddress": "http://localhost:5002"