decouple fileservice to be able to run standalone

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs

@@ -2,13 +2,11 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Runtime.CompilerServices;
 using System.Security.Cryptography;
-using System.Threading;
 using System.Threading.Tasks;
+using Google.Protobuf;
 using MareSynchronos.API;
 using MareSynchronosShared.Authentication;
-using MareSynchronosShared.Metrics;
 using MareSynchronosShared.Models;
 using MareSynchronosShared.Protos;
 using Microsoft.AspNetCore.Authorization;
@@ -20,8 +18,6 @@ namespace MareSynchronosServer.Hubs
 {
     public partial class MareHub
     {
-        private string BasePath => _configuration["CacheDirectory"];
-
         [Authorize(AuthenticationSchemes = SecretKeyGrpcAuthenticationHandler.AuthScheme)]
         [HubMethodName(Api.SendFileAbortUpload)]
         public async Task AbortUpload()
@@ -40,20 +36,9 @@ namespace MareSynchronosServer.Hubs
             _logger.LogInformation("User {AuthenticatedUserId} deleted all their files", AuthenticatedUserId);
 
             var ownFiles = await _dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == AuthenticatedUserId).ToListAsync().ConfigureAwait(false);
-            foreach (var file in ownFiles)
+            var request = new DeleteFilesRequest();
-            {
+            request.Hash.AddRange(ownFiles.Select(f => f.Hash));
-                var fi = new FileInfo(Path.Combine(BasePath, file.Hash));
+            _ = await _fileServiceClient.DeleteFilesAsync(request).ConfigureAwait(false);
-                if (fi.Exists)
-                {
-                    await _metricsClient.DecGaugeAsync(new GaugeRequest()
-                        {GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = fi.Length}).ConfigureAwait(false);
-                    await _metricsClient.DecGaugeAsync(new GaugeRequest()
-                        { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1}).ConfigureAwait(false);
-                    fi.Delete();
-                }
-            }
-            _dbContext.Files.RemoveRange(ownFiles);
-            await _dbContext.SaveChangesAsync().ConfigureAwait(false);
         }
 
         [Authorize(AuthenticationSchemes = SecretKeyGrpcAuthenticationHandler.AuthScheme)]
@@ -64,37 +49,24 @@ namespace MareSynchronosServer.Hubs
             var forbiddenFiles = await _dbContext.ForbiddenUploadEntries.
                 Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false);
             List<DownloadFileDto> response = new();
-            foreach (var hash in hashes)
-            {
-                var fileInfo = new FileInfo(Path.Combine(BasePath, hash));
-                long fileSize = 0;
-                try
-                {
-                    fileSize = fileInfo.Length;
-                }
-                catch
-                {
-                    // file doesn't exist anymore
-                }
 
-                var forbiddenFile = forbiddenFiles.SingleOrDefault(f => f.Hash == hash);
+            FileSizeRequest request = new FileSizeRequest();
-                var downloadFile = allFiles.SingleOrDefault(f => f.Hash == hash);
+            var grpcResponse = await _fileServiceClient.GetFileSizesAsync(request).ConfigureAwait(false);
+
+            foreach (var hash in grpcResponse.HashToFileSize)
+            {
+                var forbiddenFile = forbiddenFiles.SingleOrDefault(f => f.Hash == hash.Key);
+                var downloadFile = allFiles.SingleOrDefault(f => f.Hash == hash.Key);
 
                 response.Add(new DownloadFileDto
                 {
-                    FileExists = fileInfo.Exists,
+                    FileExists = hash.Value > 0,
                     ForbiddenBy = forbiddenFile?.ForbiddenBy ?? string.Empty,
                     IsForbidden = forbiddenFile != null,
-                    Hash = hash,
+                    Hash = hash.Key,
-                    Size = fileSize,
+                    Size = hash.Value,
-                    Url = _configuration["CdnFullUrl"] + hash.ToUpperInvariant()
+                    Url = _configuration["CdnFullUrl"] + hash.Key.ToUpperInvariant()
                 });
-
-                if (!fileInfo.Exists && downloadFile != null)
-                {
-                    _dbContext.Files.Remove(downloadFile);
-                    await _dbContext.SaveChangesAsync().ConfigureAwait(false);
-                }
             }
 
             return response;
@@ -169,8 +141,8 @@ namespace MareSynchronosServer.Hubs
             if (relatedFile == null) return;
             var forbiddenFile = _dbContext.ForbiddenUploadEntries.SingleOrDefault(f => f.Hash == hash);
             if (forbiddenFile != null) return;
-            var finalFileName = Path.Combine(BasePath, hash);
+
-            var tempFileName = finalFileName + ".tmp";
+            var tempFileName = Path.GetTempFileName();
             using var fileStream = new FileStream(tempFileName, FileMode.OpenOrCreate);
             long length = 0;
             try
@@ -223,17 +195,10 @@ namespace MareSynchronosServer.Hubs
                     return;
                 }
 
-                File.Move(tempFileName, finalFileName, true);
+                UploadFileRequest req = new();
-                relatedFile = _dbContext.Files.Single(f => f.Hash == hash);
+                req.FileData = ByteString.CopyFrom(await File.ReadAllBytesAsync(tempFileName).ConfigureAwait(false));
-                relatedFile.Uploaded = true;
+                File.Delete(tempFileName);
-
+                _ = await _fileServiceClient.UploadFileAsync(req).ConfigureAwait(false);
-                await _metricsClient.IncGaugeAsync(new GaugeRequest()
-                    { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = length }).ConfigureAwait(false);
-                await _metricsClient.IncGaugeAsync(new GaugeRequest()
-                    { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 }).ConfigureAwait(false);
-
-                await _dbContext.SaveChangesAsync().ConfigureAwait(false);
-                _logger.LogInformation("File {hash} added to DB", hash);
             }
             catch (Exception ex)
             {

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Linq;
 using System.Security.Claims;
-using System.Security.Cryptography;
 using System.Threading.Tasks;
 using MareSynchronos.API;
 using MareSynchronosShared.Authentication;
@@ -22,22 +21,24 @@ namespace MareSynchronosServer.Hubs
     {
         private readonly MetricsService.MetricsServiceClient _metricsClient;
         private readonly AuthService.AuthServiceClient _authServiceClient;
+        private readonly FileService.FileServiceClient _fileServiceClient;
         private readonly SystemInfoService _systemInfoService;
         private readonly IConfiguration _configuration;
         private readonly IHttpContextAccessor contextAccessor;
         private readonly ILogger<MareHub> _logger;
         private readonly MareDbContext _dbContext;
-
+        public MareHub(MetricsService.MetricsServiceClient metricsClient, AuthService.AuthServiceClient authServiceClient, FileService.FileServiceClient fileServiceClient,
-        public MareHub(MetricsService.MetricsServiceClient metricsClient, AuthService.AuthServiceClient authServiceClient,
             MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService, IConfiguration configuration, IHttpContextAccessor contextAccessor)
         {
             _metricsClient = metricsClient;
             _authServiceClient = authServiceClient;
+            _fileServiceClient = fileServiceClient;
             _systemInfoService = systemInfoService;
             _configuration = configuration;
             this.contextAccessor = contextAccessor;
             _logger = logger;
             _dbContext = mareDbContext;
+            _staticFileAddress = new Uri(_configuration["StaticFileServiceAddress"]);
         }
 
         [HubMethodName(Api.InvokeHeartbeat)]

MareSynchronosServer/MareSynchronosServer/Startup.cs

@@ -53,6 +53,10 @@ namespace MareSynchronosServer
             {
                 c.Address = new Uri(Configuration.GetValue<string>("ServiceAddress"));
             });
+            services.AddGrpcClient<FileService.FileServiceClient>(c =>
+            {
+                c.Address = new Uri(Configuration.GetValue<string>("StaticFileServiceAddress"));
+            });
 
             services.AddDbContextPool<MareDbContext>(options =>
             {

MareSynchronosServer/MareSynchronosServer/appsettings.json

@@ -25,9 +25,8 @@
   },
   "DbContextPoolSize": 2000,
   "CdnFullUrl": "https://<url or ip to your server>/cache/",
-  "CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored
   "ServiceAddress": "http://localhost:5002",
-
+  "StaticFileServiceAddress": "http://localhost:5001",
   "AllowedHosts": "*",
   "Kestrel": {
     "Endpoints": {
@@ -51,14 +50,8 @@
     "RealIpHeader": "X-Real-IP",
     "ClientIdHeader": "X-ClientId",
     "HttpStatusCode": 429,
-    "IpWhitelist": [ "127.0.0.1", "::1/10", "192.168.0.0/24" ],
+    "IpWhitelist": [ ],
-    "GeneralRules": [
+    "GeneralRules": [ ]
-      {
-        "Endpoint": "*",
-        "Period": "1s",
-        "Limit": 2
-      }
-    ]
   },
   "IPRateLimitPolicies": {
     "IpRules": []

MareSynchronosServer/MareSynchronosServices/Program.cs

@@ -1,7 +1,4 @@
 using MareSynchronosServices;
-using MareSynchronosServices.Metrics;
-using MareSynchronosShared.Data;
-using Microsoft.EntityFrameworkCore;
 
 public class Program
 {

MareSynchronosServer/MareSynchronosServices/appsettings.json

@@ -18,10 +18,8 @@
   },
   "DbContextPoolSize": 1024,
   "DiscordBotToken": "",
-  "UnusedFileRetentionPeriodInDays": 7,
   "PurgeUnusedAccounts": true,
   "PurgeUnusedAccountsPeriodInDays": 14,
-  "CacheSizeHardLimitInGiB": -1,
   "CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored
   "FailedAuthForTempBan": 5,
   "TempBanDurationInMinutes": 30,

MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto

@@ -17,8 +17,32 @@ service MetricsService {
 	rpc IncGauge (GaugeRequest) returns (Empty);
 }
 
+service FileService {
+	rpc UploadFile (UploadFileRequest) returns (Empty);
+	rpc GetFileSizes (FileSizeRequest) returns (FileSizeResponse);
+	rpc DeleteFiles (DeleteFilesRequest) returns (Empty);
+}
+
 message Empty { }
 
+message UploadFileRequest {
+	string hash = 1;
+	string uploader = 2;
+	bytes fileData = 3;
+}
+
+message DeleteFilesRequest {
+	repeated string hash = 1;
+}
+
+message FileSizeRequest {
+	repeated string hash = 1;
+}
+
+message FileSizeResponse {
+	map<string, int64> hashToFileSize = 1;
+}
+
 message GaugeRequest {
 	string gaugeName = 1;
 	double value = 2;

MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs

@@ -0,0 +1,120 @@
+using MareSynchronosShared.Data;
+using MareSynchronosShared.Metrics;
+using static MareSynchronosShared.Protos.MetricsService;
+
+namespace MareSynchronosStaticFilesServer;
+
+public class CleanupService : IHostedService, IDisposable
+{
+    private readonly MetricsServiceClient _metrics;
+    private readonly ILogger<CleanupService> _logger;
+    private readonly IServiceProvider _services;
+    private readonly IConfiguration _configuration;
+    private Timer? _timer;
+
+    public CleanupService(MetricsServiceClient metrics, ILogger<CleanupService> logger, IServiceProvider services, IConfiguration configuration)
+    {
+        _metrics = metrics;
+        _logger = logger;
+        _services = services;
+        _configuration = configuration;
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Cleanup Service started");
+
+        _timer = new Timer(CleanUp, null, TimeSpan.Zero, TimeSpan.FromMinutes(10));
+
+        return Task.CompletedTask;
+    }
+
+    private void CleanUp(object? state)
+    {
+        if (!int.TryParse(_configuration["UnusedFileRetentionPeriodInDays"], out var filesOlderThanDays))
+        {
+            filesOlderThanDays = 7;
+        }
+
+        using var scope = _services.CreateScope();
+        using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
+
+        _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", filesOlderThanDays);
+
+        try
+        {
+            var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays));
+
+            var allFiles = dbContext.Files.ToList();
+            var cachedir = _configuration["CacheDirectory"];
+            foreach (var file in allFiles.Where(f => f.Uploaded))
+            {
+                var fileName = Path.Combine(cachedir, file.Hash);
+                var fi = new FileInfo(fileName);
+                if (!fi.Exists)
+                {
+                    _logger.LogInformation("File does not exist anymore: {fileName}", fileName);
+                    dbContext.Files.Remove(file);
+                }
+                else if (fi.LastAccessTime < prevTime)
+                {
+                    _metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = fi.Length });
+                    _metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 });
+                    _logger.LogInformation("File outdated: {fileName}", fileName);
+                    dbContext.Files.Remove(file);
+                    fi.Delete();
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error during file cleanup");
+        }
+
+        var cacheSizeLimitInGiB = _configuration.GetValue<double>("CacheSizeHardLimitInGiB", -1);
+
+        try
+        {
+            if (cacheSizeLimitInGiB > 0)
+            {
+                _logger.LogInformation("Cleaning up files beyond the cache size limit");
+                var allLocalFiles = Directory.EnumerateFiles(_configuration["CacheDirectory"]).Select(f => new FileInfo(f)).ToList().OrderBy(f => f.LastAccessTimeUtc).ToList();
+                var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
+                long cacheSizeLimitInBytes = (long)(cacheSizeLimitInGiB * 1024 * 1024 * 1024);
+                HashSet<string> removedHashes = new();
+                while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any())
+                {
+                    var oldestFile = allLocalFiles.First();
+                    removedHashes.Add(oldestFile.Name.ToLower());
+                    allLocalFiles.Remove(oldestFile);
+                    totalCacheSizeInBytes -= oldestFile.Length;
+                    _metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = oldestFile.Length });
+                    _metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 });
+                    oldestFile.Delete();
+                }
+
+                dbContext.Files.RemoveRange(dbContext.Files.Where(f => removedHashes.Contains(f.Hash.ToLower())));
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error during cache size limit cleanup");
+        }
+
+        _logger.LogInformation($"Cleanup complete");
+
+        dbContext.SaveChanges();
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        _timer?.Change(Timeout.Infinite, 0);
+
+        return Task.CompletedTask;
+    }
+
+    public void Dispose()
+    {
+        _timer?.Dispose();
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs

@@ -0,0 +1,95 @@
+using Grpc.Core;
+using MareSynchronosShared.Data;
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Protos;
+using Microsoft.EntityFrameworkCore;
+using System.Security.Policy;
+
+namespace MareSynchronosStaticFilesServer;
+
+public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBase
+{
+    private readonly string _basePath;
+    private readonly MareDbContext _mareDbContext;
+    private readonly ILogger<FileService> _logger;
+    private readonly MetricsService.MetricsServiceClient _metricsClient;
+
+    public FileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger<FileService> logger, MetricsService.MetricsServiceClient metricsClient)
+    {
+        _basePath = configuration["CacheDirectory"];
+        _mareDbContext = mareDbContext;
+        _logger = logger;
+        _metricsClient = metricsClient;
+    }
+
+    public override async Task<Empty> UploadFile(UploadFileRequest request, ServerCallContext context)
+    {
+
+        var filePath = Path.Combine(_basePath, request.Hash);
+        var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == request.Hash && f.UploaderUID == request.Uploader);
+        if (file != null)
+        {
+            var byteData = request.FileData.ToArray();
+            await File.WriteAllBytesAsync(filePath, byteData);
+            file.Uploaded = true;
+
+            await _metricsClient.IncGaugeAsync(new GaugeRequest()
+            { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = byteData.Length }).ConfigureAwait(false);
+            await _metricsClient.IncGaugeAsync(new GaugeRequest()
+            { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 }).ConfigureAwait(false);
+
+            await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
+            _logger.LogInformation("User {user} uploaded file {hash}", request.Uploader, request.Hash);
+        }
+
+        return new Empty();
+    }
+
+    public override async Task<Empty> DeleteFiles(DeleteFilesRequest request, ServerCallContext context)
+    {
+        foreach (var hash in request.Hash)
+        {
+            try
+            {
+                FileInfo fi = new FileInfo(Path.Combine(_basePath, hash));
+                fi.Delete();
+                var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
+                if (file != null)
+                {
+                    _mareDbContext.Files.Remove(file);
+
+                    await _metricsClient.DecGaugeAsync(new GaugeRequest()
+                    { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = fi.Length }).ConfigureAwait(false);
+                    await _metricsClient.DecGaugeAsync(new GaugeRequest()
+                    { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 }).ConfigureAwait(false);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Could not delete file for hash {hash}", hash);
+            }
+        }
+
+        await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
+        return new Empty();
+    }
+
+    public override Task<FileSizeResponse> GetFileSizes(FileSizeRequest request, ServerCallContext context)
+    {
+        FileSizeResponse response = new();
+        foreach (var hash in request.Hash)
+        {
+            FileInfo fi = new(Path.Combine(_basePath, hash));
+            if (fi.Exists)
+            {
+                response.HashToFileSize.Add(hash, fi.Length);
+            }
+            else
+            {
+                response.HashToFileSize.Add(hash, 0);
+            }
+        }
+
+        return Task.FromResult(response);
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs

@@ -1,6 +1,5 @@
 using MareSynchronosShared.Authentication;
 using MareSynchronosShared.Data;
-using MareSynchronosShared.Models;
 using MareSynchronosShared.Protos;
 using Microsoft.AspNetCore.Authentication;
 using Microsoft.AspNetCore.Authorization;
@@ -28,6 +27,19 @@ public class Startup
         {
             c.Address = new Uri(Configuration.GetValue<string>("ServiceAddress"));
         });
+        services.AddGrpcClient<MetricsService.MetricsServiceClient>(c =>
+        {
+            c.Address = new Uri(Configuration.GetValue<string>("ServiceAddress"));
+        });
+
+        services.AddDbContextPool<MareDbContext>(options =>
+        {
+            options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder =>
+            {
+                builder.MigrationsHistoryTable("_efmigrationshistory", "public");
+            }).UseSnakeCaseNamingConvention();
+            options.EnableThreadSafetyChecks(false);
+        }, Configuration.GetValue("DbContextPoolSize", 1024));
 
         services.AddAuthentication(options =>
             {
@@ -35,6 +47,11 @@ public class Startup
             })
             .AddScheme<AuthenticationSchemeOptions, SecretKeyGrpcAuthenticationHandler>(SecretKeyGrpcAuthenticationHandler.AuthScheme, options => { });
         services.AddAuthorization(options => options.FallbackPolicy = new AuthorizationPolicyBuilder().RequireAuthenticatedUser().Build());
+
+        services.AddGrpc(o =>
+        {
+            o.MaxReceiveMessageSize = null;
+        });
     }
 
     public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
@@ -46,12 +63,17 @@ public class Startup
 
         app.UseAuthentication();
         app.UseAuthorization();
-        
+
         app.UseStaticFiles(new StaticFileOptions()
         {
             FileProvider = new PhysicalFileProvider(Configuration["CacheDirectory"]),
             RequestPath = "/cache",
             ServeUnknownFileTypes = true
         });
+
+        app.UseEndpoints(e =>
+        {
+            e.MapGrpcService<FileService>();
+        });
     }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json

@@ -12,6 +12,8 @@
       }
     }
   },
+  "CacheSizeHardLimitInGiB": -1,
+  "UnusedFileRetentionPeriodInDays": 7,
   "AllowedHosts": "*",
   "CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored
   "ServiceAddress": "http://localhost:5002"