decouple fileservice to be able to run standalone

MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs

@@ -0,0 +1,120 @@
+using MareSynchronosShared.Data;
+using MareSynchronosShared.Metrics;
+using static MareSynchronosShared.Protos.MetricsService;
+
+namespace MareSynchronosStaticFilesServer;
+
+public class CleanupService : IHostedService, IDisposable
+{
+    private readonly MetricsServiceClient _metrics;
+    private readonly ILogger<CleanupService> _logger;
+    private readonly IServiceProvider _services;
+    private readonly IConfiguration _configuration;
+    private Timer? _timer;
+
+    public CleanupService(MetricsServiceClient metrics, ILogger<CleanupService> logger, IServiceProvider services, IConfiguration configuration)
+    {
+        _metrics = metrics;
+        _logger = logger;
+        _services = services;
+        _configuration = configuration;
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Cleanup Service started");
+
+        _timer = new Timer(CleanUp, null, TimeSpan.Zero, TimeSpan.FromMinutes(10));
+
+        return Task.CompletedTask;
+    }
+
+    private void CleanUp(object? state)
+    {
+        if (!int.TryParse(_configuration["UnusedFileRetentionPeriodInDays"], out var filesOlderThanDays))
+        {
+            filesOlderThanDays = 7;
+        }
+
+        using var scope = _services.CreateScope();
+        using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
+
+        _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", filesOlderThanDays);
+
+        try
+        {
+            var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays));
+
+            var allFiles = dbContext.Files.ToList();
+            var cachedir = _configuration["CacheDirectory"];
+            foreach (var file in allFiles.Where(f => f.Uploaded))
+            {
+                var fileName = Path.Combine(cachedir, file.Hash);
+                var fi = new FileInfo(fileName);
+                if (!fi.Exists)
+                {
+                    _logger.LogInformation("File does not exist anymore: {fileName}", fileName);
+                    dbContext.Files.Remove(file);
+                }
+                else if (fi.LastAccessTime < prevTime)
+                {
+                    _metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = fi.Length });
+                    _metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 });
+                    _logger.LogInformation("File outdated: {fileName}", fileName);
+                    dbContext.Files.Remove(file);
+                    fi.Delete();
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error during file cleanup");
+        }
+
+        var cacheSizeLimitInGiB = _configuration.GetValue<double>("CacheSizeHardLimitInGiB", -1);
+
+        try
+        {
+            if (cacheSizeLimitInGiB > 0)
+            {
+                _logger.LogInformation("Cleaning up files beyond the cache size limit");
+                var allLocalFiles = Directory.EnumerateFiles(_configuration["CacheDirectory"]).Select(f => new FileInfo(f)).ToList().OrderBy(f => f.LastAccessTimeUtc).ToList();
+                var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
+                long cacheSizeLimitInBytes = (long)(cacheSizeLimitInGiB * 1024 * 1024 * 1024);
+                HashSet<string> removedHashes = new();
+                while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any())
+                {
+                    var oldestFile = allLocalFiles.First();
+                    removedHashes.Add(oldestFile.Name.ToLower());
+                    allLocalFiles.Remove(oldestFile);
+                    totalCacheSizeInBytes -= oldestFile.Length;
+                    _metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = oldestFile.Length });
+                    _metrics.DecGauge(new() { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 });
+                    oldestFile.Delete();
+                }
+
+                dbContext.Files.RemoveRange(dbContext.Files.Where(f => removedHashes.Contains(f.Hash.ToLower())));
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error during cache size limit cleanup");
+        }
+
+        _logger.LogInformation($"Cleanup complete");
+
+        dbContext.SaveChanges();
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        _timer?.Change(Timeout.Infinite, 0);
+
+        return Task.CompletedTask;
+    }
+
+    public void Dispose()
+    {
+        _timer?.Dispose();
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs

@@ -0,0 +1,95 @@
+using Grpc.Core;
+using MareSynchronosShared.Data;
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Protos;
+using Microsoft.EntityFrameworkCore;
+using System.Security.Policy;
+
+namespace MareSynchronosStaticFilesServer;
+
+public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBase
+{
+    private readonly string _basePath;
+    private readonly MareDbContext _mareDbContext;
+    private readonly ILogger<FileService> _logger;
+    private readonly MetricsService.MetricsServiceClient _metricsClient;
+
+    public FileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger<FileService> logger, MetricsService.MetricsServiceClient metricsClient)
+    {
+        _basePath = configuration["CacheDirectory"];
+        _mareDbContext = mareDbContext;
+        _logger = logger;
+        _metricsClient = metricsClient;
+    }
+
+    public override async Task<Empty> UploadFile(UploadFileRequest request, ServerCallContext context)
+    {
+
+        var filePath = Path.Combine(_basePath, request.Hash);
+        var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == request.Hash && f.UploaderUID == request.Uploader);
+        if (file != null)
+        {
+            var byteData = request.FileData.ToArray();
+            await File.WriteAllBytesAsync(filePath, byteData);
+            file.Uploaded = true;
+
+            await _metricsClient.IncGaugeAsync(new GaugeRequest()
+            { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = byteData.Length }).ConfigureAwait(false);
+            await _metricsClient.IncGaugeAsync(new GaugeRequest()
+            { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 }).ConfigureAwait(false);
+
+            await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
+            _logger.LogInformation("User {user} uploaded file {hash}", request.Uploader, request.Hash);
+        }
+
+        return new Empty();
+    }
+
+    public override async Task<Empty> DeleteFiles(DeleteFilesRequest request, ServerCallContext context)
+    {
+        foreach (var hash in request.Hash)
+        {
+            try
+            {
+                FileInfo fi = new FileInfo(Path.Combine(_basePath, hash));
+                fi.Delete();
+                var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
+                if (file != null)
+                {
+                    _mareDbContext.Files.Remove(file);
+
+                    await _metricsClient.DecGaugeAsync(new GaugeRequest()
+                    { GaugeName = MetricsAPI.GaugeFilesTotalSize, Value = fi.Length }).ConfigureAwait(false);
+                    await _metricsClient.DecGaugeAsync(new GaugeRequest()
+                    { GaugeName = MetricsAPI.GaugeFilesTotal, Value = 1 }).ConfigureAwait(false);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Could not delete file for hash {hash}", hash);
+            }
+        }
+
+        await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
+        return new Empty();
+    }
+
+    public override Task<FileSizeResponse> GetFileSizes(FileSizeRequest request, ServerCallContext context)
+    {
+        FileSizeResponse response = new();
+        foreach (var hash in request.Hash)
+        {
+            FileInfo fi = new(Path.Combine(_basePath, hash));
+            if (fi.Exists)
+            {
+                response.HashToFileSize.Add(hash, fi.Length);
+            }
+            else
+            {
+                response.HashToFileSize.Add(hash, 0);
+            }
+        }
+
+        return Task.FromResult(response);
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs

@@ -1,6 +1,5 @@
 using MareSynchronosShared.Authentication;
 using MareSynchronosShared.Data;
-using MareSynchronosShared.Models;
 using MareSynchronosShared.Protos;
 using Microsoft.AspNetCore.Authentication;
 using Microsoft.AspNetCore.Authorization;
@@ -28,6 +27,19 @@ public class Startup
         {
             c.Address = new Uri(Configuration.GetValue<string>("ServiceAddress"));
         });
+        services.AddGrpcClient<MetricsService.MetricsServiceClient>(c =>
+        {
+            c.Address = new Uri(Configuration.GetValue<string>("ServiceAddress"));
+        });
+
+        services.AddDbContextPool<MareDbContext>(options =>
+        {
+            options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder =>
+            {
+                builder.MigrationsHistoryTable("_efmigrationshistory", "public");
+            }).UseSnakeCaseNamingConvention();
+            options.EnableThreadSafetyChecks(false);
+        }, Configuration.GetValue("DbContextPoolSize", 1024));
 
         services.AddAuthentication(options =>
             {
@@ -35,6 +47,11 @@ public class Startup
             })
             .AddScheme<AuthenticationSchemeOptions, SecretKeyGrpcAuthenticationHandler>(SecretKeyGrpcAuthenticationHandler.AuthScheme, options => { });
         services.AddAuthorization(options => options.FallbackPolicy = new AuthorizationPolicyBuilder().RequireAuthenticatedUser().Build());
+
+        services.AddGrpc(o =>
+        {
+            o.MaxReceiveMessageSize = null;
+        });
     }
 
     public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
@@ -46,12 +63,17 @@ public class Startup
 
         app.UseAuthentication();
         app.UseAuthorization();
-        
+
         app.UseStaticFiles(new StaticFileOptions()
         {
             FileProvider = new PhysicalFileProvider(Configuration["CacheDirectory"]),
             RequestPath = "/cache",
             ServeUnknownFileTypes = true
         });
+
+        app.UseEndpoints(e =>
+        {
+            e.MapGrpcService<FileService>();
+        });
     }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json

@@ -12,6 +12,8 @@
       }
     }
   },
+  "CacheSizeHardLimitInGiB": -1,
+  "UnusedFileRetentionPeriodInDays": 7,
   "AllowedHosts": "*",
   "CacheDirectory": "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored
   "ServiceAddress": "http://localhost:5002"