rework static files server

MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj

@@ -18,6 +18,7 @@
 	</ItemGroup>
 
 	<ItemGroup>
+		<PackageReference Include="ByteSize" Version="2.1.1" />
 		<PackageReference Include="EFCore.NamingConventions" Version="7.0.0" />
 		<PackageReference Include="Grpc.AspNetCore" Version="2.50.0" />
 		<PackageReference Include="Meziantou.Analyzer" Version="1.0.756">

MareSynchronosServer/MareSynchronosStaticFilesServer/CachedFileProvider.cs

@@ -0,0 +1,78 @@
+using System.Collections.Concurrent;
+
+namespace MareSynchronosStaticFilesServer;
+
+public class CachedFileProvider
+{
+    private readonly ILogger<CachedFileProvider> _logger;
+    private readonly FileStatisticsService _fileStatisticsService;
+    private readonly Uri _remoteCacheSourceUri;
+    private readonly string _basePath;
+    private readonly ConcurrentDictionary<string, Task> _currentTransfers = new(StringComparer.Ordinal);
+    private bool IsMainServer => _remoteCacheSourceUri == null;
+
+    public CachedFileProvider(IConfiguration configuration, ILogger<CachedFileProvider> logger, FileStatisticsService fileStatisticsService)
+    {
+        _logger = logger;
+        _fileStatisticsService = fileStatisticsService;
+        var configurationSection = configuration.GetRequiredSection("MareSynchronos");
+        _remoteCacheSourceUri = configurationSection.GetValue<Uri>("RemoteCacheSourceUri", null);
+        _basePath = configurationSection["CacheDirectory"];
+    }
+
+    public async Task<FileStream?> GetFileStream(string hash, string auth)
+    {
+        var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
+        if (fi == null)
+        {
+            if (IsMainServer) return null;
+            if (!_currentTransfers.ContainsKey(hash))
+            {
+                _currentTransfers[hash] = Task.Run(async () =>
+                {
+                    // download file from remote
+                    var downloadUrl = new Uri(_remoteCacheSourceUri, hash);
+                    _logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl);
+                    using var client = new HttpClient();
+                    client.DefaultRequestHeaders.Add("Authorization", auth);
+                    var response = await client.GetAsync(downloadUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
+
+                    try
+                    {
+                        response.EnsureSuccessStatusCode();
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogWarning(ex, "Failed to download {url}", downloadUrl);
+                        return;
+                    }
+
+                    var fileName = FilePathUtil.GetFilePath(_basePath, hash);
+                    var fileStream = File.Create(fileName);
+                    await using (fileStream.ConfigureAwait(false))
+                    {
+                        var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024;
+                        var buffer = new byte[bufferSize];
+
+                        var bytesRead = 0;
+                        while ((bytesRead = await (await response.Content.ReadAsStreamAsync().ConfigureAwait(false)).ReadAsync(buffer).ConfigureAwait(false)) > 0)
+                        {
+                            await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead)).ConfigureAwait(false);
+                        }
+                    }
+                });
+            }
+
+            await _currentTransfers[hash].ConfigureAwait(false);
+            _currentTransfers.Remove(hash, out _);
+
+            fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
+
+            if (fi == null) return null;
+        }
+
+        _fileStatisticsService.LogFile(hash, fi.Length);
+
+        return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read);
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs

@@ -1,153 +0,0 @@
-using MareSynchronosShared.Data;
-using MareSynchronosShared.Metrics;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-
-namespace MareSynchronosStaticFilesServer;
-
-public class CleanupService : IHostedService, IDisposable
-{
-    private readonly MareMetrics _metrics;
-    private readonly ILogger<CleanupService> _logger;
-    private readonly IServiceProvider _services;
-    private readonly IConfiguration _configuration;
-    private Timer? _timer;
-
-    public CleanupService(MareMetrics metrics, ILogger<CleanupService> logger, IServiceProvider services, IConfiguration configuration)
-    {
-        _metrics = metrics;
-        _logger = logger;
-        _services = services;
-        _configuration = configuration.GetRequiredSection("MareSynchronos");
-    }
-
-    public Task StartAsync(CancellationToken cancellationToken)
-    {
-        _logger.LogInformation("Cleanup Service started");
-
-        _logger.LogInformation("Calculating initial files");
-
-        DirectoryInfo dir = new DirectoryInfo(_configuration["CacheDirectory"]);
-        var allFiles = dir.GetFiles();
-        _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFiles.Sum(f => f.Length));
-        _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFiles.Length);
-
-        _logger.LogInformation("Initial file calculation finished, starting periodic cleanup task");
-
-        _timer = new Timer(CleanUp, null, TimeSpan.FromSeconds(15), TimeSpan.FromMinutes(10));
-
-        return Task.CompletedTask;
-    }
-
-    private void CleanUp(object? state)
-    {
-        if (!int.TryParse(_configuration["UnusedFileRetentionPeriodInDays"], out var filesOlderThanDays))
-        {
-            filesOlderThanDays = 7;
-        }
-
-        using var scope = _services.CreateScope();
-        using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
-
-        _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", filesOlderThanDays);
-
-        try
-        {
-            var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays));
-
-            var allFiles = dbContext.Files.ToList();
-            var cachedir = _configuration["CacheDirectory"];
-            foreach (var file in allFiles.Where(f => f.Uploaded))
-            {
-                var fileName = Path.Combine(cachedir, file.Hash);
-                var fi = new FileInfo(fileName);
-                if (!fi.Exists)
-                {
-                    _logger.LogInformation("File does not exist anymore: {fileName}", fileName);
-                    dbContext.Files.Remove(file);
-                }
-                else if (fi.LastAccessTime < prevTime)
-                {
-                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi.Length);
-                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
-                    _logger.LogInformation("File outdated: {fileName}", fileName);
-                    dbContext.Files.Remove(file);
-                    fi.Delete();
-                }
-            }
-
-            var allFilesHashes = new HashSet<string>(allFiles.Select(a => a.Hash.ToUpperInvariant()));
-            DirectoryInfo dir = new DirectoryInfo(cachedir);
-            var allFilesInDir = dir.GetFiles();
-            foreach (var file in allFilesInDir)
-            {
-                if (!allFilesHashes.Contains(file.Name.ToUpperInvariant()))
-                {
-                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
-                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
-                    file.Delete();
-                    _logger.LogInformation("File not in DB, deleting: {fileName}", file.FullName);
-                }
-            }
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning(ex, "Error during file cleanup");
-        }
-
-        var cacheSizeLimitInGiB = _configuration.GetValue<double>("CacheSizeHardLimitInGiB", -1);
-
-        try
-        {
-            if (cacheSizeLimitInGiB > 0)
-            {
-                _logger.LogInformation("Cleaning up files beyond the cache size limit");
-                var allLocalFiles = Directory.EnumerateFiles(_configuration["CacheDirectory"])
-                    .Select(f => new FileInfo(f)).ToList().OrderBy(f => f.LastAccessTimeUtc).ToList();
-                var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
-                long cacheSizeLimitInBytes = (long)(cacheSizeLimitInGiB * 1024 * 1024 * 1024);
-                HashSet<string> removedHashes = new();
-                while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any())
-                {
-                    var oldestFile = allLocalFiles.First();
-                    removedHashes.Add(oldestFile.Name.ToLower());
-                    allLocalFiles.Remove(oldestFile);
-                    totalCacheSizeInBytes -= oldestFile.Length;
-                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, oldestFile.Length);
-                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
-                    oldestFile.Delete();
-                }
-
-                dbContext.Files.RemoveRange(dbContext.Files.Where(f => removedHashes.Contains(f.Hash.ToLower())));
-            }
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning(ex, "Error during cache size limit cleanup");
-        }
-
-        _logger.LogInformation($"Cleanup complete");
-
-        dbContext.SaveChanges();
-    }
-
-    public Task StopAsync(CancellationToken cancellationToken)
-    {
-        _timer?.Change(Timeout.Infinite, 0);
-
-        return Task.CompletedTask;
-    }
-
-    public void Dispose()
-    {
-        _timer?.Dispose();
-    }
-}

MareSynchronosServer/MareSynchronosStaticFilesServer/FileCleanupService.cs

@@ -0,0 +1,177 @@
+using ByteSizeLib;
+using MareSynchronosShared.Data;
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Models;
+using System.Globalization;
+
+namespace MareSynchronosStaticFilesServer;
+
+public class FileCleanupService : IHostedService
+{
+    private readonly MareMetrics _metrics;
+    private readonly ILogger<FileCleanupService> _logger;
+    private readonly IServiceProvider _services;
+    private readonly IConfiguration _configuration;
+    private readonly bool _isMainServer;
+    private readonly string _cacheDir;
+    private CancellationTokenSource _cleanupCts;
+
+    public FileCleanupService(MareMetrics metrics, ILogger<FileCleanupService> logger, IServiceProvider services, IConfiguration configuration)
+    {
+        _metrics = metrics;
+        _logger = logger;
+        _services = services;
+        _configuration = configuration.GetRequiredSection("MareSynchronos");
+        _isMainServer = !string.IsNullOrEmpty(_configuration.GetValue("RemoteCacheSource", string.Empty));
+        _cacheDir = _configuration.GetValue<string>("CacheDirectory");
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Cleanup Service started");
+
+        _logger.LogInformation("Calculating initial files");
+
+        _cleanupCts = new();
+
+        DirectoryInfo dir = new(_cacheDir);
+        var allFiles = dir.GetFiles();
+        _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFiles.Sum(f => f.Length));
+        _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFiles.Length);
+
+
+        _ = CleanUpTask(_cleanupCts.Token);
+
+        return Task.CompletedTask;
+    }
+
+    public async Task CleanUpTask(CancellationToken ct)
+    {
+        _logger.LogInformation("Starting periodic cleanup task");
+
+        while (!ct.IsCancellationRequested)
+        {
+            using var scope = _services.CreateScope();
+            using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
+
+            CleanUpOutdatedFiles(dbContext, ct);
+
+            CleanUpFilesBeyondSizeLimit(dbContext, ct);
+
+            if (_isMainServer)
+            {
+                await dbContext.SaveChangesAsync(ct).ConfigureAwait(false);
+            }
+
+            _logger.LogInformation("File Cleanup Complete, next run at {date}", DateTime.Now.Add(TimeSpan.FromMinutes(10)));
+            await Task.Delay(TimeSpan.FromMinutes(10), ct).ConfigureAwait(false);
+        }
+    }
+
+    private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct)
+    {
+        var cacheSizeLimitInGiB = _configuration.GetValue<double>("CacheSizeHardLimitInGiB", -1);
+
+        if (cacheSizeLimitInGiB <= 0)
+        {
+            return;
+        }
+
+        try
+        {
+            _logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", cacheSizeLimitInGiB);
+            var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories)
+                .Select(f => new FileInfo(f)).ToList()
+                .OrderBy(f => f.LastAccessTimeUtc).ToList();
+            var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
+            long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(cacheSizeLimitInGiB).Bytes;
+            while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested)
+            {
+                var oldestFile = allLocalFiles[0];
+                allLocalFiles.Remove(oldestFile);
+                totalCacheSizeInBytes -= oldestFile.Length;
+                _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, oldestFile.Length);
+                _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
+                _logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes);
+                oldestFile.Delete();
+                if (_isMainServer)
+                {
+                    FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() };
+                    dbContext.Entry(f).State = Microsoft.EntityFrameworkCore.EntityState.Deleted;
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error during cache size limit cleanup");
+        }
+    }
+
+    private void CleanUpOutdatedFiles(MareDbContext dbContext, CancellationToken ct)
+    {
+        try
+        {
+            if (!int.TryParse(_configuration["UnusedFileRetentionPeriodInDays"], CultureInfo.InvariantCulture, out int filesOlderThanDays))
+            {
+                filesOlderThanDays = 7;
+            }
+
+            _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", filesOlderThanDays);
+
+            // clean up files in DB but not on disk or last access is expired
+            var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays));
+            var allFiles = dbContext.Files.ToList();
+            foreach (var fileCache in allFiles.Where(f => f.Uploaded))
+            {
+                var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash);
+                if (file == null && _isMainServer)
+                {
+                    _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash);
+                    dbContext.Files.Remove(fileCache);
+                }
+                else if (file != null && file.LastAccessTime < prevTime)
+                {
+                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
+                    _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
+                    _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
+                    file.Delete();
+                    if (_isMainServer)
+                        dbContext.Files.Remove(fileCache);
+                }
+
+                ct.ThrowIfCancellationRequested();
+            }
+
+            // clean up files that are on disk but not in DB for some reason
+            if (_isMainServer)
+            {
+                var allFilesHashes = new HashSet<string>(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal);
+                DirectoryInfo dir = new(_cacheDir);
+                var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories);
+                foreach (var file in allFilesInDir)
+                {
+                    if (!allFilesHashes.Contains(file.Name.ToUpperInvariant()))
+                    {
+                        _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
+                        _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
+                        file.Delete();
+                        _logger.LogInformation("File not in DB, deleting: {fileName}", file.FullName);
+                    }
+
+                    ct.ThrowIfCancellationRequested();
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error during file cleanup of old files");
+        }
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        _cleanupCts.Cancel();
+
+        return Task.CompletedTask;
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/FilePathUtil.cs

@@ -0,0 +1,27 @@
+namespace MareSynchronosStaticFilesServer;
+
+public static class FilePathUtil
+{
+    public static FileInfo? GetFileInfoForHash(string basePath, string hash)
+    {
+        FileInfo fi = new(Path.Combine(basePath, hash[0].ToString(), hash));
+        if (!fi.Exists)
+        {
+            fi = new FileInfo(Path.Combine(basePath, hash));
+            if (!fi.Exists)
+            {
+                return null;
+            }
+        }
+
+        return fi;
+    }
+
+    public static string GetFilePath(string basePath, string hash)
+    {
+        var dirPath = Path.Combine(basePath, hash[0].ToString());
+        var path = Path.Combine(dirPath, hash);
+        if (!Directory.Exists(dirPath)) Directory.CreateDirectory(dirPath);
+        return path;
+    }
+}

MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs

@@ -1,11 +1,5 @@
 using MareSynchronosShared.Metrics;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
-using System;
 using System.Collections.Concurrent;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
 
 namespace MareSynchronosStaticFilesServer;
 

MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs

@@ -1,8 +1,4 @@
 using Microsoft.AspNetCore.Mvc;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Logging;
-using System.IO;
-using System.Linq;
 using System.Security.Claims;
 
 namespace MareSynchronosStaticFilesServer;
@@ -11,29 +7,23 @@ namespace MareSynchronosStaticFilesServer;
 public class FilesController : Controller
 {
     private readonly ILogger<FilesController> _logger;
-    private readonly IConfiguration _configuration;
+    private readonly CachedFileProvider _cachedFileProvider;
-    private readonly FileStatisticsService _fileStatisticsService;
 
-    public FilesController(ILogger<FilesController> logger, IConfiguration configuration, FileStatisticsService fileStatisticsService)
+    public FilesController(ILogger<FilesController> logger, CachedFileProvider cachedFileProvider)
     {
         _logger = logger;
-        _configuration = configuration;
+        _cachedFileProvider = cachedFileProvider;
-        _fileStatisticsService = fileStatisticsService;
     }
 
     [HttpGet("{fileId}")]
-    public IActionResult GetFile(string fileId)
+    public async Task<IActionResult> GetFile(string fileId)
     {
         var authedUser = HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, ClaimTypes.NameIdentifier, System.StringComparison.Ordinal))?.Value ?? "Unknown";
         _logger.LogInformation($"GetFile:{authedUser}:{fileId}");
 
-        FileInfo fi = new(Path.Combine(_configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"], fileId));
+        var fs = await _cachedFileProvider.GetFileStream(fileId, Request.Headers["Authorization"]);
-        if (!fi.Exists) return NotFound();
+        if (fs == null) return NotFound();
 
-        _fileStatisticsService.LogFile(fileId, fi.Length);
+        return File(fs, "application/octet-stream");
-
-        var fileStream = new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read);
-
-        return File(fileStream, "application/octet-stream");
     }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/GrpcFileService.cs

@@ -3,23 +3,17 @@ using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
 using MareSynchronosShared.Protos;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Logging;
-using System;
-using System.IO;
-using System.Linq;
-using System.Threading.Tasks;
 
 namespace MareSynchronosStaticFilesServer;
 
-public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBase
+public class GrpcFileService : FileService.FileServiceBase
 {
     private readonly string _basePath;
     private readonly MareDbContext _mareDbContext;
-    private readonly ILogger<FileService> _logger;
+    private readonly ILogger<GrpcFileService> _logger;
     private readonly MareMetrics _metricsClient;
 
-    public FileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger<FileService> logger, MareMetrics metricsClient)
+    public GrpcFileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger<GrpcFileService> logger, MareMetrics metricsClient)
     {
         _basePath = configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"];
         _mareDbContext = mareDbContext;
@@ -29,21 +23,21 @@ public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBa
 
     public override async Task<Empty> UploadFile(IAsyncStreamReader<UploadFileRequest> requestStream, ServerCallContext context)
     {
-        await requestStream.MoveNext();
+        _ = await requestStream.MoveNext().ConfigureAwait(false);
         var uploadMsg = requestStream.Current;
-        var filePath = Path.Combine(_basePath, uploadMsg.Hash);
+        var filePath = FilePathUtil.GetFilePath(_basePath, uploadMsg.Hash);
         using var fileWriter = File.OpenWrite(filePath);
-        var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == uploadMsg.Hash && f.UploaderUID == uploadMsg.Uploader);
+        var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == uploadMsg.Hash && f.UploaderUID == uploadMsg.Uploader).ConfigureAwait(false);
         if (file != null)
         {
-            await fileWriter.WriteAsync(uploadMsg.FileData.ToArray());
+            await fileWriter.WriteAsync(uploadMsg.FileData.ToArray()).ConfigureAwait(false);
 
-            while (await requestStream.MoveNext())
+            while (await requestStream.MoveNext().ConfigureAwait(false))
             {
-                await fileWriter.WriteAsync(requestStream.Current.FileData.ToArray());
+                await fileWriter.WriteAsync(requestStream.Current.FileData.ToArray()).ConfigureAwait(false);
             }
 
-            await fileWriter.FlushAsync();
+            await fileWriter.FlushAsync().ConfigureAwait(false);
             fileWriter.Close();
 
             var fileSize = new FileInfo(filePath).Length;
@@ -65,15 +59,15 @@ public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBa
         {
             try
             {
-                FileInfo fi = new FileInfo(Path.Combine(_basePath, hash));
+                var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
-                fi.Delete();
+                fi?.Delete();
-                var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
+                var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash).ConfigureAwait(false);
                 if (file != null)
                 {
                     _mareDbContext.Files.Remove(file);
 
-                    _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, 1);
+                    _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1);
-                    _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi.Length);
+                    _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0);
                 }
             }
             catch (Exception ex)
@@ -89,17 +83,10 @@ public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBa
     public override Task<FileSizeResponse> GetFileSizes(FileSizeRequest request, ServerCallContext context)
     {
         FileSizeResponse response = new();
-        foreach (var hash in request.Hash.Distinct())
+        foreach (var hash in request.Hash.Distinct(StringComparer.Ordinal))
         {
-            FileInfo fi = new(Path.Combine(_basePath, hash));
+            FileInfo? fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
-            if (fi.Exists)
+            response.HashToFileSize.Add(hash, fi?.Length ?? 0);
-            {
-                response.HashToFileSize.Add(hash, fi.Length);
-            }
-            else
-            {
-                response.HashToFileSize.Add(hash, 0);
-            }
         }
 
         return Task.FromResult(response);

MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj

@@ -2,6 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
 
   <ItemGroup>
@@ -23,7 +24,6 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="7.0.0" />
     <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="7.0.0" />
     <PackageReference Include="prometheus-net.AspNetCore" Version="7.0.0" />
   </ItemGroup>

MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs

@@ -1,7 +1,3 @@
-using Microsoft.AspNetCore.Hosting;
-using Microsoft.Extensions.Hosting;
-using System;
-
 namespace MareSynchronosStaticFilesServer;
 
 public class Program

MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs

@@ -4,15 +4,8 @@ using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
 using Microsoft.AspNetCore.Authentication;
 using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Builder;
-using Microsoft.AspNetCore.Hosting;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
 using Prometheus;
-using System;
-using System.Collections.Generic;
 
 namespace MareSynchronosStaticFilesServer;
 
@@ -65,10 +58,11 @@ public class Startup
             MetricsAPI.GaugeFilesUniquePastHour,
             MetricsAPI.GaugeFilesUniquePastHourSize
         }));
+        services.AddSingleton<CachedFileProvider>();
         services.AddSingleton<FileStatisticsService>();
 
         services.AddHostedService(m => m.GetService<FileStatisticsService>());
-        services.AddHostedService<CleanupService>();
+        services.AddHostedService<FileCleanupService>();
 
         services.AddSingleton<SecretKeyAuthenticatorService>();
         services.AddDbContextPool<MareDbContext>(options =>
@@ -99,8 +93,8 @@ public class Startup
 
         app.UseRouting();
 
-        var metricServer = new KestrelMetricServer(4981);
+        //var metricServer = new KestrelMetricServer(4981);
-        metricServer.Start();
+        //metricServer.Start();
 
         app.UseHttpMetrics();
 
@@ -109,7 +103,7 @@ public class Startup
 
         app.UseEndpoints(e =>
         {
-            e.MapGrpcService<FileService>();
+            e.MapGrpcService<GrpcFileService>();
             e.MapControllers();
         });
     }

MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json

@@ -24,7 +24,7 @@
     "UnusedFileRetentionPeriodInDays": 7,
     "CacheDirectory": "G:\\ServerTest",
     "ServiceAddress": "http://localhost:5002",
-    "IsSecondaryInstance": "false"
+    "RemoteCacheSourceUri": ""
   },
   "AllowedHosts": "*"
 }