diff --git a/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj b/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj index ff66ad6..9b6fcee 100644 --- a/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj +++ b/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj @@ -18,6 +18,7 @@ + diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/CachedFileProvider.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/CachedFileProvider.cs new file mode 100644 index 0000000..a99b06b --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/CachedFileProvider.cs @@ -0,0 +1,78 @@ +using System.Collections.Concurrent; + +namespace MareSynchronosStaticFilesServer; + +public class CachedFileProvider +{ + private readonly ILogger _logger; + private readonly FileStatisticsService _fileStatisticsService; + private readonly Uri _remoteCacheSourceUri; + private readonly string _basePath; + private readonly ConcurrentDictionary _currentTransfers = new(StringComparer.Ordinal); + private bool IsMainServer => _remoteCacheSourceUri == null; + + public CachedFileProvider(IConfiguration configuration, ILogger logger, FileStatisticsService fileStatisticsService) + { + _logger = logger; + _fileStatisticsService = fileStatisticsService; + var configurationSection = configuration.GetRequiredSection("MareSynchronos"); + _remoteCacheSourceUri = configurationSection.GetValue("RemoteCacheSourceUri", null); + _basePath = configurationSection["CacheDirectory"]; + } + + public async Task GetFileStream(string hash, string auth) + { + var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash); + if (fi == null) + { + if (IsMainServer) return null; + if (!_currentTransfers.ContainsKey(hash)) + { + _currentTransfers[hash] = Task.Run(async () => + { + // download file from remote + var downloadUrl = new Uri(_remoteCacheSourceUri, hash); + _logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl); + using var client = new HttpClient(); + client.DefaultRequestHeaders.Add("Authorization", auth); + var response = await client.GetAsync(downloadUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); + + try + { + response.EnsureSuccessStatusCode(); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to download {url}", downloadUrl); + return; + } + + var fileName = FilePathUtil.GetFilePath(_basePath, hash); + var fileStream = File.Create(fileName); + await using (fileStream.ConfigureAwait(false)) + { + var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024; + var buffer = new byte[bufferSize]; + + var bytesRead = 0; + while ((bytesRead = await (await response.Content.ReadAsStreamAsync().ConfigureAwait(false)).ReadAsync(buffer).ConfigureAwait(false)) > 0) + { + await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead)).ConfigureAwait(false); + } + } + }); + } + + await _currentTransfers[hash].ConfigureAwait(false); + _currentTransfers.Remove(hash, out _); + + fi = FilePathUtil.GetFileInfoForHash(_basePath, hash); + + if (fi == null) return null; + } + + _fileStatisticsService.LogFile(hash, fi.Length); + + return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read); + } +} \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs deleted file mode 100644 index ea7089b..0000000 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs +++ /dev/null @@ -1,153 +0,0 @@ -using MareSynchronosShared.Data; -using MareSynchronosShared.Metrics; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -namespace MareSynchronosStaticFilesServer; - -public class CleanupService : IHostedService, IDisposable -{ - private readonly MareMetrics _metrics; - private readonly ILogger _logger; - private readonly IServiceProvider _services; - private readonly IConfiguration _configuration; - private Timer? _timer; - - public CleanupService(MareMetrics metrics, ILogger logger, IServiceProvider services, IConfiguration configuration) - { - _metrics = metrics; - _logger = logger; - _services = services; - _configuration = configuration.GetRequiredSection("MareSynchronos"); - } - - public Task StartAsync(CancellationToken cancellationToken) - { - _logger.LogInformation("Cleanup Service started"); - - _logger.LogInformation("Calculating initial files"); - - DirectoryInfo dir = new DirectoryInfo(_configuration["CacheDirectory"]); - var allFiles = dir.GetFiles(); - _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFiles.Sum(f => f.Length)); - _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFiles.Length); - - _logger.LogInformation("Initial file calculation finished, starting periodic cleanup task"); - - _timer = new Timer(CleanUp, null, TimeSpan.FromSeconds(15), TimeSpan.FromMinutes(10)); - - return Task.CompletedTask; - } - - private void CleanUp(object? state) - { - if (!int.TryParse(_configuration["UnusedFileRetentionPeriodInDays"], out var filesOlderThanDays)) - { - filesOlderThanDays = 7; - } - - using var scope = _services.CreateScope(); - using var dbContext = scope.ServiceProvider.GetService()!; - - _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", filesOlderThanDays); - - try - { - var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays)); - - var allFiles = dbContext.Files.ToList(); - var cachedir = _configuration["CacheDirectory"]; - foreach (var file in allFiles.Where(f => f.Uploaded)) - { - var fileName = Path.Combine(cachedir, file.Hash); - var fi = new FileInfo(fileName); - if (!fi.Exists) - { - _logger.LogInformation("File does not exist anymore: {fileName}", fileName); - dbContext.Files.Remove(file); - } - else if (fi.LastAccessTime < prevTime) - { - _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi.Length); - _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); - _logger.LogInformation("File outdated: {fileName}", fileName); - dbContext.Files.Remove(file); - fi.Delete(); - } - } - - var allFilesHashes = new HashSet(allFiles.Select(a => a.Hash.ToUpperInvariant())); - DirectoryInfo dir = new DirectoryInfo(cachedir); - var allFilesInDir = dir.GetFiles(); - foreach (var file in allFilesInDir) - { - if (!allFilesHashes.Contains(file.Name.ToUpperInvariant())) - { - _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); - _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); - file.Delete(); - _logger.LogInformation("File not in DB, deleting: {fileName}", file.FullName); - } - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Error during file cleanup"); - } - - var cacheSizeLimitInGiB = _configuration.GetValue("CacheSizeHardLimitInGiB", -1); - - try - { - if (cacheSizeLimitInGiB > 0) - { - _logger.LogInformation("Cleaning up files beyond the cache size limit"); - var allLocalFiles = Directory.EnumerateFiles(_configuration["CacheDirectory"]) - .Select(f => new FileInfo(f)).ToList().OrderBy(f => f.LastAccessTimeUtc).ToList(); - var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length); - long cacheSizeLimitInBytes = (long)(cacheSizeLimitInGiB * 1024 * 1024 * 1024); - HashSet removedHashes = new(); - while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any()) - { - var oldestFile = allLocalFiles.First(); - removedHashes.Add(oldestFile.Name.ToLower()); - allLocalFiles.Remove(oldestFile); - totalCacheSizeInBytes -= oldestFile.Length; - _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, oldestFile.Length); - _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); - oldestFile.Delete(); - } - - dbContext.Files.RemoveRange(dbContext.Files.Where(f => removedHashes.Contains(f.Hash.ToLower()))); - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Error during cache size limit cleanup"); - } - - _logger.LogInformation($"Cleanup complete"); - - dbContext.SaveChanges(); - } - - public Task StopAsync(CancellationToken cancellationToken) - { - _timer?.Change(Timeout.Infinite, 0); - - return Task.CompletedTask; - } - - public void Dispose() - { - _timer?.Dispose(); - } -} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileCleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileCleanupService.cs new file mode 100644 index 0000000..7d11cd0 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileCleanupService.cs @@ -0,0 +1,177 @@ +using ByteSizeLib; +using MareSynchronosShared.Data; +using MareSynchronosShared.Metrics; +using MareSynchronosShared.Models; +using System.Globalization; + +namespace MareSynchronosStaticFilesServer; + +public class FileCleanupService : IHostedService +{ + private readonly MareMetrics _metrics; + private readonly ILogger _logger; + private readonly IServiceProvider _services; + private readonly IConfiguration _configuration; + private readonly bool _isMainServer; + private readonly string _cacheDir; + private CancellationTokenSource _cleanupCts; + + public FileCleanupService(MareMetrics metrics, ILogger logger, IServiceProvider services, IConfiguration configuration) + { + _metrics = metrics; + _logger = logger; + _services = services; + _configuration = configuration.GetRequiredSection("MareSynchronos"); + _isMainServer = !string.IsNullOrEmpty(_configuration.GetValue("RemoteCacheSource", string.Empty)); + _cacheDir = _configuration.GetValue("CacheDirectory"); + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Cleanup Service started"); + + _logger.LogInformation("Calculating initial files"); + + _cleanupCts = new(); + + DirectoryInfo dir = new(_cacheDir); + var allFiles = dir.GetFiles(); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFiles.Sum(f => f.Length)); + _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFiles.Length); + + + _ = CleanUpTask(_cleanupCts.Token); + + return Task.CompletedTask; + } + + public async Task CleanUpTask(CancellationToken ct) + { + _logger.LogInformation("Starting periodic cleanup task"); + + while (!ct.IsCancellationRequested) + { + using var scope = _services.CreateScope(); + using var dbContext = scope.ServiceProvider.GetService()!; + + CleanUpOutdatedFiles(dbContext, ct); + + CleanUpFilesBeyondSizeLimit(dbContext, ct); + + if (_isMainServer) + { + await dbContext.SaveChangesAsync(ct).ConfigureAwait(false); + } + + _logger.LogInformation("File Cleanup Complete, next run at {date}", DateTime.Now.Add(TimeSpan.FromMinutes(10))); + await Task.Delay(TimeSpan.FromMinutes(10), ct).ConfigureAwait(false); + } + } + + private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct) + { + var cacheSizeLimitInGiB = _configuration.GetValue("CacheSizeHardLimitInGiB", -1); + + if (cacheSizeLimitInGiB <= 0) + { + return; + } + + try + { + _logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", cacheSizeLimitInGiB); + var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories) + .Select(f => new FileInfo(f)).ToList() + .OrderBy(f => f.LastAccessTimeUtc).ToList(); + var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length); + long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(cacheSizeLimitInGiB).Bytes; + while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested) + { + var oldestFile = allLocalFiles[0]; + allLocalFiles.Remove(oldestFile); + totalCacheSizeInBytes -= oldestFile.Length; + _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, oldestFile.Length); + _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); + _logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes); + oldestFile.Delete(); + if (_isMainServer) + { + FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() }; + dbContext.Entry(f).State = Microsoft.EntityFrameworkCore.EntityState.Deleted; + } + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error during cache size limit cleanup"); + } + } + + private void CleanUpOutdatedFiles(MareDbContext dbContext, CancellationToken ct) + { + try + { + if (!int.TryParse(_configuration["UnusedFileRetentionPeriodInDays"], CultureInfo.InvariantCulture, out int filesOlderThanDays)) + { + filesOlderThanDays = 7; + } + + _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", filesOlderThanDays); + + // clean up files in DB but not on disk or last access is expired + var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays)); + var allFiles = dbContext.Files.ToList(); + foreach (var fileCache in allFiles.Where(f => f.Uploaded)) + { + var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash); + if (file == null && _isMainServer) + { + _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash); + dbContext.Files.Remove(fileCache); + } + else if (file != null && file.LastAccessTime < prevTime) + { + _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); + _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); + _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); + file.Delete(); + if (_isMainServer) + dbContext.Files.Remove(fileCache); + } + + ct.ThrowIfCancellationRequested(); + } + + // clean up files that are on disk but not in DB for some reason + if (_isMainServer) + { + var allFilesHashes = new HashSet(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal); + DirectoryInfo dir = new(_cacheDir); + var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories); + foreach (var file in allFilesInDir) + { + if (!allFilesHashes.Contains(file.Name.ToUpperInvariant())) + { + _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); + _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); + file.Delete(); + _logger.LogInformation("File not in DB, deleting: {fileName}", file.FullName); + } + + ct.ThrowIfCancellationRequested(); + } + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error during file cleanup of old files"); + } + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _cleanupCts.Cancel(); + + return Task.CompletedTask; + } +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/FilePathUtil.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/FilePathUtil.cs new file mode 100644 index 0000000..a78943e --- /dev/null +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/FilePathUtil.cs @@ -0,0 +1,27 @@ +namespace MareSynchronosStaticFilesServer; + +public static class FilePathUtil +{ + public static FileInfo? GetFileInfoForHash(string basePath, string hash) + { + FileInfo fi = new(Path.Combine(basePath, hash[0].ToString(), hash)); + if (!fi.Exists) + { + fi = new FileInfo(Path.Combine(basePath, hash)); + if (!fi.Exists) + { + return null; + } + } + + return fi; + } + + public static string GetFilePath(string basePath, string hash) + { + var dirPath = Path.Combine(basePath, hash[0].ToString()); + var path = Path.Combine(dirPath, hash); + if (!Directory.Exists(dirPath)) Directory.CreateDirectory(dirPath); + return path; + } +} diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs index d493a55..2fcba3d 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs @@ -1,11 +1,5 @@ using MareSynchronosShared.Metrics; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using System; using System.Collections.Concurrent; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; namespace MareSynchronosStaticFilesServer; diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs index b3f63af..9bf2fbb 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs @@ -1,8 +1,4 @@ using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using System.IO; -using System.Linq; using System.Security.Claims; namespace MareSynchronosStaticFilesServer; @@ -11,29 +7,23 @@ namespace MareSynchronosStaticFilesServer; public class FilesController : Controller { private readonly ILogger _logger; - private readonly IConfiguration _configuration; - private readonly FileStatisticsService _fileStatisticsService; + private readonly CachedFileProvider _cachedFileProvider; - public FilesController(ILogger logger, IConfiguration configuration, FileStatisticsService fileStatisticsService) + public FilesController(ILogger logger, CachedFileProvider cachedFileProvider) { _logger = logger; - _configuration = configuration; - _fileStatisticsService = fileStatisticsService; + _cachedFileProvider = cachedFileProvider; } [HttpGet("{fileId}")] - public IActionResult GetFile(string fileId) + public async Task GetFile(string fileId) { var authedUser = HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, ClaimTypes.NameIdentifier, System.StringComparison.Ordinal))?.Value ?? "Unknown"; _logger.LogInformation($"GetFile:{authedUser}:{fileId}"); - FileInfo fi = new(Path.Combine(_configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"], fileId)); - if (!fi.Exists) return NotFound(); + var fs = await _cachedFileProvider.GetFileStream(fileId, Request.Headers["Authorization"]); + if (fs == null) return NotFound(); - _fileStatisticsService.LogFile(fileId, fi.Length); - - var fileStream = new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read); - - return File(fileStream, "application/octet-stream"); + return File(fs, "application/octet-stream"); } } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/GrpcFileService.cs similarity index 69% rename from MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs rename to MareSynchronosServer/MareSynchronosStaticFilesServer/GrpcFileService.cs index 1a234c8..9c5babd 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/GrpcFileService.cs @@ -3,23 +3,17 @@ using MareSynchronosShared.Data; using MareSynchronosShared.Metrics; using MareSynchronosShared.Protos; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using System; -using System.IO; -using System.Linq; -using System.Threading.Tasks; namespace MareSynchronosStaticFilesServer; -public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBase +public class GrpcFileService : FileService.FileServiceBase { private readonly string _basePath; private readonly MareDbContext _mareDbContext; - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly MareMetrics _metricsClient; - public FileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger logger, MareMetrics metricsClient) + public GrpcFileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger logger, MareMetrics metricsClient) { _basePath = configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"]; _mareDbContext = mareDbContext; @@ -29,21 +23,21 @@ public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBa public override async Task UploadFile(IAsyncStreamReader requestStream, ServerCallContext context) { - await requestStream.MoveNext(); + _ = await requestStream.MoveNext().ConfigureAwait(false); var uploadMsg = requestStream.Current; - var filePath = Path.Combine(_basePath, uploadMsg.Hash); + var filePath = FilePathUtil.GetFilePath(_basePath, uploadMsg.Hash); using var fileWriter = File.OpenWrite(filePath); - var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == uploadMsg.Hash && f.UploaderUID == uploadMsg.Uploader); + var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == uploadMsg.Hash && f.UploaderUID == uploadMsg.Uploader).ConfigureAwait(false); if (file != null) { - await fileWriter.WriteAsync(uploadMsg.FileData.ToArray()); + await fileWriter.WriteAsync(uploadMsg.FileData.ToArray()).ConfigureAwait(false); - while (await requestStream.MoveNext()) + while (await requestStream.MoveNext().ConfigureAwait(false)) { - await fileWriter.WriteAsync(requestStream.Current.FileData.ToArray()); + await fileWriter.WriteAsync(requestStream.Current.FileData.ToArray()).ConfigureAwait(false); } - await fileWriter.FlushAsync(); + await fileWriter.FlushAsync().ConfigureAwait(false); fileWriter.Close(); var fileSize = new FileInfo(filePath).Length; @@ -65,15 +59,15 @@ public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBa { try { - FileInfo fi = new FileInfo(Path.Combine(_basePath, hash)); - fi.Delete(); - var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash); + var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash); + fi?.Delete(); + var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash).ConfigureAwait(false); if (file != null) { _mareDbContext.Files.Remove(file); - _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, 1); - _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi.Length); + _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1); + _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0); } } catch (Exception ex) @@ -89,17 +83,10 @@ public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBa public override Task GetFileSizes(FileSizeRequest request, ServerCallContext context) { FileSizeResponse response = new(); - foreach (var hash in request.Hash.Distinct()) + foreach (var hash in request.Hash.Distinct(StringComparer.Ordinal)) { - FileInfo fi = new(Path.Combine(_basePath, hash)); - if (fi.Exists) - { - response.HashToFileSize.Add(hash, fi.Length); - } - else - { - response.HashToFileSize.Add(hash, 0); - } + FileInfo? fi = FilePathUtil.GetFileInfoForHash(_basePath, hash); + response.HashToFileSize.Add(hash, fi?.Length ?? 0); } return Task.FromResult(response); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj b/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj index 41aecaf..05180b9 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj @@ -2,6 +2,7 @@ net7.0 + enable @@ -23,7 +24,6 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs index 8b791bc..64e1003 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs @@ -1,7 +1,3 @@ -using Microsoft.AspNetCore.Hosting; -using Microsoft.Extensions.Hosting; -using System; - namespace MareSynchronosStaticFilesServer; public class Program diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs index 3cbdfa1..b9648c0 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs @@ -4,15 +4,8 @@ using MareSynchronosShared.Data; using MareSynchronosShared.Metrics; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; using Prometheus; -using System; -using System.Collections.Generic; namespace MareSynchronosStaticFilesServer; @@ -65,10 +58,11 @@ public class Startup MetricsAPI.GaugeFilesUniquePastHour, MetricsAPI.GaugeFilesUniquePastHourSize })); + services.AddSingleton(); services.AddSingleton(); services.AddHostedService(m => m.GetService()); - services.AddHostedService(); + services.AddHostedService(); services.AddSingleton(); services.AddDbContextPool(options => @@ -99,8 +93,8 @@ public class Startup app.UseRouting(); - var metricServer = new KestrelMetricServer(4981); - metricServer.Start(); + //var metricServer = new KestrelMetricServer(4981); + //metricServer.Start(); app.UseHttpMetrics(); @@ -109,7 +103,7 @@ public class Startup app.UseEndpoints(e => { - e.MapGrpcService(); + e.MapGrpcService(); e.MapControllers(); }); } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json b/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json index 75889bc..910547f 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json @@ -24,7 +24,7 @@ "UnusedFileRetentionPeriodInDays": 7, "CacheDirectory": "G:\\ServerTest", "ServiceAddress": "http://localhost:5002", - "IsSecondaryInstance": "false" + "RemoteCacheSourceUri": "" }, "AllowedHosts": "*" }