diff --git a/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj b/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj
index ff66ad6..9b6fcee 100644
--- a/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj
+++ b/MareSynchronosServer/MareSynchronosShared/MareSynchronosShared.csproj
@@ -18,6 +18,7 @@
+
diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/CachedFileProvider.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/CachedFileProvider.cs
new file mode 100644
index 0000000..a99b06b
--- /dev/null
+++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/CachedFileProvider.cs
@@ -0,0 +1,78 @@
+using System.Collections.Concurrent;
+
+namespace MareSynchronosStaticFilesServer;
+
+public class CachedFileProvider
+{
+ private readonly ILogger _logger;
+ private readonly FileStatisticsService _fileStatisticsService;
+ private readonly Uri _remoteCacheSourceUri;
+ private readonly string _basePath;
+ private readonly ConcurrentDictionary _currentTransfers = new(StringComparer.Ordinal);
+ private bool IsMainServer => _remoteCacheSourceUri == null;
+
+ public CachedFileProvider(IConfiguration configuration, ILogger logger, FileStatisticsService fileStatisticsService)
+ {
+ _logger = logger;
+ _fileStatisticsService = fileStatisticsService;
+ var configurationSection = configuration.GetRequiredSection("MareSynchronos");
+ _remoteCacheSourceUri = configurationSection.GetValue("RemoteCacheSourceUri", null);
+ _basePath = configurationSection["CacheDirectory"];
+ }
+
+ public async Task GetFileStream(string hash, string auth)
+ {
+ var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
+ if (fi == null)
+ {
+ if (IsMainServer) return null;
+ if (!_currentTransfers.ContainsKey(hash))
+ {
+ _currentTransfers[hash] = Task.Run(async () =>
+ {
+ // download file from remote
+ var downloadUrl = new Uri(_remoteCacheSourceUri, hash);
+ _logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl);
+ using var client = new HttpClient();
+ client.DefaultRequestHeaders.Add("Authorization", auth);
+ var response = await client.GetAsync(downloadUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
+
+ try
+ {
+ response.EnsureSuccessStatusCode();
+ }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex, "Failed to download {url}", downloadUrl);
+ return;
+ }
+
+ var fileName = FilePathUtil.GetFilePath(_basePath, hash);
+ var fileStream = File.Create(fileName);
+ await using (fileStream.ConfigureAwait(false))
+ {
+ var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024;
+ var buffer = new byte[bufferSize];
+
+ var bytesRead = 0;
+ while ((bytesRead = await (await response.Content.ReadAsStreamAsync().ConfigureAwait(false)).ReadAsync(buffer).ConfigureAwait(false)) > 0)
+ {
+ await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead)).ConfigureAwait(false);
+ }
+ }
+ });
+ }
+
+ await _currentTransfers[hash].ConfigureAwait(false);
+ _currentTransfers.Remove(hash, out _);
+
+ fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
+
+ if (fi == null) return null;
+ }
+
+ _fileStatisticsService.LogFile(hash, fi.Length);
+
+ return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read);
+ }
+}
\ No newline at end of file
diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs
deleted file mode 100644
index ea7089b..0000000
--- a/MareSynchronosServer/MareSynchronosStaticFilesServer/CleanupService.cs
+++ /dev/null
@@ -1,153 +0,0 @@
-using MareSynchronosShared.Data;
-using MareSynchronosShared.Metrics;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-
-namespace MareSynchronosStaticFilesServer;
-
-public class CleanupService : IHostedService, IDisposable
-{
- private readonly MareMetrics _metrics;
- private readonly ILogger _logger;
- private readonly IServiceProvider _services;
- private readonly IConfiguration _configuration;
- private Timer? _timer;
-
- public CleanupService(MareMetrics metrics, ILogger logger, IServiceProvider services, IConfiguration configuration)
- {
- _metrics = metrics;
- _logger = logger;
- _services = services;
- _configuration = configuration.GetRequiredSection("MareSynchronos");
- }
-
- public Task StartAsync(CancellationToken cancellationToken)
- {
- _logger.LogInformation("Cleanup Service started");
-
- _logger.LogInformation("Calculating initial files");
-
- DirectoryInfo dir = new DirectoryInfo(_configuration["CacheDirectory"]);
- var allFiles = dir.GetFiles();
- _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFiles.Sum(f => f.Length));
- _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFiles.Length);
-
- _logger.LogInformation("Initial file calculation finished, starting periodic cleanup task");
-
- _timer = new Timer(CleanUp, null, TimeSpan.FromSeconds(15), TimeSpan.FromMinutes(10));
-
- return Task.CompletedTask;
- }
-
- private void CleanUp(object? state)
- {
- if (!int.TryParse(_configuration["UnusedFileRetentionPeriodInDays"], out var filesOlderThanDays))
- {
- filesOlderThanDays = 7;
- }
-
- using var scope = _services.CreateScope();
- using var dbContext = scope.ServiceProvider.GetService()!;
-
- _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", filesOlderThanDays);
-
- try
- {
- var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays));
-
- var allFiles = dbContext.Files.ToList();
- var cachedir = _configuration["CacheDirectory"];
- foreach (var file in allFiles.Where(f => f.Uploaded))
- {
- var fileName = Path.Combine(cachedir, file.Hash);
- var fi = new FileInfo(fileName);
- if (!fi.Exists)
- {
- _logger.LogInformation("File does not exist anymore: {fileName}", fileName);
- dbContext.Files.Remove(file);
- }
- else if (fi.LastAccessTime < prevTime)
- {
- _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi.Length);
- _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
- _logger.LogInformation("File outdated: {fileName}", fileName);
- dbContext.Files.Remove(file);
- fi.Delete();
- }
- }
-
- var allFilesHashes = new HashSet(allFiles.Select(a => a.Hash.ToUpperInvariant()));
- DirectoryInfo dir = new DirectoryInfo(cachedir);
- var allFilesInDir = dir.GetFiles();
- foreach (var file in allFilesInDir)
- {
- if (!allFilesHashes.Contains(file.Name.ToUpperInvariant()))
- {
- _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
- _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
- file.Delete();
- _logger.LogInformation("File not in DB, deleting: {fileName}", file.FullName);
- }
- }
- }
- catch (Exception ex)
- {
- _logger.LogWarning(ex, "Error during file cleanup");
- }
-
- var cacheSizeLimitInGiB = _configuration.GetValue("CacheSizeHardLimitInGiB", -1);
-
- try
- {
- if (cacheSizeLimitInGiB > 0)
- {
- _logger.LogInformation("Cleaning up files beyond the cache size limit");
- var allLocalFiles = Directory.EnumerateFiles(_configuration["CacheDirectory"])
- .Select(f => new FileInfo(f)).ToList().OrderBy(f => f.LastAccessTimeUtc).ToList();
- var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
- long cacheSizeLimitInBytes = (long)(cacheSizeLimitInGiB * 1024 * 1024 * 1024);
- HashSet removedHashes = new();
- while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any())
- {
- var oldestFile = allLocalFiles.First();
- removedHashes.Add(oldestFile.Name.ToLower());
- allLocalFiles.Remove(oldestFile);
- totalCacheSizeInBytes -= oldestFile.Length;
- _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, oldestFile.Length);
- _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
- oldestFile.Delete();
- }
-
- dbContext.Files.RemoveRange(dbContext.Files.Where(f => removedHashes.Contains(f.Hash.ToLower())));
- }
- }
- catch (Exception ex)
- {
- _logger.LogWarning(ex, "Error during cache size limit cleanup");
- }
-
- _logger.LogInformation($"Cleanup complete");
-
- dbContext.SaveChanges();
- }
-
- public Task StopAsync(CancellationToken cancellationToken)
- {
- _timer?.Change(Timeout.Infinite, 0);
-
- return Task.CompletedTask;
- }
-
- public void Dispose()
- {
- _timer?.Dispose();
- }
-}
diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileCleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileCleanupService.cs
new file mode 100644
index 0000000..7d11cd0
--- /dev/null
+++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileCleanupService.cs
@@ -0,0 +1,177 @@
+using ByteSizeLib;
+using MareSynchronosShared.Data;
+using MareSynchronosShared.Metrics;
+using MareSynchronosShared.Models;
+using System.Globalization;
+
+namespace MareSynchronosStaticFilesServer;
+
+public class FileCleanupService : IHostedService
+{
+ private readonly MareMetrics _metrics;
+ private readonly ILogger _logger;
+ private readonly IServiceProvider _services;
+ private readonly IConfiguration _configuration;
+ private readonly bool _isMainServer;
+ private readonly string _cacheDir;
+ private CancellationTokenSource _cleanupCts;
+
+ public FileCleanupService(MareMetrics metrics, ILogger logger, IServiceProvider services, IConfiguration configuration)
+ {
+ _metrics = metrics;
+ _logger = logger;
+ _services = services;
+ _configuration = configuration.GetRequiredSection("MareSynchronos");
+ _isMainServer = !string.IsNullOrEmpty(_configuration.GetValue("RemoteCacheSource", string.Empty));
+ _cacheDir = _configuration.GetValue("CacheDirectory");
+ }
+
+ public Task StartAsync(CancellationToken cancellationToken)
+ {
+ _logger.LogInformation("Cleanup Service started");
+
+ _logger.LogInformation("Calculating initial files");
+
+ _cleanupCts = new();
+
+ DirectoryInfo dir = new(_cacheDir);
+ var allFiles = dir.GetFiles();
+ _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotalSize, allFiles.Sum(f => f.Length));
+ _metrics.SetGaugeTo(MetricsAPI.GaugeFilesTotal, allFiles.Length);
+
+
+ _ = CleanUpTask(_cleanupCts.Token);
+
+ return Task.CompletedTask;
+ }
+
+ public async Task CleanUpTask(CancellationToken ct)
+ {
+ _logger.LogInformation("Starting periodic cleanup task");
+
+ while (!ct.IsCancellationRequested)
+ {
+ using var scope = _services.CreateScope();
+ using var dbContext = scope.ServiceProvider.GetService()!;
+
+ CleanUpOutdatedFiles(dbContext, ct);
+
+ CleanUpFilesBeyondSizeLimit(dbContext, ct);
+
+ if (_isMainServer)
+ {
+ await dbContext.SaveChangesAsync(ct).ConfigureAwait(false);
+ }
+
+ _logger.LogInformation("File Cleanup Complete, next run at {date}", DateTime.Now.Add(TimeSpan.FromMinutes(10)));
+ await Task.Delay(TimeSpan.FromMinutes(10), ct).ConfigureAwait(false);
+ }
+ }
+
+ private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct)
+ {
+ var cacheSizeLimitInGiB = _configuration.GetValue("CacheSizeHardLimitInGiB", -1);
+
+ if (cacheSizeLimitInGiB <= 0)
+ {
+ return;
+ }
+
+ try
+ {
+ _logger.LogInformation("Cleaning up files beyond the cache size limit of {cacheSizeLimit} GiB", cacheSizeLimitInGiB);
+ var allLocalFiles = Directory.EnumerateFiles(_cacheDir, "*", SearchOption.AllDirectories)
+ .Select(f => new FileInfo(f)).ToList()
+ .OrderBy(f => f.LastAccessTimeUtc).ToList();
+ var totalCacheSizeInBytes = allLocalFiles.Sum(s => s.Length);
+ long cacheSizeLimitInBytes = (long)ByteSize.FromGibiBytes(cacheSizeLimitInGiB).Bytes;
+ while (totalCacheSizeInBytes > cacheSizeLimitInBytes && allLocalFiles.Any() && !ct.IsCancellationRequested)
+ {
+ var oldestFile = allLocalFiles[0];
+ allLocalFiles.Remove(oldestFile);
+ totalCacheSizeInBytes -= oldestFile.Length;
+ _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, oldestFile.Length);
+ _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
+ _logger.LogInformation("Deleting {oldestFile} with size {size}MiB", oldestFile.FullName, ByteSize.FromBytes(oldestFile.Length).MebiBytes);
+ oldestFile.Delete();
+ if (_isMainServer)
+ {
+ FileCache f = new() { Hash = oldestFile.Name.ToUpperInvariant() };
+ dbContext.Entry(f).State = Microsoft.EntityFrameworkCore.EntityState.Deleted;
+ }
+ }
+ }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex, "Error during cache size limit cleanup");
+ }
+ }
+
+ private void CleanUpOutdatedFiles(MareDbContext dbContext, CancellationToken ct)
+ {
+ try
+ {
+ if (!int.TryParse(_configuration["UnusedFileRetentionPeriodInDays"], CultureInfo.InvariantCulture, out int filesOlderThanDays))
+ {
+ filesOlderThanDays = 7;
+ }
+
+ _logger.LogInformation("Cleaning up files older than {filesOlderThanDays} days", filesOlderThanDays);
+
+ // clean up files in DB but not on disk or last access is expired
+ var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays));
+ var allFiles = dbContext.Files.ToList();
+ foreach (var fileCache in allFiles.Where(f => f.Uploaded))
+ {
+ var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash);
+ if (file == null && _isMainServer)
+ {
+ _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash);
+ dbContext.Files.Remove(fileCache);
+ }
+ else if (file != null && file.LastAccessTime < prevTime)
+ {
+ _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
+ _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
+ _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
+ file.Delete();
+ if (_isMainServer)
+ dbContext.Files.Remove(fileCache);
+ }
+
+ ct.ThrowIfCancellationRequested();
+ }
+
+ // clean up files that are on disk but not in DB for some reason
+ if (_isMainServer)
+ {
+ var allFilesHashes = new HashSet(allFiles.Select(a => a.Hash.ToUpperInvariant()), StringComparer.Ordinal);
+ DirectoryInfo dir = new(_cacheDir);
+ var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories);
+ foreach (var file in allFilesInDir)
+ {
+ if (!allFilesHashes.Contains(file.Name.ToUpperInvariant()))
+ {
+ _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
+ _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
+ file.Delete();
+ _logger.LogInformation("File not in DB, deleting: {fileName}", file.FullName);
+ }
+
+ ct.ThrowIfCancellationRequested();
+ }
+ }
+ }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex, "Error during file cleanup of old files");
+ }
+ }
+
+ public Task StopAsync(CancellationToken cancellationToken)
+ {
+ _cleanupCts.Cancel();
+
+ return Task.CompletedTask;
+ }
+}
diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/FilePathUtil.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/FilePathUtil.cs
new file mode 100644
index 0000000..a78943e
--- /dev/null
+++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/FilePathUtil.cs
@@ -0,0 +1,27 @@
+namespace MareSynchronosStaticFilesServer;
+
+public static class FilePathUtil
+{
+ public static FileInfo? GetFileInfoForHash(string basePath, string hash)
+ {
+ FileInfo fi = new(Path.Combine(basePath, hash[0].ToString(), hash));
+ if (!fi.Exists)
+ {
+ fi = new FileInfo(Path.Combine(basePath, hash));
+ if (!fi.Exists)
+ {
+ return null;
+ }
+ }
+
+ return fi;
+ }
+
+ public static string GetFilePath(string basePath, string hash)
+ {
+ var dirPath = Path.Combine(basePath, hash[0].ToString());
+ var path = Path.Combine(dirPath, hash);
+ if (!Directory.Exists(dirPath)) Directory.CreateDirectory(dirPath);
+ return path;
+ }
+}
diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs
index d493a55..2fcba3d 100644
--- a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs
+++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileStatisticsService.cs
@@ -1,11 +1,5 @@
using MareSynchronosShared.Metrics;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
-using System;
using System.Collections.Concurrent;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
namespace MareSynchronosStaticFilesServer;
diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs
index b3f63af..9bf2fbb 100644
--- a/MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs
+++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/FilesController.cs
@@ -1,8 +1,4 @@
using Microsoft.AspNetCore.Mvc;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Logging;
-using System.IO;
-using System.Linq;
using System.Security.Claims;
namespace MareSynchronosStaticFilesServer;
@@ -11,29 +7,23 @@ namespace MareSynchronosStaticFilesServer;
public class FilesController : Controller
{
private readonly ILogger _logger;
- private readonly IConfiguration _configuration;
- private readonly FileStatisticsService _fileStatisticsService;
+ private readonly CachedFileProvider _cachedFileProvider;
- public FilesController(ILogger logger, IConfiguration configuration, FileStatisticsService fileStatisticsService)
+ public FilesController(ILogger logger, CachedFileProvider cachedFileProvider)
{
_logger = logger;
- _configuration = configuration;
- _fileStatisticsService = fileStatisticsService;
+ _cachedFileProvider = cachedFileProvider;
}
[HttpGet("{fileId}")]
- public IActionResult GetFile(string fileId)
+ public async Task GetFile(string fileId)
{
var authedUser = HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, ClaimTypes.NameIdentifier, System.StringComparison.Ordinal))?.Value ?? "Unknown";
_logger.LogInformation($"GetFile:{authedUser}:{fileId}");
- FileInfo fi = new(Path.Combine(_configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"], fileId));
- if (!fi.Exists) return NotFound();
+ var fs = await _cachedFileProvider.GetFileStream(fileId, Request.Headers["Authorization"]);
+ if (fs == null) return NotFound();
- _fileStatisticsService.LogFile(fileId, fi.Length);
-
- var fileStream = new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read);
-
- return File(fileStream, "application/octet-stream");
+ return File(fs, "application/octet-stream");
}
}
diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/GrpcFileService.cs
similarity index 69%
rename from MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs
rename to MareSynchronosServer/MareSynchronosStaticFilesServer/GrpcFileService.cs
index 1a234c8..9c5babd 100644
--- a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs
+++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/GrpcFileService.cs
@@ -3,23 +3,17 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Protos;
using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Logging;
-using System;
-using System.IO;
-using System.Linq;
-using System.Threading.Tasks;
namespace MareSynchronosStaticFilesServer;
-public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBase
+public class GrpcFileService : FileService.FileServiceBase
{
private readonly string _basePath;
private readonly MareDbContext _mareDbContext;
- private readonly ILogger _logger;
+ private readonly ILogger _logger;
private readonly MareMetrics _metricsClient;
- public FileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger logger, MareMetrics metricsClient)
+ public GrpcFileService(MareDbContext mareDbContext, IConfiguration configuration, ILogger logger, MareMetrics metricsClient)
{
_basePath = configuration.GetRequiredSection("MareSynchronos")["CacheDirectory"];
_mareDbContext = mareDbContext;
@@ -29,21 +23,21 @@ public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBa
public override async Task UploadFile(IAsyncStreamReader requestStream, ServerCallContext context)
{
- await requestStream.MoveNext();
+ _ = await requestStream.MoveNext().ConfigureAwait(false);
var uploadMsg = requestStream.Current;
- var filePath = Path.Combine(_basePath, uploadMsg.Hash);
+ var filePath = FilePathUtil.GetFilePath(_basePath, uploadMsg.Hash);
using var fileWriter = File.OpenWrite(filePath);
- var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == uploadMsg.Hash && f.UploaderUID == uploadMsg.Uploader);
+ var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == uploadMsg.Hash && f.UploaderUID == uploadMsg.Uploader).ConfigureAwait(false);
if (file != null)
{
- await fileWriter.WriteAsync(uploadMsg.FileData.ToArray());
+ await fileWriter.WriteAsync(uploadMsg.FileData.ToArray()).ConfigureAwait(false);
- while (await requestStream.MoveNext())
+ while (await requestStream.MoveNext().ConfigureAwait(false))
{
- await fileWriter.WriteAsync(requestStream.Current.FileData.ToArray());
+ await fileWriter.WriteAsync(requestStream.Current.FileData.ToArray()).ConfigureAwait(false);
}
- await fileWriter.FlushAsync();
+ await fileWriter.FlushAsync().ConfigureAwait(false);
fileWriter.Close();
var fileSize = new FileInfo(filePath).Length;
@@ -65,15 +59,15 @@ public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBa
{
try
{
- FileInfo fi = new FileInfo(Path.Combine(_basePath, hash));
- fi.Delete();
- var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
+ var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
+ fi?.Delete();
+ var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash).ConfigureAwait(false);
if (file != null)
{
_mareDbContext.Files.Remove(file);
- _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, 1);
- _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi.Length);
+ _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotal, fi == null ? 0 : 1);
+ _metricsClient.DecGauge(MetricsAPI.GaugeFilesTotalSize, fi?.Length ?? 0);
}
}
catch (Exception ex)
@@ -89,17 +83,10 @@ public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBa
public override Task GetFileSizes(FileSizeRequest request, ServerCallContext context)
{
FileSizeResponse response = new();
- foreach (var hash in request.Hash.Distinct())
+ foreach (var hash in request.Hash.Distinct(StringComparer.Ordinal))
{
- FileInfo fi = new(Path.Combine(_basePath, hash));
- if (fi.Exists)
- {
- response.HashToFileSize.Add(hash, fi.Length);
- }
- else
- {
- response.HashToFileSize.Add(hash, 0);
- }
+ FileInfo? fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
+ response.HashToFileSize.Add(hash, fi?.Length ?? 0);
}
return Task.FromResult(response);
diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj b/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj
index 41aecaf..05180b9 100644
--- a/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj
+++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj
@@ -2,6 +2,7 @@
net7.0
+ enable
@@ -23,7 +24,6 @@
all
runtime; build; native; contentfiles; analyzers; buildtransitive
-
diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs
index 8b791bc..64e1003 100644
--- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs
+++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs
@@ -1,7 +1,3 @@
-using Microsoft.AspNetCore.Hosting;
-using Microsoft.Extensions.Hosting;
-using System;
-
namespace MareSynchronosStaticFilesServer;
public class Program
diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs
index 3cbdfa1..b9648c0 100644
--- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs
+++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs
@@ -4,15 +4,8 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Builder;
-using Microsoft.AspNetCore.Hosting;
using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
using Prometheus;
-using System;
-using System.Collections.Generic;
namespace MareSynchronosStaticFilesServer;
@@ -65,10 +58,11 @@ public class Startup
MetricsAPI.GaugeFilesUniquePastHour,
MetricsAPI.GaugeFilesUniquePastHourSize
}));
+ services.AddSingleton();
services.AddSingleton();
services.AddHostedService(m => m.GetService());
- services.AddHostedService();
+ services.AddHostedService();
services.AddSingleton();
services.AddDbContextPool(options =>
@@ -99,8 +93,8 @@ public class Startup
app.UseRouting();
- var metricServer = new KestrelMetricServer(4981);
- metricServer.Start();
+ //var metricServer = new KestrelMetricServer(4981);
+ //metricServer.Start();
app.UseHttpMetrics();
@@ -109,7 +103,7 @@ public class Startup
app.UseEndpoints(e =>
{
- e.MapGrpcService();
+ e.MapGrpcService();
e.MapControllers();
});
}
diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json b/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json
index 75889bc..910547f 100644
--- a/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json
+++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/appsettings.json
@@ -24,7 +24,7 @@
"UnusedFileRetentionPeriodInDays": 7,
"CacheDirectory": "G:\\ServerTest",
"ServiceAddress": "http://localhost:5002",
- "IsSecondaryInstance": "false"
+ "RemoteCacheSourceUri": ""
},
"AllowedHosts": "*"
}