move GetFileSize to database

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs

@@ -50,27 +50,21 @@ public partial class MareHub
             Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false);
         List<DownloadFileDto> response = new();
 
-        FileSizeRequest request = new FileSizeRequest();
+        var cacheFile = await _dbContext.Files.Where(f => hashes.Contains(f.Hash)).Select(k => new { k.Hash, k.Size }).ToListAsync().ConfigureAwait(false);
-        request.Hash.AddRange(hashes);
-        Metadata headers = new Metadata()
-        {
-            { "Authorization", Context.User!.Claims.SingleOrDefault(c => string.Equals(c.Type, ClaimTypes.Authentication, StringComparison.Ordinal))?.Value }
-        };
-        var grpcResponse = await _fileServiceClient.GetFileSizesAsync(request, headers).ConfigureAwait(false);
 
-        foreach (var hash in grpcResponse.HashToFileSize)
+        foreach (var file in cacheFile)
         {
-            var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, hash.Key, StringComparison.Ordinal));
+            var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
-            var downloadFile = allFiles.SingleOrDefault(f => string.Equals(f.Hash, hash.Key, StringComparison.Ordinal));
+            var downloadFile = allFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
 
             response.Add(new DownloadFileDto
             {
-                FileExists = hash.Value > 0,
+                FileExists = file.Size > 0,
                 ForbiddenBy = forbiddenFile?.ForbiddenBy ?? string.Empty,
                 IsForbidden = forbiddenFile != null,
-                Hash = hash.Key,
+                Hash = file.Hash,
-                Size = hash.Value,
+                Size = file.Size,
-                Url = new Uri(_cdnFullUri, hash.Key.ToUpperInvariant()).ToString()
+                Url = new Uri(_cdnFullUri, file.Hash.ToUpperInvariant()).ToString()
             });
         }
 

MareSynchronosServer/MareSynchronosShared/Migrations/20221228033214_FileCacheSize.Designer.cs

@@ -0,0 +1,506 @@
+// <auto-generated />
+using System;
+using MareSynchronosShared.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+#nullable disable
+
+namespace MareSynchronosServer.Migrations
+{
+    [DbContext(typeof(MareDbContext))]
+    [Migration("20221228033214_FileCacheSize")]
+    partial class FileCacheSize
+    {
+        /// <inheritdoc />
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("ProductVersion", "7.0.0")
+                .HasAnnotation("Relational:MaxIdentifierLength", 63);
+
+            NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
+
+            modelBuilder.Entity("MareSynchronosShared.Models.Auth", b =>
+                {
+                    b.Property<string>("HashedKey")
+                        .HasMaxLength(64)
+                        .HasColumnType("character varying(64)")
+                        .HasColumnName("hashed_key");
+
+                    b.Property<string>("UserUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("user_uid");
+
+                    b.HasKey("HashedKey")
+                        .HasName("pk_auth");
+
+                    b.HasIndex("UserUID")
+                        .HasDatabaseName("ix_auth_user_uid");
+
+                    b.ToTable("auth", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.Banned", b =>
+                {
+                    b.Property<string>("CharacterIdentification")
+                        .HasMaxLength(100)
+                        .HasColumnType("character varying(100)")
+                        .HasColumnName("character_identification");
+
+                    b.Property<string>("Reason")
+                        .HasColumnType("text")
+                        .HasColumnName("reason");
+
+                    b.Property<byte[]>("Timestamp")
+                        .IsConcurrencyToken()
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("bytea")
+                        .HasColumnName("timestamp");
+
+                    b.HasKey("CharacterIdentification")
+                        .HasName("pk_banned_users");
+
+                    b.ToTable("banned_users", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.BannedRegistrations", b =>
+                {
+                    b.Property<string>("DiscordIdOrLodestoneAuth")
+                        .HasMaxLength(100)
+                        .HasColumnType("character varying(100)")
+                        .HasColumnName("discord_id_or_lodestone_auth");
+
+                    b.HasKey("DiscordIdOrLodestoneAuth")
+                        .HasName("pk_banned_registrations");
+
+                    b.ToTable("banned_registrations", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b =>
+                {
+                    b.Property<string>("UserUID")
+                        .HasMaxLength(10)
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("user_uid");
+
+                    b.Property<string>("OtherUserUID")
+                        .HasMaxLength(10)
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("other_user_uid");
+
+                    b.Property<bool>("AllowReceivingMessages")
+                        .HasColumnType("boolean")
+                        .HasColumnName("allow_receiving_messages");
+
+                    b.Property<bool>("IsPaused")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_paused");
+
+                    b.Property<byte[]>("Timestamp")
+                        .IsConcurrencyToken()
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("bytea")
+                        .HasColumnName("timestamp");
+
+                    b.HasKey("UserUID", "OtherUserUID")
+                        .HasName("pk_client_pairs");
+
+                    b.HasIndex("OtherUserUID")
+                        .HasDatabaseName("ix_client_pairs_other_user_uid");
+
+                    b.HasIndex("UserUID")
+                        .HasDatabaseName("ix_client_pairs_user_uid");
+
+                    b.ToTable("client_pairs", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b =>
+                {
+                    b.Property<string>("Hash")
+                        .HasMaxLength(40)
+                        .HasColumnType("character varying(40)")
+                        .HasColumnName("hash");
+
+                    b.Property<long>("Size")
+                        .HasColumnType("bigint")
+                        .HasColumnName("size");
+
+                    b.Property<byte[]>("Timestamp")
+                        .IsConcurrencyToken()
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("bytea")
+                        .HasColumnName("timestamp");
+
+                    b.Property<bool>("Uploaded")
+                        .HasColumnType("boolean")
+                        .HasColumnName("uploaded");
+
+                    b.Property<string>("UploaderUID")
+                        .HasMaxLength(10)
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("uploader_uid");
+
+                    b.HasKey("Hash")
+                        .HasName("pk_file_caches");
+
+                    b.HasIndex("UploaderUID")
+                        .HasDatabaseName("ix_file_caches_uploader_uid");
+
+                    b.ToTable("file_caches", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.ForbiddenUploadEntry", b =>
+                {
+                    b.Property<string>("Hash")
+                        .HasMaxLength(40)
+                        .HasColumnType("character varying(40)")
+                        .HasColumnName("hash");
+
+                    b.Property<string>("ForbiddenBy")
+                        .HasMaxLength(100)
+                        .HasColumnType("character varying(100)")
+                        .HasColumnName("forbidden_by");
+
+                    b.Property<byte[]>("Timestamp")
+                        .IsConcurrencyToken()
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("bytea")
+                        .HasColumnName("timestamp");
+
+                    b.HasKey("Hash")
+                        .HasName("pk_forbidden_upload_entries");
+
+                    b.ToTable("forbidden_upload_entries", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.Group", b =>
+                {
+                    b.Property<string>("GID")
+                        .HasMaxLength(20)
+                        .HasColumnType("character varying(20)")
+                        .HasColumnName("gid");
+
+                    b.Property<string>("Alias")
+                        .HasMaxLength(50)
+                        .HasColumnType("character varying(50)")
+                        .HasColumnName("alias");
+
+                    b.Property<string>("HashedPassword")
+                        .HasColumnType("text")
+                        .HasColumnName("hashed_password");
+
+                    b.Property<bool>("InvitesEnabled")
+                        .HasColumnType("boolean")
+                        .HasColumnName("invites_enabled");
+
+                    b.Property<string>("OwnerUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("owner_uid");
+
+                    b.HasKey("GID")
+                        .HasName("pk_groups");
+
+                    b.HasIndex("OwnerUID")
+                        .HasDatabaseName("ix_groups_owner_uid");
+
+                    b.ToTable("groups", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b =>
+                {
+                    b.Property<string>("GroupGID")
+                        .HasColumnType("character varying(20)")
+                        .HasColumnName("group_gid");
+
+                    b.Property<string>("BannedUserUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("banned_user_uid");
+
+                    b.Property<string>("BannedByUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("banned_by_uid");
+
+                    b.Property<DateTime>("BannedOn")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("banned_on");
+
+                    b.Property<string>("BannedReason")
+                        .HasColumnType("text")
+                        .HasColumnName("banned_reason");
+
+                    b.HasKey("GroupGID", "BannedUserUID")
+                        .HasName("pk_group_bans");
+
+                    b.HasIndex("BannedByUID")
+                        .HasDatabaseName("ix_group_bans_banned_by_uid");
+
+                    b.HasIndex("BannedUserUID")
+                        .HasDatabaseName("ix_group_bans_banned_user_uid");
+
+                    b.HasIndex("GroupGID")
+                        .HasDatabaseName("ix_group_bans_group_gid");
+
+                    b.ToTable("group_bans", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b =>
+                {
+                    b.Property<string>("GroupGID")
+                        .HasColumnType("character varying(20)")
+                        .HasColumnName("group_gid");
+
+                    b.Property<string>("GroupUserUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("group_user_uid");
+
+                    b.Property<bool>("IsModerator")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_moderator");
+
+                    b.Property<bool>("IsPaused")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_paused");
+
+                    b.Property<bool>("IsPinned")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_pinned");
+
+                    b.HasKey("GroupGID", "GroupUserUID")
+                        .HasName("pk_group_pairs");
+
+                    b.HasIndex("GroupGID")
+                        .HasDatabaseName("ix_group_pairs_group_gid");
+
+                    b.HasIndex("GroupUserUID")
+                        .HasDatabaseName("ix_group_pairs_group_user_uid");
+
+                    b.ToTable("group_pairs", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b =>
+                {
+                    b.Property<string>("GroupGID")
+                        .HasColumnType("character varying(20)")
+                        .HasColumnName("group_gid");
+
+                    b.Property<string>("Invite")
+                        .HasMaxLength(64)
+                        .HasColumnType("character varying(64)")
+                        .HasColumnName("invite");
+
+                    b.Property<DateTime>("ExpirationDate")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("expiration_date");
+
+                    b.HasKey("GroupGID", "Invite")
+                        .HasName("pk_group_temp_invites");
+
+                    b.HasIndex("GroupGID")
+                        .HasDatabaseName("ix_group_temp_invites_group_gid");
+
+                    b.HasIndex("Invite")
+                        .HasDatabaseName("ix_group_temp_invites_invite");
+
+                    b.ToTable("group_temp_invites", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b =>
+                {
+                    b.Property<decimal>("DiscordId")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("numeric(20,0)")
+                        .HasColumnName("discord_id");
+
+                    b.Property<string>("HashedLodestoneId")
+                        .HasMaxLength(100)
+                        .HasColumnType("character varying(100)")
+                        .HasColumnName("hashed_lodestone_id");
+
+                    b.Property<string>("LodestoneAuthString")
+                        .HasMaxLength(100)
+                        .HasColumnType("character varying(100)")
+                        .HasColumnName("lodestone_auth_string");
+
+                    b.Property<DateTime?>("StartedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("started_at");
+
+                    b.Property<string>("UserUID")
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("user_uid");
+
+                    b.HasKey("DiscordId")
+                        .HasName("pk_lodestone_auth");
+
+                    b.HasIndex("UserUID")
+                        .HasDatabaseName("ix_lodestone_auth_user_uid");
+
+                    b.ToTable("lodestone_auth", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.User", b =>
+                {
+                    b.Property<string>("UID")
+                        .HasMaxLength(10)
+                        .HasColumnType("character varying(10)")
+                        .HasColumnName("uid");
+
+                    b.Property<string>("Alias")
+                        .HasMaxLength(15)
+                        .HasColumnType("character varying(15)")
+                        .HasColumnName("alias");
+
+                    b.Property<bool>("IsAdmin")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_admin");
+
+                    b.Property<bool>("IsModerator")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_moderator");
+
+                    b.Property<DateTime>("LastLoggedIn")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("last_logged_in");
+
+                    b.Property<byte[]>("Timestamp")
+                        .IsConcurrencyToken()
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("bytea")
+                        .HasColumnName("timestamp");
+
+                    b.HasKey("UID")
+                        .HasName("pk_users");
+
+                    b.ToTable("users", (string)null);
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.Auth", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "User")
+                        .WithMany()
+                        .HasForeignKey("UserUID")
+                        .HasConstraintName("fk_auth_users_user_temp_id");
+
+                    b.Navigation("User");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "OtherUser")
+                        .WithMany()
+                        .HasForeignKey("OtherUserUID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_client_pairs_users_other_user_temp_id1");
+
+                    b.HasOne("MareSynchronosShared.Models.User", "User")
+                        .WithMany()
+                        .HasForeignKey("UserUID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_client_pairs_users_user_temp_id2");
+
+                    b.Navigation("OtherUser");
+
+                    b.Navigation("User");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "Uploader")
+                        .WithMany()
+                        .HasForeignKey("UploaderUID")
+                        .HasConstraintName("fk_file_caches_users_uploader_uid");
+
+                    b.Navigation("Uploader");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.Group", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "Owner")
+                        .WithMany()
+                        .HasForeignKey("OwnerUID")
+                        .HasConstraintName("fk_groups_users_owner_temp_id7");
+
+                    b.Navigation("Owner");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "BannedBy")
+                        .WithMany()
+                        .HasForeignKey("BannedByUID")
+                        .HasConstraintName("fk_group_bans_users_banned_by_temp_id4");
+
+                    b.HasOne("MareSynchronosShared.Models.User", "BannedUser")
+                        .WithMany()
+                        .HasForeignKey("BannedUserUID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_group_bans_users_banned_user_temp_id5");
+
+                    b.HasOne("MareSynchronosShared.Models.Group", "Group")
+                        .WithMany()
+                        .HasForeignKey("GroupGID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_group_bans_groups_group_temp_id");
+
+                    b.Navigation("BannedBy");
+
+                    b.Navigation("BannedUser");
+
+                    b.Navigation("Group");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.Group", "Group")
+                        .WithMany()
+                        .HasForeignKey("GroupGID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_group_pairs_groups_group_temp_id1");
+
+                    b.HasOne("MareSynchronosShared.Models.User", "GroupUser")
+                        .WithMany()
+                        .HasForeignKey("GroupUserUID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_group_pairs_users_group_user_temp_id6");
+
+                    b.Navigation("Group");
+
+                    b.Navigation("GroupUser");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.Group", "Group")
+                        .WithMany()
+                        .HasForeignKey("GroupGID")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_group_temp_invites_groups_group_gid");
+
+                    b.Navigation("Group");
+                });
+
+            modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b =>
+                {
+                    b.HasOne("MareSynchronosShared.Models.User", "User")
+                        .WithMany()
+                        .HasForeignKey("UserUID")
+                        .HasConstraintName("fk_lodestone_auth_users_user_uid");
+
+                    b.Navigation("User");
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}

MareSynchronosServer/MareSynchronosShared/Migrations/20221228033214_FileCacheSize.cs

@@ -0,0 +1,29 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace MareSynchronosServer.Migrations
+{
+    /// <inheritdoc />
+    public partial class FileCacheSize : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<long>(
+                name: "size",
+                table: "file_caches",
+                type: "bigint",
+                nullable: false,
+                defaultValue: 0L);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                name: "size",
+                table: "file_caches");
+        }
+    }
+}

MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs

@@ -17,7 +17,7 @@ namespace MareSynchronosServer.Migrations
         {
 #pragma warning disable 612, 618
             modelBuilder
-                .HasAnnotation("ProductVersion", "6.0.8")
+                .HasAnnotation("ProductVersion", "7.0.0")
                 .HasAnnotation("Relational:MaxIdentifierLength", 63);
 
             NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
@@ -123,6 +123,10 @@ namespace MareSynchronosServer.Migrations
                         .HasColumnType("character varying(40)")
                         .HasColumnName("hash");
 
+                    b.Property<long>("Size")
+                        .HasColumnType("bigint")
+                        .HasColumnName("size");
+
                     b.Property<byte[]>("Timestamp")
                         .IsConcurrencyToken()
                         .ValueGeneratedOnAddOrUpdate()

MareSynchronosServer/MareSynchronosShared/Models/FileCache.cs

@@ -13,4 +13,5 @@ public class FileCache
     public bool Uploaded { get; set; }
     [Timestamp]
     public byte[] Timestamp { get; set; }
+    public long Size { get; set; }
 }

MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto

@@ -6,7 +6,6 @@ package mareservices;
 
 service FileService {
 	rpc UploadFile (stream UploadFileRequest) returns (Empty);
-	rpc GetFileSizes (FileSizeRequest) returns (FileSizeResponse);
 	rpc DeleteFiles (DeleteFilesRequest) returns (Empty);
 }
 
@@ -100,14 +99,6 @@ message DeleteFilesRequest {
 	repeated string hash = 1;
 }
 
-message FileSizeRequest {
-	repeated string hash = 1;
-}
-
-message FileSizeResponse {
-	map<string, int64> hashToFileSize = 1;
-}
-
 message AuthRequest {
 	string ip = 1;
 	string secretKey = 2;

MareSynchronosServer/MareSynchronosStaticFilesServer/FileCleanupService.cs

@@ -3,7 +3,7 @@ using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
 using MareSynchronosShared.Models;
 using MareSynchronosShared.Services;
-using Microsoft.Extensions.Options;
+using Microsoft.EntityFrameworkCore;
 
 namespace MareSynchronosStaticFilesServer;
 
@@ -52,7 +52,7 @@ public class FileCleanupService : IHostedService
             using var scope = _services.CreateScope();
             using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
 
-            CleanUpOutdatedFiles(dbContext, ct);
+            await CleanUpOutdatedFiles(dbContext, ct).ConfigureAwait(false);
 
             CleanUpFilesBeyondSizeLimit(dbContext, ct);
 
@@ -109,7 +109,7 @@ public class FileCleanupService : IHostedService
         }
     }
 
-    private void CleanUpOutdatedFiles(MareDbContext dbContext, CancellationToken ct)
+    private async Task CleanUpOutdatedFiles(MareDbContext dbContext, CancellationToken ct)
     {
         try
         {
@@ -125,14 +125,17 @@ public class FileCleanupService : IHostedService
             // clean up files in DB but not on disk or last access is expired
             var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention));
             var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
-            var allFiles = dbContext.Files.ToList();
+            var allFiles = await dbContext.Files.ToListAsync().ConfigureAwait(false);
+            int fileCounter = 0;
             foreach (var fileCache in allFiles.Where(f => f.Uploaded))
             {
                 var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash);
+                bool fileDeleted = false;
                 if (file == null && _isMainServer)
                 {
                     _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash);
                     dbContext.Files.Remove(fileCache);
+                    fileDeleted = true;
                 }
                 else if (file != null && file.LastAccessTime < prevTime)
                 {
@@ -141,7 +144,10 @@ public class FileCleanupService : IHostedService
                     _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
                     file.Delete();
                     if (_isMainServer)
+                    {
+                        fileDeleted = true;
                         dbContext.Files.Remove(fileCache);
+                    }
                 }
                 else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
                 {
@@ -150,9 +156,21 @@ public class FileCleanupService : IHostedService
                     _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
                     file.Delete();
                     if (_isMainServer)
+                    {
+                        fileDeleted = true;
                         dbContext.Files.Remove(fileCache);
+                    }
                 }
 
+                if (_isMainServer && !fileDeleted && file != null && fileCache.Size == 0)
+                {
+                    fileCache.Size = file.Length;
+                    // commit every 1000 files to db
+                    if (fileCounter % 1000 == 0) await dbContext.SaveChangesAsync().ConfigureAwait(false);
+                }
+
+                fileCounter++;
+
                 ct.ThrowIfCancellationRequested();
             }
 

MareSynchronosServer/MareSynchronosStaticFilesServer/GrpcFileService.cs

@@ -46,6 +46,7 @@ public class GrpcFileService : FileService.FileServiceBase
 
                 var fileSize = new FileInfo(filePath).Length;
                 file.Uploaded = true;
+                file.Size = fileSize;
 
                 await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
 
@@ -96,16 +97,4 @@ public class GrpcFileService : FileService.FileServiceBase
         await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
         return new Empty();
     }
-
-    public override Task<FileSizeResponse> GetFileSizes(FileSizeRequest request, ServerCallContext context)
-    {
-        FileSizeResponse response = new();
-        foreach (var hash in request.Hash.Distinct(StringComparer.Ordinal))
-        {
-            FileInfo? fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
-            response.HashToFileSize.Add(hash, fi?.Length ?? 0);
-        }
-
-        return Task.FromResult(response);
-    }
 }