t0w0bi/ClubPenguinServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move GetFileSize to database

Browse Source
This commit is contained in:
rootdarkarchon
2022-12-28 04:50:37 +01:00
parent de2defe692
commit 31550b0aeb
8 changed files with 572 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs
View File

@@ -50,27 +50,21 @@ public partial class MareHub
Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false);
List<DownloadFileDto> response = new();
FileSizeRequest request = new FileSizeRequest();
request.Hash.AddRange(hashes);
Metadata headers = new Metadata()
{
{ "Authorization", Context.User!.Claims.SingleOrDefault(c => string.Equals(c.Type, ClaimTypes.Authentication, StringComparison.Ordinal))?.Value }
};
var grpcResponse = await _fileServiceClient.GetFileSizesAsync(request, headers).ConfigureAwait(false);
var cacheFile = await _dbContext.Files.Where(f => hashes.Contains(f.Hash)).Select(k => new { k.Hash, k.Size }).ToListAsync().ConfigureAwait(false);
foreach (var hash in grpcResponse.HashToFileSize)
foreach (var file in cacheFile)
{
var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, hash.Key, StringComparison.Ordinal));
var downloadFile = allFiles.SingleOrDefault(f => string.Equals(f.Hash, hash.Key, StringComparison.Ordinal));
var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
var downloadFile = allFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
response.Add(new DownloadFileDto
{
FileExists = hash.Value > 0,
FileExists = file.Size > 0,
ForbiddenBy = forbiddenFile?.ForbiddenBy ?? string.Empty,
IsForbidden = forbiddenFile != null,
Hash = hash.Key,
Size = hash.Value,
Url = new Uri(_cdnFullUri, hash.Key.ToUpperInvariant()).ToString()
Hash = file.Hash,
Size = file.Size,
Url = new Uri(_cdnFullUri, file.Hash.ToUpperInvariant()).ToString()
});
}

506
MareSynchronosServer/MareSynchronosShared/Migrations/20221228033214_FileCacheSize.Designer.cs generated Normal file
View File

@@ -0,0 +1,506 @@
// <auto-generated />
using System;
using MareSynchronosShared.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace MareSynchronosServer.Migrations
{
[DbContext(typeof(MareDbContext))]
[Migration("20221228033214_FileCacheSize")]
partial class FileCacheSize
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "7.0.0")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("MareSynchronosShared.Models.Auth", b =>
{
b.Property<string>("HashedKey")
.HasMaxLength(64)
.HasColumnType("character varying(64)")
.HasColumnName("hashed_key");
b.Property<string>("UserUID")
.HasColumnType("character varying(10)")
.HasColumnName("user_uid");
b.HasKey("HashedKey")
.HasName("pk_auth");
b.HasIndex("UserUID")
.HasDatabaseName("ix_auth_user_uid");
b.ToTable("auth", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.Banned", b =>
{
b.Property<string>("CharacterIdentification")
.HasMaxLength(100)
.HasColumnType("character varying(100)")
.HasColumnName("character_identification");
b.Property<string>("Reason")
.HasColumnType("text")
.HasColumnName("reason");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.HasKey("CharacterIdentification")
.HasName("pk_banned_users");
b.ToTable("banned_users", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.BannedRegistrations", b =>
{
b.Property<string>("DiscordIdOrLodestoneAuth")
.HasMaxLength(100)
.HasColumnType("character varying(100)")
.HasColumnName("discord_id_or_lodestone_auth");
b.HasKey("DiscordIdOrLodestoneAuth")
.HasName("pk_banned_registrations");
b.ToTable("banned_registrations", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b =>
{
b.Property<string>("UserUID")
.HasMaxLength(10)
.HasColumnType("character varying(10)")
.HasColumnName("user_uid");
b.Property<string>("OtherUserUID")
.HasMaxLength(10)
.HasColumnType("character varying(10)")
.HasColumnName("other_user_uid");
b.Property<bool>("AllowReceivingMessages")
.HasColumnType("boolean")
.HasColumnName("allow_receiving_messages");
b.Property<bool>("IsPaused")
.HasColumnType("boolean")
.HasColumnName("is_paused");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.HasKey("UserUID", "OtherUserUID")
.HasName("pk_client_pairs");
b.HasIndex("OtherUserUID")
.HasDatabaseName("ix_client_pairs_other_user_uid");
b.HasIndex("UserUID")
.HasDatabaseName("ix_client_pairs_user_uid");
b.ToTable("client_pairs", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b =>
{
b.Property<string>("Hash")
.HasMaxLength(40)
.HasColumnType("character varying(40)")
.HasColumnName("hash");
b.Property<long>("Size")
.HasColumnType("bigint")
.HasColumnName("size");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.Property<bool>("Uploaded")
.HasColumnType("boolean")
.HasColumnName("uploaded");
b.Property<string>("UploaderUID")
.HasMaxLength(10)
.HasColumnType("character varying(10)")
.HasColumnName("uploader_uid");
b.HasKey("Hash")
.HasName("pk_file_caches");
b.HasIndex("UploaderUID")
.HasDatabaseName("ix_file_caches_uploader_uid");
b.ToTable("file_caches", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.ForbiddenUploadEntry", b =>
{
b.Property<string>("Hash")
.HasMaxLength(40)
.HasColumnType("character varying(40)")
.HasColumnName("hash");
b.Property<string>("ForbiddenBy")
.HasMaxLength(100)
.HasColumnType("character varying(100)")
.HasColumnName("forbidden_by");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.HasKey("Hash")
.HasName("pk_forbidden_upload_entries");
b.ToTable("forbidden_upload_entries", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.Group", b =>
{
b.Property<string>("GID")
.HasMaxLength(20)
.HasColumnType("character varying(20)")
.HasColumnName("gid");
b.Property<string>("Alias")
.HasMaxLength(50)
.HasColumnType("character varying(50)")
.HasColumnName("alias");
b.Property<string>("HashedPassword")
.HasColumnType("text")
.HasColumnName("hashed_password");
b.Property<bool>("InvitesEnabled")
.HasColumnType("boolean")
.HasColumnName("invites_enabled");
b.Property<string>("OwnerUID")
.HasColumnType("character varying(10)")
.HasColumnName("owner_uid");
b.HasKey("GID")
.HasName("pk_groups");
b.HasIndex("OwnerUID")
.HasDatabaseName("ix_groups_owner_uid");
b.ToTable("groups", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b =>
{
b.Property<string>("GroupGID")
.HasColumnType("character varying(20)")
.HasColumnName("group_gid");
b.Property<string>("BannedUserUID")
.HasColumnType("character varying(10)")
.HasColumnName("banned_user_uid");
b.Property<string>("BannedByUID")
.HasColumnType("character varying(10)")
.HasColumnName("banned_by_uid");
b.Property<DateTime>("BannedOn")
.HasColumnType("timestamp with time zone")
.HasColumnName("banned_on");
b.Property<string>("BannedReason")
.HasColumnType("text")
.HasColumnName("banned_reason");
b.HasKey("GroupGID", "BannedUserUID")
.HasName("pk_group_bans");
b.HasIndex("BannedByUID")
.HasDatabaseName("ix_group_bans_banned_by_uid");
b.HasIndex("BannedUserUID")
.HasDatabaseName("ix_group_bans_banned_user_uid");
b.HasIndex("GroupGID")
.HasDatabaseName("ix_group_bans_group_gid");
b.ToTable("group_bans", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b =>
{
b.Property<string>("GroupGID")
.HasColumnType("character varying(20)")
.HasColumnName("group_gid");
b.Property<string>("GroupUserUID")
.HasColumnType("character varying(10)")
.HasColumnName("group_user_uid");
b.Property<bool>("IsModerator")
.HasColumnType("boolean")
.HasColumnName("is_moderator");
b.Property<bool>("IsPaused")
.HasColumnType("boolean")
.HasColumnName("is_paused");
b.Property<bool>("IsPinned")
.HasColumnType("boolean")
.HasColumnName("is_pinned");
b.HasKey("GroupGID", "GroupUserUID")
.HasName("pk_group_pairs");
b.HasIndex("GroupGID")
.HasDatabaseName("ix_group_pairs_group_gid");
b.HasIndex("GroupUserUID")
.HasDatabaseName("ix_group_pairs_group_user_uid");
b.ToTable("group_pairs", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b =>
{
b.Property<string>("GroupGID")
.HasColumnType("character varying(20)")
.HasColumnName("group_gid");
b.Property<string>("Invite")
.HasMaxLength(64)
.HasColumnType("character varying(64)")
.HasColumnName("invite");
b.Property<DateTime>("ExpirationDate")
.HasColumnType("timestamp with time zone")
.HasColumnName("expiration_date");
b.HasKey("GroupGID", "Invite")
.HasName("pk_group_temp_invites");
b.HasIndex("GroupGID")
.HasDatabaseName("ix_group_temp_invites_group_gid");
b.HasIndex("Invite")
.HasDatabaseName("ix_group_temp_invites_invite");
b.ToTable("group_temp_invites", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b =>
{
b.Property<decimal>("DiscordId")
.ValueGeneratedOnAdd()
.HasColumnType("numeric(20,0)")
.HasColumnName("discord_id");
b.Property<string>("HashedLodestoneId")
.HasMaxLength(100)
.HasColumnType("character varying(100)")
.HasColumnName("hashed_lodestone_id");
b.Property<string>("LodestoneAuthString")
.HasMaxLength(100)
.HasColumnType("character varying(100)")
.HasColumnName("lodestone_auth_string");
b.Property<DateTime?>("StartedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("started_at");
b.Property<string>("UserUID")
.HasColumnType("character varying(10)")
.HasColumnName("user_uid");
b.HasKey("DiscordId")
.HasName("pk_lodestone_auth");
b.HasIndex("UserUID")
.HasDatabaseName("ix_lodestone_auth_user_uid");
b.ToTable("lodestone_auth", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.User", b =>
{
b.Property<string>("UID")
.HasMaxLength(10)
.HasColumnType("character varying(10)")
.HasColumnName("uid");
b.Property<string>("Alias")
.HasMaxLength(15)
.HasColumnType("character varying(15)")
.HasColumnName("alias");
b.Property<bool>("IsAdmin")
.HasColumnType("boolean")
.HasColumnName("is_admin");
b.Property<bool>("IsModerator")
.HasColumnType("boolean")
.HasColumnName("is_moderator");
b.Property<DateTime>("LastLoggedIn")
.HasColumnType("timestamp with time zone")
.HasColumnName("last_logged_in");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.HasKey("UID")
.HasName("pk_users");
b.ToTable("users", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.Auth", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "User")
.WithMany()
.HasForeignKey("UserUID")
.HasConstraintName("fk_auth_users_user_temp_id");
b.Navigation("User");
});
modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "OtherUser")
.WithMany()
.HasForeignKey("OtherUserUID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_client_pairs_users_other_user_temp_id1");
b.HasOne("MareSynchronosShared.Models.User", "User")
.WithMany()
.HasForeignKey("UserUID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_client_pairs_users_user_temp_id2");
b.Navigation("OtherUser");
b.Navigation("User");
});
modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "Uploader")
.WithMany()
.HasForeignKey("UploaderUID")
.HasConstraintName("fk_file_caches_users_uploader_uid");
b.Navigation("Uploader");
});
modelBuilder.Entity("MareSynchronosShared.Models.Group", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "Owner")
.WithMany()
.HasForeignKey("OwnerUID")
.HasConstraintName("fk_groups_users_owner_temp_id7");
b.Navigation("Owner");
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "BannedBy")
.WithMany()
.HasForeignKey("BannedByUID")
.HasConstraintName("fk_group_bans_users_banned_by_temp_id4");
b.HasOne("MareSynchronosShared.Models.User", "BannedUser")
.WithMany()
.HasForeignKey("BannedUserUID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_group_bans_users_banned_user_temp_id5");
b.HasOne("MareSynchronosShared.Models.Group", "Group")
.WithMany()
.HasForeignKey("GroupGID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_group_bans_groups_group_temp_id");
b.Navigation("BannedBy");
b.Navigation("BannedUser");
b.Navigation("Group");
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b =>
{
b.HasOne("MareSynchronosShared.Models.Group", "Group")
.WithMany()
.HasForeignKey("GroupGID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_group_pairs_groups_group_temp_id1");
b.HasOne("MareSynchronosShared.Models.User", "GroupUser")
.WithMany()
.HasForeignKey("GroupUserUID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_group_pairs_users_group_user_temp_id6");
b.Navigation("Group");
b.Navigation("GroupUser");
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b =>
{
b.HasOne("MareSynchronosShared.Models.Group", "Group")
.WithMany()
.HasForeignKey("GroupGID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_group_temp_invites_groups_group_gid");
b.Navigation("Group");
});
modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "User")
.WithMany()
.HasForeignKey("UserUID")
.HasConstraintName("fk_lodestone_auth_users_user_uid");
b.Navigation("User");
});
#pragma warning restore 612, 618
}
}
}

29
MareSynchronosServer/MareSynchronosShared/Migrations/20221228033214_FileCacheSize.cs Normal file
View File

@@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace MareSynchronosServer.Migrations
{
/// <inheritdoc />
public partial class FileCacheSize : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<long>(
name: "size",
table: "file_caches",
type: "bigint",
nullable: false,
defaultValue: 0L);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "size",
table: "file_caches");
}
}
}

6
MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs
View File

@@ -17,7 +17,7 @@ namespace MareSynchronosServer.Migrations
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "6.0.8")
.HasAnnotation("ProductVersion", "7.0.0")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
@@ -123,6 +123,10 @@ namespace MareSynchronosServer.Migrations
.HasColumnType("character varying(40)")
.HasColumnName("hash");
b.Property<long>("Size")
.HasColumnType("bigint")
.HasColumnName("size");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()

1
MareSynchronosServer/MareSynchronosShared/Models/FileCache.cs
View File

@@ -13,4 +13,5 @@ public class FileCache
public bool Uploaded { get; set; }
[Timestamp]
public byte[] Timestamp { get; set; }
public long Size { get; set; }
}

9
MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto
View File

@@ -6,7 +6,6 @@ package mareservices;
service FileService {
rpc UploadFile (stream UploadFileRequest) returns (Empty);
rpc GetFileSizes (FileSizeRequest) returns (FileSizeResponse);
rpc DeleteFiles (DeleteFilesRequest) returns (Empty);
}
@@ -100,14 +99,6 @@ message DeleteFilesRequest {
repeated string hash = 1;
}
message FileSizeRequest {
repeated string hash = 1;
}
message FileSizeResponse {
map<string, int64> hashToFileSize = 1;
}
message AuthRequest {
string ip = 1;
string secretKey = 2;

26
MareSynchronosServer/MareSynchronosStaticFilesServer/FileCleanupService.cs
View File

@@ -3,7 +3,7 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics;
using MareSynchronosShared.Models;
using MareSynchronosShared.Services;
using Microsoft.Extensions.Options;
using Microsoft.EntityFrameworkCore;
namespace MareSynchronosStaticFilesServer;
@@ -52,7 +52,7 @@ public class FileCleanupService : IHostedService
using var scope = _services.CreateScope();
using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
CleanUpOutdatedFiles(dbContext, ct);
await CleanUpOutdatedFiles(dbContext, ct).ConfigureAwait(false);
CleanUpFilesBeyondSizeLimit(dbContext, ct);
@@ -109,7 +109,7 @@ public class FileCleanupService : IHostedService
}
}
private void CleanUpOutdatedFiles(MareDbContext dbContext, CancellationToken ct)
private async Task CleanUpOutdatedFiles(MareDbContext dbContext, CancellationToken ct)
{
try
{
@@ -125,14 +125,17 @@ public class FileCleanupService : IHostedService
// clean up files in DB but not on disk or last access is expired
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention));
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
var allFiles = dbContext.Files.ToList();
var allFiles = await dbContext.Files.ToListAsync().ConfigureAwait(false);
int fileCounter = 0;
foreach (var fileCache in allFiles.Where(f => f.Uploaded))
{
var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash);
bool fileDeleted = false;
if (file == null && _isMainServer)
{
_logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash);
dbContext.Files.Remove(fileCache);
fileDeleted = true;
}
else if (file != null && file.LastAccessTime < prevTime)
{
@@ -141,8 +144,11 @@ public class FileCleanupService : IHostedService
_logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
file.Delete();
if (_isMainServer)
{
fileDeleted = true;
dbContext.Files.Remove(fileCache);
}
}
else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion)
{
_metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length);
@@ -150,8 +156,20 @@ public class FileCleanupService : IHostedService
_logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes);
file.Delete();
if (_isMainServer)
{
fileDeleted = true;
dbContext.Files.Remove(fileCache);
}
}
if (_isMainServer && !fileDeleted && file != null && fileCache.Size == 0)
{
fileCache.Size = file.Length;
// commit every 1000 files to db
if (fileCounter % 1000 == 0) await dbContext.SaveChangesAsync().ConfigureAwait(false);
}
fileCounter++;
ct.ThrowIfCancellationRequested();
}

13
MareSynchronosServer/MareSynchronosStaticFilesServer/GrpcFileService.cs
View File

@@ -46,6 +46,7 @@ public class GrpcFileService : FileService.FileServiceBase
var fileSize = new FileInfo(filePath).Length;
file.Uploaded = true;
file.Size = fileSize;
await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
@@ -96,16 +97,4 @@ public class GrpcFileService : FileService.FileServiceBase
await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
return new Empty();
}
public override Task<FileSizeResponse> GetFileSizes(FileSizeRequest request, ServerCallContext context)
{
FileSizeResponse response = new();
foreach (var hash in request.Hash.Distinct(StringComparer.Ordinal))
{
FileInfo? fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
response.HashToFileSize.Add(hash, fi?.Length ?? 0);
}
return Task.FromResult(response);
}
}