diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs index 64f448e..0312388 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs @@ -95,36 +95,40 @@ public partial class MareHub var uploader = await _dbContext.Users.SingleAsync(u => u.UID == UserUID).ConfigureAwait(false); List fileCachesToUpload = new(); - foreach (var file in userSentHashes) + foreach (var hash in userSentHashes) { // Skip empty file hashes, duplicate file hashes, forbidden file hashes and existing file hashes - if (string.IsNullOrEmpty(file)) { continue; } - if (notCoveredFiles.ContainsKey(file)) { continue; } - if (forbiddenFiles.ContainsKey(file)) + if (string.IsNullOrEmpty(hash)) { continue; } + if (notCoveredFiles.ContainsKey(hash)) { continue; } + if (forbiddenFiles.ContainsKey(hash)) { - notCoveredFiles[file] = new UploadFileDto() + notCoveredFiles[hash] = new UploadFileDto() { - ForbiddenBy = forbiddenFiles[file].ForbiddenBy, - Hash = file, + ForbiddenBy = forbiddenFiles[hash].ForbiddenBy, + Hash = hash, IsForbidden = true, }; continue; } - if (existingFiles.ContainsKey(file)) { continue; } + if (existingFiles.TryGetValue(hash, out var file) && file.Uploaded) { continue; } - _logger.LogCallInfo(MareHubLogger.Args(file, "Missing")); + _logger.LogCallInfo(MareHubLogger.Args(hash, "Missing")); - fileCachesToUpload.Add(new FileCache() + if (file == null) { - Hash = file, - Uploaded = false, - Uploader = uploader, - }); + fileCachesToUpload.Add(new FileCache() + { + Hash = hash, + Uploaded = false, + Uploader = uploader, + UploadDate = DateTime.UtcNow, + }); + } - notCoveredFiles[file] = new UploadFileDto() + notCoveredFiles[hash] = new UploadFileDto() { - Hash = file, + Hash = hash, }; } //Save bulk diff --git a/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj b/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj index f5f168d..a69a645 100644 --- a/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj +++ b/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj @@ -50,9 +50,4 @@ - - - - - diff --git a/MareSynchronosServer/MareSynchronosShared/Migrations/20230118184347_FilesUploadDate.Designer.cs b/MareSynchronosServer/MareSynchronosShared/Migrations/20230118184347_FilesUploadDate.Designer.cs new file mode 100644 index 0000000..1b5bad8 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosShared/Migrations/20230118184347_FilesUploadDate.Designer.cs @@ -0,0 +1,514 @@ +// +using System; +using MareSynchronosShared.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace MareSynchronosServer.Migrations +{ + [DbContext(typeof(MareDbContext))] + [Migration("20230118184347_FilesUploadDate")] + partial class FilesUploadDate + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "7.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("MareSynchronosShared.Models.Auth", b => + { + b.Property("HashedKey") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("hashed_key"); + + b.Property("IsBanned") + .HasColumnType("boolean") + .HasColumnName("is_banned"); + + b.Property("UserUID") + .HasColumnType("character varying(10)") + .HasColumnName("user_uid"); + + b.HasKey("HashedKey") + .HasName("pk_auth"); + + b.HasIndex("UserUID") + .HasDatabaseName("ix_auth_user_uid"); + + b.ToTable("auth", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.Banned", b => + { + b.Property("CharacterIdentification") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("character_identification"); + + b.Property("Reason") + .HasColumnType("text") + .HasColumnName("reason"); + + b.Property("Timestamp") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("bytea") + .HasColumnName("timestamp"); + + b.HasKey("CharacterIdentification") + .HasName("pk_banned_users"); + + b.ToTable("banned_users", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.BannedRegistrations", b => + { + b.Property("DiscordIdOrLodestoneAuth") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("discord_id_or_lodestone_auth"); + + b.HasKey("DiscordIdOrLodestoneAuth") + .HasName("pk_banned_registrations"); + + b.ToTable("banned_registrations", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b => + { + b.Property("UserUID") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("user_uid"); + + b.Property("OtherUserUID") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("other_user_uid"); + + b.Property("AllowReceivingMessages") + .HasColumnType("boolean") + .HasColumnName("allow_receiving_messages"); + + b.Property("IsPaused") + .HasColumnType("boolean") + .HasColumnName("is_paused"); + + b.Property("Timestamp") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("bytea") + .HasColumnName("timestamp"); + + b.HasKey("UserUID", "OtherUserUID") + .HasName("pk_client_pairs"); + + b.HasIndex("OtherUserUID") + .HasDatabaseName("ix_client_pairs_other_user_uid"); + + b.HasIndex("UserUID") + .HasDatabaseName("ix_client_pairs_user_uid"); + + b.ToTable("client_pairs", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b => + { + b.Property("Hash") + .HasMaxLength(40) + .HasColumnType("character varying(40)") + .HasColumnName("hash"); + + b.Property("Size") + .HasColumnType("bigint") + .HasColumnName("size"); + + b.Property("Timestamp") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("bytea") + .HasColumnName("timestamp"); + + b.Property("UploadDate") + .HasColumnType("timestamp with time zone") + .HasColumnName("upload_date"); + + b.Property("Uploaded") + .HasColumnType("boolean") + .HasColumnName("uploaded"); + + b.Property("UploaderUID") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("uploader_uid"); + + b.HasKey("Hash") + .HasName("pk_file_caches"); + + b.HasIndex("UploaderUID") + .HasDatabaseName("ix_file_caches_uploader_uid"); + + b.ToTable("file_caches", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.ForbiddenUploadEntry", b => + { + b.Property("Hash") + .HasMaxLength(40) + .HasColumnType("character varying(40)") + .HasColumnName("hash"); + + b.Property("ForbiddenBy") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("forbidden_by"); + + b.Property("Timestamp") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("bytea") + .HasColumnName("timestamp"); + + b.HasKey("Hash") + .HasName("pk_forbidden_upload_entries"); + + b.ToTable("forbidden_upload_entries", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.Group", b => + { + b.Property("GID") + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("gid"); + + b.Property("Alias") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("alias"); + + b.Property("HashedPassword") + .HasColumnType("text") + .HasColumnName("hashed_password"); + + b.Property("InvitesEnabled") + .HasColumnType("boolean") + .HasColumnName("invites_enabled"); + + b.Property("OwnerUID") + .HasColumnType("character varying(10)") + .HasColumnName("owner_uid"); + + b.HasKey("GID") + .HasName("pk_groups"); + + b.HasIndex("OwnerUID") + .HasDatabaseName("ix_groups_owner_uid"); + + b.ToTable("groups", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b => + { + b.Property("GroupGID") + .HasColumnType("character varying(20)") + .HasColumnName("group_gid"); + + b.Property("BannedUserUID") + .HasColumnType("character varying(10)") + .HasColumnName("banned_user_uid"); + + b.Property("BannedByUID") + .HasColumnType("character varying(10)") + .HasColumnName("banned_by_uid"); + + b.Property("BannedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("banned_on"); + + b.Property("BannedReason") + .HasColumnType("text") + .HasColumnName("banned_reason"); + + b.HasKey("GroupGID", "BannedUserUID") + .HasName("pk_group_bans"); + + b.HasIndex("BannedByUID") + .HasDatabaseName("ix_group_bans_banned_by_uid"); + + b.HasIndex("BannedUserUID") + .HasDatabaseName("ix_group_bans_banned_user_uid"); + + b.HasIndex("GroupGID") + .HasDatabaseName("ix_group_bans_group_gid"); + + b.ToTable("group_bans", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b => + { + b.Property("GroupGID") + .HasColumnType("character varying(20)") + .HasColumnName("group_gid"); + + b.Property("GroupUserUID") + .HasColumnType("character varying(10)") + .HasColumnName("group_user_uid"); + + b.Property("IsModerator") + .HasColumnType("boolean") + .HasColumnName("is_moderator"); + + b.Property("IsPaused") + .HasColumnType("boolean") + .HasColumnName("is_paused"); + + b.Property("IsPinned") + .HasColumnType("boolean") + .HasColumnName("is_pinned"); + + b.HasKey("GroupGID", "GroupUserUID") + .HasName("pk_group_pairs"); + + b.HasIndex("GroupGID") + .HasDatabaseName("ix_group_pairs_group_gid"); + + b.HasIndex("GroupUserUID") + .HasDatabaseName("ix_group_pairs_group_user_uid"); + + b.ToTable("group_pairs", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b => + { + b.Property("GroupGID") + .HasColumnType("character varying(20)") + .HasColumnName("group_gid"); + + b.Property("Invite") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("invite"); + + b.Property("ExpirationDate") + .HasColumnType("timestamp with time zone") + .HasColumnName("expiration_date"); + + b.HasKey("GroupGID", "Invite") + .HasName("pk_group_temp_invites"); + + b.HasIndex("GroupGID") + .HasDatabaseName("ix_group_temp_invites_group_gid"); + + b.HasIndex("Invite") + .HasDatabaseName("ix_group_temp_invites_invite"); + + b.ToTable("group_temp_invites", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b => + { + b.Property("DiscordId") + .ValueGeneratedOnAdd() + .HasColumnType("numeric(20,0)") + .HasColumnName("discord_id"); + + b.Property("HashedLodestoneId") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("hashed_lodestone_id"); + + b.Property("LodestoneAuthString") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("lodestone_auth_string"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("UserUID") + .HasColumnType("character varying(10)") + .HasColumnName("user_uid"); + + b.HasKey("DiscordId") + .HasName("pk_lodestone_auth"); + + b.HasIndex("UserUID") + .HasDatabaseName("ix_lodestone_auth_user_uid"); + + b.ToTable("lodestone_auth", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.User", b => + { + b.Property("UID") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("uid"); + + b.Property("Alias") + .HasMaxLength(15) + .HasColumnType("character varying(15)") + .HasColumnName("alias"); + + b.Property("IsAdmin") + .HasColumnType("boolean") + .HasColumnName("is_admin"); + + b.Property("IsModerator") + .HasColumnType("boolean") + .HasColumnName("is_moderator"); + + b.Property("LastLoggedIn") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_logged_in"); + + b.Property("Timestamp") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("bytea") + .HasColumnName("timestamp"); + + b.HasKey("UID") + .HasName("pk_users"); + + b.ToTable("users", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.Auth", b => + { + b.HasOne("MareSynchronosShared.Models.User", "User") + .WithMany() + .HasForeignKey("UserUID") + .HasConstraintName("fk_auth_users_user_temp_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b => + { + b.HasOne("MareSynchronosShared.Models.User", "OtherUser") + .WithMany() + .HasForeignKey("OtherUserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_client_pairs_users_other_user_temp_id1"); + + b.HasOne("MareSynchronosShared.Models.User", "User") + .WithMany() + .HasForeignKey("UserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_client_pairs_users_user_temp_id2"); + + b.Navigation("OtherUser"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b => + { + b.HasOne("MareSynchronosShared.Models.User", "Uploader") + .WithMany() + .HasForeignKey("UploaderUID") + .HasConstraintName("fk_file_caches_users_uploader_uid"); + + b.Navigation("Uploader"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.Group", b => + { + b.HasOne("MareSynchronosShared.Models.User", "Owner") + .WithMany() + .HasForeignKey("OwnerUID") + .HasConstraintName("fk_groups_users_owner_temp_id7"); + + b.Navigation("Owner"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b => + { + b.HasOne("MareSynchronosShared.Models.User", "BannedBy") + .WithMany() + .HasForeignKey("BannedByUID") + .HasConstraintName("fk_group_bans_users_banned_by_temp_id4"); + + b.HasOne("MareSynchronosShared.Models.User", "BannedUser") + .WithMany() + .HasForeignKey("BannedUserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_bans_users_banned_user_temp_id5"); + + b.HasOne("MareSynchronosShared.Models.Group", "Group") + .WithMany() + .HasForeignKey("GroupGID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_bans_groups_group_temp_id"); + + b.Navigation("BannedBy"); + + b.Navigation("BannedUser"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b => + { + b.HasOne("MareSynchronosShared.Models.Group", "Group") + .WithMany() + .HasForeignKey("GroupGID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_pairs_groups_group_temp_id1"); + + b.HasOne("MareSynchronosShared.Models.User", "GroupUser") + .WithMany() + .HasForeignKey("GroupUserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_pairs_users_group_user_temp_id6"); + + b.Navigation("Group"); + + b.Navigation("GroupUser"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b => + { + b.HasOne("MareSynchronosShared.Models.Group", "Group") + .WithMany() + .HasForeignKey("GroupGID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_temp_invites_groups_group_gid"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b => + { + b.HasOne("MareSynchronosShared.Models.User", "User") + .WithMany() + .HasForeignKey("UserUID") + .HasConstraintName("fk_lodestone_auth_users_user_uid"); + + b.Navigation("User"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/MareSynchronosServer/MareSynchronosShared/Migrations/20230118184347_FilesUploadDate.cs b/MareSynchronosServer/MareSynchronosShared/Migrations/20230118184347_FilesUploadDate.cs new file mode 100644 index 0000000..8f6a46a --- /dev/null +++ b/MareSynchronosServer/MareSynchronosShared/Migrations/20230118184347_FilesUploadDate.cs @@ -0,0 +1,30 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace MareSynchronosServer.Migrations +{ + /// + public partial class FilesUploadDate : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "upload_date", + table: "file_caches", + type: "timestamp with time zone", + nullable: false, + defaultValue: new DateTime(2000, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc)); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "upload_date", + table: "file_caches"); + } + } +} diff --git a/MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs b/MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs index c697c9f..776462c 100644 --- a/MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs +++ b/MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs @@ -137,6 +137,10 @@ namespace MareSynchronosServer.Migrations .HasColumnType("bytea") .HasColumnName("timestamp"); + b.Property("UploadDate") + .HasColumnType("timestamp with time zone") + .HasColumnName("upload_date"); + b.Property("Uploaded") .HasColumnType("boolean") .HasColumnName("uploaded"); diff --git a/MareSynchronosServer/MareSynchronosShared/Models/FileCache.cs b/MareSynchronosServer/MareSynchronosShared/Models/FileCache.cs index d386fa8..3ffd2c2 100644 --- a/MareSynchronosServer/MareSynchronosShared/Models/FileCache.cs +++ b/MareSynchronosServer/MareSynchronosShared/Models/FileCache.cs @@ -11,6 +11,7 @@ public class FileCache public string UploaderUID { get; set; } public User Uploader { get; set; } public bool Uploaded { get; set; } + public DateTime UploadDate { get; set; } [Timestamp] public byte[] Timestamp { get; set; } public long Size { get; set; } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs index 256c80c..d8266b7 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs @@ -59,6 +59,8 @@ public class FileCleanupService : IHostedService if (_isMainServer) { + CleanUpStuckUploads(dbContext); + await dbContext.SaveChangesAsync(ct).ConfigureAwait(false); } @@ -72,6 +74,12 @@ public class FileCleanupService : IHostedService } } + private void CleanUpStuckUploads(MareDbContext dbContext) + { + var stuckUploads = dbContext.Files.Where(f => !f.Uploaded && f.UploadDate < DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10))); + dbContext.Files.RemoveRange(stuckUploads); + } + private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct) { var sizeLimit = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);