potentially fix stuck uploads

This commit is contained in:
rootdarkarchon
2023-01-18 19:46:03 +01:00
parent 20d8970a15
commit fe80faab6b
7 changed files with 577 additions and 21 deletions

View File

@@ -95,36 +95,40 @@ public partial class MareHub
var uploader = await _dbContext.Users.SingleAsync(u => u.UID == UserUID).ConfigureAwait(false);
List<FileCache> fileCachesToUpload = new();
foreach (var file in userSentHashes)
foreach (var hash in userSentHashes)
{
// Skip empty file hashes, duplicate file hashes, forbidden file hashes and existing file hashes
if (string.IsNullOrEmpty(file)) { continue; }
if (notCoveredFiles.ContainsKey(file)) { continue; }
if (forbiddenFiles.ContainsKey(file))
if (string.IsNullOrEmpty(hash)) { continue; }
if (notCoveredFiles.ContainsKey(hash)) { continue; }
if (forbiddenFiles.ContainsKey(hash))
{
notCoveredFiles[file] = new UploadFileDto()
notCoveredFiles[hash] = new UploadFileDto()
{
ForbiddenBy = forbiddenFiles[file].ForbiddenBy,
Hash = file,
ForbiddenBy = forbiddenFiles[hash].ForbiddenBy,
Hash = hash,
IsForbidden = true,
};
continue;
}
if (existingFiles.ContainsKey(file)) { continue; }
if (existingFiles.TryGetValue(hash, out var file) && file.Uploaded) { continue; }
_logger.LogCallInfo(MareHubLogger.Args(file, "Missing"));
_logger.LogCallInfo(MareHubLogger.Args(hash, "Missing"));
fileCachesToUpload.Add(new FileCache()
if (file == null)
{
Hash = file,
Uploaded = false,
Uploader = uploader,
});
fileCachesToUpload.Add(new FileCache()
{
Hash = hash,
Uploaded = false,
Uploader = uploader,
UploadDate = DateTime.UtcNow,
});
}
notCoveredFiles[file] = new UploadFileDto()
notCoveredFiles[hash] = new UploadFileDto()
{
Hash = file,
Hash = hash,
};
}
//Save bulk

View File

@@ -50,9 +50,4 @@
<ProjectReference Include="..\MareSynchronosShared\MareSynchronosShared.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="Identity\" />
<Folder Include="Redis\" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,514 @@
// <auto-generated />
using System;
using MareSynchronosShared.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace MareSynchronosServer.Migrations
{
[DbContext(typeof(MareDbContext))]
[Migration("20230118184347_FilesUploadDate")]
partial class FilesUploadDate
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "7.0.0")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("MareSynchronosShared.Models.Auth", b =>
{
b.Property<string>("HashedKey")
.HasMaxLength(64)
.HasColumnType("character varying(64)")
.HasColumnName("hashed_key");
b.Property<bool>("IsBanned")
.HasColumnType("boolean")
.HasColumnName("is_banned");
b.Property<string>("UserUID")
.HasColumnType("character varying(10)")
.HasColumnName("user_uid");
b.HasKey("HashedKey")
.HasName("pk_auth");
b.HasIndex("UserUID")
.HasDatabaseName("ix_auth_user_uid");
b.ToTable("auth", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.Banned", b =>
{
b.Property<string>("CharacterIdentification")
.HasMaxLength(100)
.HasColumnType("character varying(100)")
.HasColumnName("character_identification");
b.Property<string>("Reason")
.HasColumnType("text")
.HasColumnName("reason");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.HasKey("CharacterIdentification")
.HasName("pk_banned_users");
b.ToTable("banned_users", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.BannedRegistrations", b =>
{
b.Property<string>("DiscordIdOrLodestoneAuth")
.HasMaxLength(100)
.HasColumnType("character varying(100)")
.HasColumnName("discord_id_or_lodestone_auth");
b.HasKey("DiscordIdOrLodestoneAuth")
.HasName("pk_banned_registrations");
b.ToTable("banned_registrations", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b =>
{
b.Property<string>("UserUID")
.HasMaxLength(10)
.HasColumnType("character varying(10)")
.HasColumnName("user_uid");
b.Property<string>("OtherUserUID")
.HasMaxLength(10)
.HasColumnType("character varying(10)")
.HasColumnName("other_user_uid");
b.Property<bool>("AllowReceivingMessages")
.HasColumnType("boolean")
.HasColumnName("allow_receiving_messages");
b.Property<bool>("IsPaused")
.HasColumnType("boolean")
.HasColumnName("is_paused");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.HasKey("UserUID", "OtherUserUID")
.HasName("pk_client_pairs");
b.HasIndex("OtherUserUID")
.HasDatabaseName("ix_client_pairs_other_user_uid");
b.HasIndex("UserUID")
.HasDatabaseName("ix_client_pairs_user_uid");
b.ToTable("client_pairs", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b =>
{
b.Property<string>("Hash")
.HasMaxLength(40)
.HasColumnType("character varying(40)")
.HasColumnName("hash");
b.Property<long>("Size")
.HasColumnType("bigint")
.HasColumnName("size");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.Property<DateTime>("UploadDate")
.HasColumnType("timestamp with time zone")
.HasColumnName("upload_date");
b.Property<bool>("Uploaded")
.HasColumnType("boolean")
.HasColumnName("uploaded");
b.Property<string>("UploaderUID")
.HasMaxLength(10)
.HasColumnType("character varying(10)")
.HasColumnName("uploader_uid");
b.HasKey("Hash")
.HasName("pk_file_caches");
b.HasIndex("UploaderUID")
.HasDatabaseName("ix_file_caches_uploader_uid");
b.ToTable("file_caches", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.ForbiddenUploadEntry", b =>
{
b.Property<string>("Hash")
.HasMaxLength(40)
.HasColumnType("character varying(40)")
.HasColumnName("hash");
b.Property<string>("ForbiddenBy")
.HasMaxLength(100)
.HasColumnType("character varying(100)")
.HasColumnName("forbidden_by");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.HasKey("Hash")
.HasName("pk_forbidden_upload_entries");
b.ToTable("forbidden_upload_entries", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.Group", b =>
{
b.Property<string>("GID")
.HasMaxLength(20)
.HasColumnType("character varying(20)")
.HasColumnName("gid");
b.Property<string>("Alias")
.HasMaxLength(50)
.HasColumnType("character varying(50)")
.HasColumnName("alias");
b.Property<string>("HashedPassword")
.HasColumnType("text")
.HasColumnName("hashed_password");
b.Property<bool>("InvitesEnabled")
.HasColumnType("boolean")
.HasColumnName("invites_enabled");
b.Property<string>("OwnerUID")
.HasColumnType("character varying(10)")
.HasColumnName("owner_uid");
b.HasKey("GID")
.HasName("pk_groups");
b.HasIndex("OwnerUID")
.HasDatabaseName("ix_groups_owner_uid");
b.ToTable("groups", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b =>
{
b.Property<string>("GroupGID")
.HasColumnType("character varying(20)")
.HasColumnName("group_gid");
b.Property<string>("BannedUserUID")
.HasColumnType("character varying(10)")
.HasColumnName("banned_user_uid");
b.Property<string>("BannedByUID")
.HasColumnType("character varying(10)")
.HasColumnName("banned_by_uid");
b.Property<DateTime>("BannedOn")
.HasColumnType("timestamp with time zone")
.HasColumnName("banned_on");
b.Property<string>("BannedReason")
.HasColumnType("text")
.HasColumnName("banned_reason");
b.HasKey("GroupGID", "BannedUserUID")
.HasName("pk_group_bans");
b.HasIndex("BannedByUID")
.HasDatabaseName("ix_group_bans_banned_by_uid");
b.HasIndex("BannedUserUID")
.HasDatabaseName("ix_group_bans_banned_user_uid");
b.HasIndex("GroupGID")
.HasDatabaseName("ix_group_bans_group_gid");
b.ToTable("group_bans", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b =>
{
b.Property<string>("GroupGID")
.HasColumnType("character varying(20)")
.HasColumnName("group_gid");
b.Property<string>("GroupUserUID")
.HasColumnType("character varying(10)")
.HasColumnName("group_user_uid");
b.Property<bool>("IsModerator")
.HasColumnType("boolean")
.HasColumnName("is_moderator");
b.Property<bool>("IsPaused")
.HasColumnType("boolean")
.HasColumnName("is_paused");
b.Property<bool>("IsPinned")
.HasColumnType("boolean")
.HasColumnName("is_pinned");
b.HasKey("GroupGID", "GroupUserUID")
.HasName("pk_group_pairs");
b.HasIndex("GroupGID")
.HasDatabaseName("ix_group_pairs_group_gid");
b.HasIndex("GroupUserUID")
.HasDatabaseName("ix_group_pairs_group_user_uid");
b.ToTable("group_pairs", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b =>
{
b.Property<string>("GroupGID")
.HasColumnType("character varying(20)")
.HasColumnName("group_gid");
b.Property<string>("Invite")
.HasMaxLength(64)
.HasColumnType("character varying(64)")
.HasColumnName("invite");
b.Property<DateTime>("ExpirationDate")
.HasColumnType("timestamp with time zone")
.HasColumnName("expiration_date");
b.HasKey("GroupGID", "Invite")
.HasName("pk_group_temp_invites");
b.HasIndex("GroupGID")
.HasDatabaseName("ix_group_temp_invites_group_gid");
b.HasIndex("Invite")
.HasDatabaseName("ix_group_temp_invites_invite");
b.ToTable("group_temp_invites", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b =>
{
b.Property<decimal>("DiscordId")
.ValueGeneratedOnAdd()
.HasColumnType("numeric(20,0)")
.HasColumnName("discord_id");
b.Property<string>("HashedLodestoneId")
.HasMaxLength(100)
.HasColumnType("character varying(100)")
.HasColumnName("hashed_lodestone_id");
b.Property<string>("LodestoneAuthString")
.HasMaxLength(100)
.HasColumnType("character varying(100)")
.HasColumnName("lodestone_auth_string");
b.Property<DateTime?>("StartedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("started_at");
b.Property<string>("UserUID")
.HasColumnType("character varying(10)")
.HasColumnName("user_uid");
b.HasKey("DiscordId")
.HasName("pk_lodestone_auth");
b.HasIndex("UserUID")
.HasDatabaseName("ix_lodestone_auth_user_uid");
b.ToTable("lodestone_auth", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.User", b =>
{
b.Property<string>("UID")
.HasMaxLength(10)
.HasColumnType("character varying(10)")
.HasColumnName("uid");
b.Property<string>("Alias")
.HasMaxLength(15)
.HasColumnType("character varying(15)")
.HasColumnName("alias");
b.Property<bool>("IsAdmin")
.HasColumnType("boolean")
.HasColumnName("is_admin");
b.Property<bool>("IsModerator")
.HasColumnType("boolean")
.HasColumnName("is_moderator");
b.Property<DateTime>("LastLoggedIn")
.HasColumnType("timestamp with time zone")
.HasColumnName("last_logged_in");
b.Property<byte[]>("Timestamp")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.HasKey("UID")
.HasName("pk_users");
b.ToTable("users", (string)null);
});
modelBuilder.Entity("MareSynchronosShared.Models.Auth", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "User")
.WithMany()
.HasForeignKey("UserUID")
.HasConstraintName("fk_auth_users_user_temp_id");
b.Navigation("User");
});
modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "OtherUser")
.WithMany()
.HasForeignKey("OtherUserUID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_client_pairs_users_other_user_temp_id1");
b.HasOne("MareSynchronosShared.Models.User", "User")
.WithMany()
.HasForeignKey("UserUID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_client_pairs_users_user_temp_id2");
b.Navigation("OtherUser");
b.Navigation("User");
});
modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "Uploader")
.WithMany()
.HasForeignKey("UploaderUID")
.HasConstraintName("fk_file_caches_users_uploader_uid");
b.Navigation("Uploader");
});
modelBuilder.Entity("MareSynchronosShared.Models.Group", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "Owner")
.WithMany()
.HasForeignKey("OwnerUID")
.HasConstraintName("fk_groups_users_owner_temp_id7");
b.Navigation("Owner");
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "BannedBy")
.WithMany()
.HasForeignKey("BannedByUID")
.HasConstraintName("fk_group_bans_users_banned_by_temp_id4");
b.HasOne("MareSynchronosShared.Models.User", "BannedUser")
.WithMany()
.HasForeignKey("BannedUserUID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_group_bans_users_banned_user_temp_id5");
b.HasOne("MareSynchronosShared.Models.Group", "Group")
.WithMany()
.HasForeignKey("GroupGID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_group_bans_groups_group_temp_id");
b.Navigation("BannedBy");
b.Navigation("BannedUser");
b.Navigation("Group");
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b =>
{
b.HasOne("MareSynchronosShared.Models.Group", "Group")
.WithMany()
.HasForeignKey("GroupGID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_group_pairs_groups_group_temp_id1");
b.HasOne("MareSynchronosShared.Models.User", "GroupUser")
.WithMany()
.HasForeignKey("GroupUserUID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_group_pairs_users_group_user_temp_id6");
b.Navigation("Group");
b.Navigation("GroupUser");
});
modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b =>
{
b.HasOne("MareSynchronosShared.Models.Group", "Group")
.WithMany()
.HasForeignKey("GroupGID")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_group_temp_invites_groups_group_gid");
b.Navigation("Group");
});
modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b =>
{
b.HasOne("MareSynchronosShared.Models.User", "User")
.WithMany()
.HasForeignKey("UserUID")
.HasConstraintName("fk_lodestone_auth_users_user_uid");
b.Navigation("User");
});
#pragma warning restore 612, 618
}
}
}

View File

@@ -0,0 +1,30 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace MareSynchronosServer.Migrations
{
/// <inheritdoc />
public partial class FilesUploadDate : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<DateTime>(
name: "upload_date",
table: "file_caches",
type: "timestamp with time zone",
nullable: false,
defaultValue: new DateTime(2000, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc));
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "upload_date",
table: "file_caches");
}
}
}

View File

@@ -137,6 +137,10 @@ namespace MareSynchronosServer.Migrations
.HasColumnType("bytea")
.HasColumnName("timestamp");
b.Property<DateTime>("UploadDate")
.HasColumnType("timestamp with time zone")
.HasColumnName("upload_date");
b.Property<bool>("Uploaded")
.HasColumnType("boolean")
.HasColumnName("uploaded");

View File

@@ -11,6 +11,7 @@ public class FileCache
public string UploaderUID { get; set; }
public User Uploader { get; set; }
public bool Uploaded { get; set; }
public DateTime UploadDate { get; set; }
[Timestamp]
public byte[] Timestamp { get; set; }
public long Size { get; set; }

View File

@@ -59,6 +59,8 @@ public class FileCleanupService : IHostedService
if (_isMainServer)
{
CleanUpStuckUploads(dbContext);
await dbContext.SaveChangesAsync(ct).ConfigureAwait(false);
}
@@ -72,6 +74,12 @@ public class FileCleanupService : IHostedService
}
}
private void CleanUpStuckUploads(MareDbContext dbContext)
{
var stuckUploads = dbContext.Files.Where(f => !f.Uploaded && f.UploadDate < DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10)));
dbContext.Files.RemoveRange(stuckUploads);
}
private void CleanUpFilesBeyondSizeLimit(MareDbContext dbContext, CancellationToken ct)
{
var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);