t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Mare 0.9 (#65)

Browse Source 
* add jwt expiry

* start of 0.9 api impl

* some stuff idk

* some more impl

* some cleanup

* remove grouppair, add configuration, rework some pair drawing stuff

* do some stuff

* rework some ui

* I don't even know anymore

* add cancellationtoken

* token bla

* ui fixes etc

* probably individual adding/removing now working fully as expected

* add working report popup

* I guess it's more syncshell shit or so

* popup shit idk

* work out most of the syncshell bullshit I guess

* delete some old crap

* are we actually getting closer to the end

* update pair info stuff

* more fixes/adjustments, idk

* refactor some things

* some rework

* some more cleanup

* cleanup

* make menu buttons w i d e

* better icon text buttons

* add all syncshell folder and ordering fixes

---------

Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>
This commit is contained in:
rootdarkarchon
2023-10-17 21:36:44 +02:00
committed by GitHub
parent f15b8f6bbd
commit 14575a4a6b
111 changed files with 3456 additions and 3174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
MareSynchronos/WebAPI/Files/FileDownloadManager.cs
View File

@@ -16,8 +16,8 @@ namespace MareSynchronos.WebAPI.Files;
public partial class FileDownloadManager : DisposableMediatorSubscriberBase
{
private readonly Dictionary<string, FileDownloadStatus> _downloadStatus;
private readonly FileCacheManager _fileDbManager;
private readonly FileCompactor _fileCompactor;
private readonly FileCacheManager _fileDbManager;
private readonly FileTransferOrchestrator _orchestrator;
public FileDownloadManager(ILogger<FileDownloadManager> logger, MareMediator mediator,
@@ -30,12 +30,20 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
_fileCompactor = fileCompactor;
}
public List<DownloadFileTransfer> CurrentDownloads { get; private set; } = new();
public List<DownloadFileTransfer> CurrentDownloads { get; private set; } = [];
public List<FileTransfer> ForbiddenTransfers => _orchestrator.ForbiddenTransfers;
public bool IsDownloading => !CurrentDownloads.Any();
public static void MungeBuffer(Span<byte> buffer)
{
for (int i = 0; i < buffer.Length; ++i)
{
buffer[i] ^= 42;
}
}
public void CancelDownload()
{
CurrentDownloads.Clear();
@@ -66,14 +74,6 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
base.Dispose(disposing);
}
public static void MungeBuffer(Span<byte> buffer)
{
for (int i = 0; i < buffer.Length; ++i)
{
buffer[i] ^= 42;
}
}
private static byte MungeByte(int byteOrEof)
{
if (byteOrEof == -1)
@@ -86,8 +86,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
private static (string fileHash, long fileLengthBytes) ReadBlockFileHeader(FileStream fileBlockStream)
{
List<char> hashName = new();
List<char> fileLength = new();
List<char> hashName = [];
List<char> fileLength = [];
var separator = (char)MungeByte(fileBlockStream.ReadByte());
if (separator != '#') throw new InvalidDataException("Data is invalid, first char is not #");
@@ -177,8 +177,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
{
Logger.LogDebug("Download start: {id}", gameObjectHandler.Name);
List<DownloadFileDto> downloadFileInfoFromService = new();
downloadFileInfoFromService.AddRange(await FilesGetSizes(fileReplacement.Select(f => f.Hash).Distinct(StringComparer.Ordinal).ToList(), ct).ConfigureAwait(false));
List<DownloadFileDto> downloadFileInfoFromService =
[
.. await FilesGetSizes(fileReplacement.Select(f => f.Hash).Distinct(StringComparer.Ordinal).ToList(), ct).ConfigureAwait(false),
];
Logger.LogDebug("Files with size 0 or less: {files}", string.Join(", ", downloadFileInfoFromService.Where(f => f.Size <= 0).Select(f => f.Hash)));
@@ -219,9 +221,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
// let server predownload files
var requestIdResponse = await _orchestrator.SendRequestAsync(HttpMethod.Post, MareFiles.RequestEnqueueFullPath(fileGroup.First().DownloadUri),
fileGroup.Select(c => c.Hash), token).ConfigureAwait(false);
Logger.LogDebug("Sent request for {n} files on server {uri} with result {result}", fileGroup.Count(), fileGroup.First().DownloadUri, requestIdResponse.Content.ReadAsStringAsync().Result);
Logger.LogDebug("Sent request for {n} files on server {uri} with result {result}", fileGroup.Count(), fileGroup.First().DownloadUri,
await requestIdResponse.Content.ReadAsStringAsync(token).ConfigureAwait(false));
Guid requestId = Guid.Parse(requestIdResponse.Content.ReadAsStringAsync().Result.Trim('"'));
Guid requestId = Guid.Parse((await requestIdResponse.Content.ReadAsStringAsync().ConfigureAwait(false)).Trim('"'));
Logger.LogDebug("GUID {requestId} for {n} files on server {uri}", requestId, fileGroup.Count(), fileGroup.First().DownloadUri);
@@ -235,15 +238,15 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
{
try
{
if (!_downloadStatus.ContainsKey(fileGroup.Key)) return;
_downloadStatus[fileGroup.Key].TransferredBytes += bytesDownloaded;
if (!_downloadStatus.TryGetValue(fileGroup.Key, out FileDownloadStatus? value)) return;
value.TransferredBytes += bytesDownloaded;
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Could not set download progress");
}
});
await DownloadAndMungeFileHttpClient(fileGroup.Key, requestId, fileGroup.ToList(), blockFile, progress, token).ConfigureAwait(false);
await DownloadAndMungeFileHttpClient(fileGroup.Key, requestId, [.. fileGroup], blockFile, progress, token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
@@ -275,7 +278,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
try
{
Logger.LogDebug("Found file {file} with length {le}, decompressing download", fileHash, fileLengthBytes);
var fileExtension = fileReplacement.First(f => string.Equals(f.Hash, fileHash, StringComparison.OrdinalIgnoreCase)).GamePaths[0].Split(".").Last();
var fileExtension = fileReplacement.First(f => string.Equals(f.Hash, fileHash, StringComparison.OrdinalIgnoreCase)).GamePaths[0].Split(".")[^1];
byte[] compressedFileContent = new byte[fileLengthBytes];
_ = await fileBlockStream.ReadAsync(compressedFileContent, token).ConfigureAwait(false);
@@ -300,7 +303,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
finally
{
_orchestrator.ReleaseDownloadSlot();
fileBlockStream?.Dispose();
if (fileBlockStream != null)
await fileBlockStream.DisposeAsync().ConfigureAwait(false);
File.Delete(blockFile);
}
}).ConfigureAwait(false);
@@ -314,7 +318,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
{
if (!_orchestrator.IsInitialized) throw new InvalidOperationException("FileTransferManager is not initialized");
var response = await _orchestrator.SendRequestAsync(HttpMethod.Get, MareFiles.ServerFilesGetSizesFullPath(_orchestrator.FilesCdnUri!), hashes, ct).ConfigureAwait(false);
return await response.Content.ReadFromJsonAsync<List<DownloadFileDto>>(cancellationToken: ct).ConfigureAwait(false) ?? new List<DownloadFileDto>();
return await response.Content.ReadFromJsonAsync<List<DownloadFileDto>>(cancellationToken: ct).ConfigureAwait(false) ?? [];
}
private void PersistFileToStorage(string fileHash, string filePath)
@@ -322,7 +326,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
var fi = new FileInfo(filePath);
Func<DateTime> RandomDayInThePast()
{
DateTime start = new(1995, 1, 1);
DateTime start = new(1995, 1, 1, 1, 1, 1, DateTimeKind.Local);
Random gen = new();
int range = (DateTime.Today - start).Days;
return () => start.AddDays(gen.Next(range));
@@ -334,9 +338,9 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
try
{
var entry = _fileDbManager.CreateCacheEntry(filePath);
if (!string.Equals(entry?.Hash, fileHash, StringComparison.OrdinalIgnoreCase))
if (entry != null && !string.Equals(entry.Hash, fileHash, StringComparison.OrdinalIgnoreCase))
{
Logger.LogError("Hash mismatch after extracting, got {hash}, expected {expectedHash}, deleting file", entry?.Hash, fileHash);
Logger.LogError("Hash mismatch after extracting, got {hash}, expected {expectedHash}, deleting file", entry.Hash, fileHash);
File.Delete(filePath);
_fileDbManager.RemoveHashedFile(entry.Hash, entry.PrefixedFilePath);
}

33
MareSynchronos/WebAPI/Files/FileTransferOrchestrator.cs
View File

@@ -1,7 +1,7 @@
using MareSynchronos.MareConfiguration;
using MareSynchronos.Services.Mediator;
using MareSynchronos.Services.ServerConfiguration;
using MareSynchronos.WebAPI.Files.Models;
using MareSynchronos.WebAPI.SignalR;
using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
using System.Net.Http.Headers;
@@ -12,20 +12,23 @@ namespace MareSynchronos.WebAPI.Files;
public class FileTransferOrchestrator : DisposableMediatorSubscriberBase
{
private readonly ConcurrentDictionary<Guid, bool> _downloadReady = new();
private readonly HttpClient _httpClient;
private readonly MareConfigService _mareConfig;
private readonly object _semaphoreModificationLock = new();
private readonly ServerConfigurationManager _serverManager;
private readonly TokenProvider _tokenProvider;
private int _availableDownloadSlots;
private SemaphoreSlim _downloadSemaphore;
private readonly ConcurrentDictionary<Guid, bool> _downloadReady = new();
public FileTransferOrchestrator(ILogger<FileTransferOrchestrator> logger, MareConfigService mareConfig, ServerConfigurationManager serverManager, MareMediator mediator) : base(logger, mediator)
public FileTransferOrchestrator(ILogger<FileTransferOrchestrator> logger, MareConfigService mareConfig,
MareMediator mediator, TokenProvider tokenProvider) : base(logger, mediator)
{
_mareConfig = mareConfig;
_serverManager = serverManager;
_httpClient = new();
_httpClient.Timeout = TimeSpan.FromSeconds(3000);
_tokenProvider = tokenProvider;
_httpClient = new()
{
Timeout = TimeSpan.FromSeconds(3000)
};
var ver = Assembly.GetExecutingAssembly().GetName().Version;
_httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronos", ver!.Major + "." + ver!.Minor + "." + ver!.Build));
@@ -48,12 +51,12 @@ public class FileTransferOrchestrator : DisposableMediatorSubscriberBase
}
public Uri? FilesCdnUri { private set; get; }
public List<FileTransfer> ForbiddenTransfers { get; } = new();
public List<FileTransfer> ForbiddenTransfers { get; } = [];
public bool IsInitialized => FilesCdnUri != null;
public void ReleaseDownloadSlot()
public void ClearDownloadRequest(Guid guid)
{
_downloadSemaphore.Release();
_downloadReady.Remove(guid, out _);
}
public bool IsDownloadReady(Guid guid)
@@ -66,12 +69,12 @@ public class FileTransferOrchestrator : DisposableMediatorSubscriberBase
return false;
}
public void ClearDownloadRequest(Guid guid)
public void ReleaseDownloadSlot()
{
_downloadReady.Remove(guid, out _);
_downloadSemaphore.Release();
}
public async Task<HttpResponseMessage> SendRequestAsync(HttpMethod method, Uri uri,
public async Task<HttpResponseMessage> SendRequestAsync(HttpMethod method, Uri uri,
CancellationToken? ct = null, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead)
{
using var requestMessage = new HttpRequestMessage(method, uri);
@@ -109,10 +112,10 @@ public class FileTransferOrchestrator : DisposableMediatorSubscriberBase
await _downloadSemaphore.WaitAsync(token).ConfigureAwait(false);
}
private async Task<HttpResponseMessage> SendRequestInternalAsync(HttpRequestMessage requestMessage,
private async Task<HttpResponseMessage> SendRequestInternalAsync(HttpRequestMessage requestMessage,
CancellationToken? ct = null, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead)
{
requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _serverManager.GetToken());
requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await _tokenProvider.GetOrUpdateToken(ct!.Value).ConfigureAwait(false));
if (requestMessage.Content != null && requestMessage.Content is not StreamContent && requestMessage.Content is not ByteArrayContent)
{

14
MareSynchronos/WebAPI/Files/FileUploadManager.cs
View File

@@ -39,7 +39,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
});
}
public List<FileTransfer> CurrentUploads { get; } = new();
public List<FileTransfer> CurrentUploads { get; } = [];
public bool IsUploading => CurrentUploads.Count > 0;
public bool CancelUpload()
@@ -81,7 +81,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
foreach (var kvp in data.FileReplacements)
{
data.FileReplacements[kvp.Key].RemoveAll(i => _orchestrator.ForbiddenTransfers.Any(f => string.Equals(f.Hash, i.Hash, StringComparison.OrdinalIgnoreCase)));
data.FileReplacements[kvp.Key].RemoveAll(i => _orchestrator.ForbiddenTransfers.Exists(f => string.Equals(f.Hash, i.Hash, StringComparison.OrdinalIgnoreCase)));
}
return data;
@@ -102,7 +102,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
UIDs = uids
};
var response = await _orchestrator.SendRequestAsync(HttpMethod.Post, MareFiles.ServerFilesFilesSendFullPath(_orchestrator.FilesCdnUri!), filesSendDto, ct).ConfigureAwait(false);
return await response.Content.ReadFromJsonAsync<List<UploadFileDto>>(cancellationToken: ct).ConfigureAwait(false) ?? new List<UploadFileDto>();
return await response.Content.ReadFromJsonAsync<List<UploadFileDto>>(cancellationToken: ct).ConfigureAwait(false) ?? [];
}
private HashSet<string> GetUnverifiedFiles(CharacterData data)
@@ -152,7 +152,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
if (!_mareConfigService.Current.UseAlternativeFileUpload && ex is not OperationCanceledException)
{
Logger.LogWarning(ex, "[{hash}] Error during file upload, trying alternative file upload", fileHash);
await UploadFileStream(compressedFile, fileHash, true, uploadToken).ConfigureAwait(false);
await UploadFileStream(compressedFile, fileHash, munged: true, uploadToken).ConfigureAwait(false);
}
else
{
@@ -196,7 +196,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
unverifiedUploadHashes = unverifiedUploadHashes.Where(h => _fileDbManager.GetFileCacheByHash(h) != null).ToHashSet(StringComparer.Ordinal);
Logger.LogDebug("Verifying {count} files", unverifiedUploadHashes.Count);
var filesToUpload = await FilesSend(unverifiedUploadHashes.ToList(), visiblePlayers.Select(p => p.UID).ToList(), uploadToken).ConfigureAwait(false);
var filesToUpload = await FilesSend([.. unverifiedUploadHashes], visiblePlayers.Select(p => p.UID).ToList(), uploadToken).ConfigureAwait(false);
foreach (var file in filesToUpload.Where(f => !f.IsForbidden))
{
@@ -215,7 +215,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
foreach (var file in filesToUpload.Where(c => c.IsForbidden))
{
if (_orchestrator.ForbiddenTransfers.All(f => !string.Equals(f.Hash, file.Hash, StringComparison.Ordinal)))
if (_orchestrator.ForbiddenTransfers.TrueForAll(f => !string.Equals(f.Hash, file.Hash, StringComparison.Ordinal)))
{
_orchestrator.ForbiddenTransfers.Add(new UploadFileTransfer(file)
{
@@ -248,7 +248,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
Logger.LogDebug("Upload complete, compressed {size} to {compressed}", UiSharedService.ByteToString(totalSize), UiSharedService.ByteToString(compressedSize));
}
foreach (var file in unverifiedUploadHashes.Where(c => !CurrentUploads.Any(u => string.Equals(u.Hash, c, StringComparison.Ordinal))))
foreach (var file in unverifiedUploadHashes.Where(c => !CurrentUploads.Exists(u => string.Equals(u.Hash, c, StringComparison.Ordinal))))
{
_verifiedUploadedHashes[file] = DateTime.UtcNow;
}

10
MareSynchronos/WebAPI/Files/Models/DownloadFileTransfer.cs
View File

@@ -4,8 +4,11 @@ namespace MareSynchronos.WebAPI.Files.Models;
public class DownloadFileTransfer : FileTransfer
{
private DownloadFileDto Dto => (DownloadFileDto)TransferDto;
public DownloadFileTransfer(DownloadFileDto dto) : base(dto) { }
public DownloadFileTransfer(DownloadFileDto dto) : base(dto)
{
}
public override bool CanBeTransferred => Dto.FileExists && !Dto.IsForbidden && Dto.Size > 0;
public Uri DownloadUri => new(Dto.Url);
public override long Total
{
@@ -15,6 +18,5 @@ public class DownloadFileTransfer : FileTransfer
}
get => Dto.Size;
}
public override bool CanBeTransferred => Dto.FileExists && !Dto.IsForbidden && Dto.Size > 0;
private DownloadFileDto Dto => (DownloadFileDto)TransferDto;
}

2
MareSynchronos/WebAPI/Files/Models/DownloadStatus.cs
View File

@@ -7,4 +7,4 @@ public enum DownloadStatus
WaitingForQueue,
Downloading,
Decompressing
}
}

6
MareSynchronos/WebAPI/Files/Models/FileDownloadStatus.cs
View File

@@ -3,8 +3,8 @@
public class FileDownloadStatus
{
public DownloadStatus DownloadStatus { get; set; }
public int TotalFiles { get; set; }
public int TransferredFiles { get; set; }
public long TotalBytes { get; set; }
public int TotalFiles { get; set; }
public long TransferredBytes { get; set; }
}
public int TransferredFiles { get; set; }
}

10
MareSynchronos/WebAPI/Files/Models/FileTransfer.cs
View File

@@ -11,17 +11,17 @@ public abstract class FileTransfer
TransferDto = transferDto;
}
public virtual bool CanBeTransferred => !TransferDto.IsForbidden && (TransferDto is not DownloadFileDto dto || dto.FileExists);
public string ForbiddenBy => TransferDto.ForbiddenBy;
public long Transferred { get; set; } = 0;
public abstract long Total { get; set; }
public string Hash => TransferDto.Hash;
public bool IsForbidden => TransferDto.IsForbidden;
public bool IsInTransfer => Transferred != Total && Transferred > 0;
public bool IsTransferred => Transferred == Total;
public virtual bool CanBeTransferred => !TransferDto.IsForbidden && (TransferDto is not DownloadFileDto dto || dto.FileExists);
public bool IsForbidden => TransferDto.IsForbidden;
public abstract long Total { get; set; }
public long Transferred { get; set; } = 0;
public override string ToString()
{
return Hash;
}
}
}

9
MareSynchronos/WebAPI/Files/Models/UploadFileTransfer.cs
View File

@@ -4,7 +4,10 @@ namespace MareSynchronos.WebAPI.Files.Models;
public class UploadFileTransfer : FileTransfer
{
public UploadFileTransfer(UploadFileDto dto) : base(dto) { }
public override long Total { get; set; }
public UploadFileTransfer(UploadFileDto dto) : base(dto)
{
}
public string LocalFile { get; set; } = string.Empty;
}
public override long Total { get; set; }
}

2
MareSynchronos/WebAPI/Files/Models/UploadProgress.cs
View File

@@ -1,3 +1,3 @@
namespace MareSynchronos.WebAPI.Files.Models;
public record UploadProgress(long Uploaded, long Size);
public record UploadProgress(long Uploaded, long Size);