t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add MCDO (#80)

Browse Source 
* update api

* mcd online editor impl

* most of chara data hub impl

* some state of things

* some refactoring

* random bullshit go

* more nearby impl

* add uid to peformance msg

* cleanup/homogenization

* some split, update nuget packages

* migrate to latest packages where possible, remove lz4net, do some split, idk

* some polish and cleanup

* more cleanup, beautification, etc.

* fixes and cleanups

---------

Co-authored-by: Stanley Dimant <root.darkarchon@outlook.com>
This commit is contained in:
rootdarkarchon
2025-01-11 22:43:11 +01:00
committed by Loporrit
parent ad42b29a44
commit 30caedbf3a
44 changed files with 5128 additions and 486 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
MareSynchronos/WebAPI/Files/FileUploadManager.cs
View File

@@ -65,6 +65,43 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
await _orchestrator.SendRequestAsync(HttpMethod.Post, MareFiles.ServerFilesDeleteAllFullPath(_orchestrator.FilesCdnUri!)).ConfigureAwait(false);
}
public async Task<List<string>> UploadFiles(List<string> hashesToUpload, IProgress<string> progress, CancellationToken? ct = null)
{
Logger.LogDebug("Trying to upload files");
var filesPresentLocally = hashesToUpload.Where(h => _fileDbManager.GetFileCacheByHash(h) != null).ToHashSet(StringComparer.Ordinal);
var locallyMissingFiles = hashesToUpload.Except(filesPresentLocally, StringComparer.Ordinal).ToList();
if (locallyMissingFiles.Any())
{
return locallyMissingFiles;
}
progress.Report($"Starting upload for {filesPresentLocally.Count} files");
var filesToUpload = await FilesSend([.. filesPresentLocally], [], ct ?? CancellationToken.None).ConfigureAwait(false);
if (filesToUpload.Exists(f => f.IsForbidden))
{
return [.. filesToUpload.Where(f => f.IsForbidden).Select(f => f.Hash)];
}
Task uploadTask = Task.CompletedTask;
int i = 1;
foreach (var file in filesToUpload)
{
progress.Report($"Uploading file {i++}/{filesToUpload.Count}. Please wait until the upload is completed.");
Logger.LogDebug("[{hash}] Compressing", file);
var data = await _fileDbManager.GetCompressedFileData(file.Hash, ct ?? CancellationToken.None).ConfigureAwait(false);
Logger.LogDebug("[{hash}] Starting upload for {filePath}", data.Item1, _fileDbManager.GetFileCacheByHash(data.Item1)!.ResolvedFilepath);
await uploadTask.ConfigureAwait(false);
uploadTask = UploadFile(data.Item2, file.Hash, false, ct ?? CancellationToken.None);
(ct ?? CancellationToken.None).ThrowIfCancellationRequested();
}
await uploadTask.ConfigureAwait(false);
return [];
}
public async Task<CharacterData> UploadFiles(CharacterData data, List<UserData> visiblePlayers)
{
CancelUpload();
@@ -135,7 +172,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
_verifiedUploadedHashes.Clear();
}
private async Task UploadFile(byte[] compressedFile, string fileHash, CancellationToken uploadToken)
private async Task UploadFile(byte[] compressedFile, string fileHash, bool postProgress, CancellationToken uploadToken)
{
if (!_orchestrator.IsInitialized) throw new InvalidOperationException("FileTransferManager is not initialized");
@@ -145,7 +182,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
try
{
await UploadFileStream(compressedFile, fileHash, false, uploadToken).ConfigureAwait(false);
await UploadFileStream(compressedFile, fileHash, false, postProgress, uploadToken).ConfigureAwait(false);
_verifiedUploadedHashes[fileHash] = DateTime.UtcNow;
}
catch (Exception ex)
@@ -153,7 +190,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
if (false && ex is not OperationCanceledException)
{
Logger.LogWarning(ex, "[{hash}] Error during file upload, trying alternative file upload", fileHash);
await UploadFileStream(compressedFile, fileHash, munged: true, uploadToken).ConfigureAwait(false);
await UploadFileStream(compressedFile, fileHash, munged: true, postProgress, uploadToken).ConfigureAwait(false);
}
else
{
@@ -162,14 +199,14 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
}
}
private async Task UploadFileStream(byte[] compressedFile, string fileHash, bool munged, CancellationToken uploadToken)
private async Task UploadFileStream(byte[] compressedFile, string fileHash, bool munged, bool postProgress, CancellationToken uploadToken)
{
if (munged)
throw new NotImplementedException();
using var ms = new MemoryStream(compressedFile);
Progress<UploadProgress> prog = new((prog) =>
Progress<UploadProgress>? prog = !postProgress ? null : new((prog) =>
{
try
{
@@ -180,6 +217,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
Logger.LogWarning(ex, "[{hash}] Could not set upload progress", fileHash);
}
});
var streamContent = new ProgressableStreamContent(ms, prog);
streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
HttpResponseMessage response;
@@ -235,7 +273,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
CurrentUploads.Single(e => string.Equals(e.Hash, file.Hash, StringComparison.Ordinal)).Total = data.Item2.Length;
Logger.LogDebug("[{hash}] Starting upload for {filePath}", file.Hash, _fileDbManager.GetFileCacheByHash(file.Hash)!.ResolvedFilepath);
await uploadTask.ConfigureAwait(false);
uploadTask = UploadFile(data.Item2, file.Hash, uploadToken);
uploadTask = UploadFile(data.Item2, file.Hash, true, uploadToken);
uploadToken.ThrowIfCancellationRequested();
}

10
MareSynchronos/WebAPI/Files/Models/ProgressableStreamContent.cs
View File

@@ -6,16 +6,16 @@ public class ProgressableStreamContent : StreamContent
{
private const int _defaultBufferSize = 4096;
private readonly int _bufferSize;
private readonly IProgress<UploadProgress> _progress;
private readonly IProgress<UploadProgress>? _progress;
private readonly Stream _streamToWrite;
private bool _contentConsumed;
public ProgressableStreamContent(Stream streamToWrite, IProgress<UploadProgress> downloader)
public ProgressableStreamContent(Stream streamToWrite, IProgress<UploadProgress>? downloader)
: this(streamToWrite, _defaultBufferSize, downloader)
{
}
public ProgressableStreamContent(Stream streamToWrite, int bufferSize, IProgress<UploadProgress> progress)
public ProgressableStreamContent(Stream streamToWrite, int bufferSize, IProgress<UploadProgress>? progress)
: base(streamToWrite, bufferSize)
{
if (streamToWrite == null)
@@ -55,14 +55,14 @@ public class ProgressableStreamContent : StreamContent
{
while (true)
{
var length = _streamToWrite.Read(buffer, 0, buffer.Length);
var length = await _streamToWrite.ReadAsync(buffer).ConfigureAwait(false);
if (length <= 0)
{
break;
}
uploaded += length;
_progress.Report(new UploadProgress(uploaded, size));
_progress?.Report(new UploadProgress(uploaded, size));
await stream.WriteAsync(buffer.AsMemory(0, length)).ConfigureAwait(false);
}
}

119
MareSynchronos/WebAPI/SignalR/ApiController.Functions.CharaData.cs Normal file
View File

@@ -0,0 +1,119 @@
using MareSynchronos.API.Dto.CharaData;
using Microsoft.AspNetCore.SignalR.Client;
using Microsoft.Extensions.Logging;
namespace MareSynchronos.WebAPI;
public partial class ApiController
{
public async Task<CharaDataFullDto?> CharaDataCreate()
{
if (!IsConnected) return null;
try
{
Logger.LogDebug("Creating new Character Data");
return await _mareHub!.InvokeAsync<CharaDataFullDto?>(nameof(CharaDataCreate)).ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Failed to create new character data");
return null;
}
}
public async Task<CharaDataFullDto?> CharaDataUpdate(CharaDataUpdateDto updateDto)
{
if (!IsConnected) return null;
try
{
Logger.LogDebug("Updating chara data for {id}", updateDto.Id);
return await _mareHub!.InvokeAsync<CharaDataFullDto?>(nameof(CharaDataUpdate), updateDto).ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Failed to update chara data for {id}", updateDto.Id);
return null;
}
}
public async Task<bool> CharaDataDelete(string id)
{
if (!IsConnected) return false;
try
{
Logger.LogDebug("Deleting chara data for {id}", id);
return await _mareHub!.InvokeAsync<bool>(nameof(CharaDataDelete), id).ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Failed to delete chara data for {id}", id);
return false;
}
}
public async Task<CharaDataMetaInfoDto?> CharaDataGetMetainfo(string id)
{
if (!IsConnected) return null;
try
{
Logger.LogDebug("Getting metainfo for chara data {id}", id);
return await _mareHub!.InvokeAsync<CharaDataMetaInfoDto?>(nameof(CharaDataGetMetainfo), id).ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Failed to get meta info for chara data {id}", id);
return null;
}
}
public async Task<List<CharaDataFullDto>> CharaDataGetOwn()
{
if (!IsConnected) return [];
try
{
Logger.LogDebug("Getting all own chara data");
return await _mareHub!.InvokeAsync<List<CharaDataFullDto>>(nameof(CharaDataGetOwn)).ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Failed to get own chara data");
return [];
}
}
public async Task<List<CharaDataMetaInfoDto>> CharaDataGetShared()
{
if (!IsConnected) return [];
try
{
Logger.LogDebug("Getting all own chara data");
return await _mareHub!.InvokeAsync<List<CharaDataMetaInfoDto>>(nameof(CharaDataGetShared)).ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Failed to get shared chara data");
return [];
}
}
public async Task<CharaDataDownloadDto?> CharaDataDownload(string id)
{
if (!IsConnected) return null;
try
{
Logger.LogDebug("Getting download chara data for {id}", id);
return await _mareHub!.InvokeAsync<CharaDataDownloadDto>(nameof(CharaDataDownload), id).ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Failed to get download chara data for {id}", id);
return null;
}
}
}