t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add alternative upload, ignore NIN pets, fix progress crashing

Browse Source
This commit is contained in:
rootdarkarchon
2023-04-02 14:10:02 +02:00
parent 3fcd843a2d
commit 1ed831f360
10 changed files with 92 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
MareSynchronos/WebAPI/Files/FileDownloadManager.cs
View File

@@ -30,19 +30,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
_fileDbManager = fileCacheManager;
}
public void Initialize()
{
Mediator.Subscribe<DownloadReadyMessage>(this, (msg) =>
{
if (_downloadReady.ContainsKey(msg.RequestId))
{
_downloadReady[msg.RequestId] = true;
}
});
}
public List<DownloadFileTransfer> CurrentDownloads { get; private set; } = new();
public List<FileTransfer> ForbiddenTransfers => _orchestrator.ForbiddenTransfers;
public bool IsDownloading => !CurrentDownloads.Any();
public void CancelDownload()
@@ -69,6 +60,17 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
}
}
public void Initialize()
{
Mediator.Subscribe<DownloadReadyMessage>(this, (msg) =>
{
if (_downloadReady.ContainsKey(msg.RequestId))
{
_downloadReady[msg.RequestId] = true;
}
});
}
protected override void Dispose(bool disposing)
{
CancelDownload();
@@ -195,9 +197,16 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
var tempPath = _fileDbManager.GetCacheFilePath(file.Hash, isTemporaryFile: true);
Progress<long> progress = new((bytesDownloaded) =>
{
if (!_downloadStatus.ContainsKey(fileGroup.Key)) return;
_downloadStatus[fileGroup.Key].TransferredBytes += bytesDownloaded;
file.Transferred += bytesDownloaded;
try
{
if (!_downloadStatus.ContainsKey(fileGroup.Key)) return;
_downloadStatus[fileGroup.Key].TransferredBytes += bytesDownloaded;
file.Transferred += bytesDownloaded;
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Could not set download progress");
}
});
try

7
MareSynchronos/WebAPI/Files/FileTransferOrchestrator.cs
View File

@@ -55,7 +55,10 @@ public class FileTransferOrchestrator : DisposableMediatorSubscriberBase
public async Task<HttpResponseMessage> SendRequestAsync<T>(HttpMethod method, Uri uri, T content, CancellationToken ct) where T : class
{
using var requestMessage = new HttpRequestMessage(method, uri);
requestMessage.Content = JsonContent.Create(content);
if (content is not ByteArrayContent)
requestMessage.Content = JsonContent.Create(content);
else
requestMessage.Content = content as ByteArrayContent;
return await SendRequestInternalAsync(requestMessage, ct).ConfigureAwait(false);
}
@@ -84,7 +87,7 @@ public class FileTransferOrchestrator : DisposableMediatorSubscriberBase
{
requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _serverManager.GetToken());
if (requestMessage.Content != null && requestMessage.Content is not StreamContent)
if (requestMessage.Content != null && requestMessage.Content is not StreamContent && requestMessage.Content is not ByteArrayContent)
{
var content = await ((JsonContent)requestMessage.Content).ReadAsStringAsync().ConfigureAwait(false);
Logger.LogDebug("Sending {method} to {uri} (Content: {content})", requestMessage.Method, requestMessage.RequestUri, content);

40
MareSynchronos/WebAPI/Files/FileUploadManager.cs
View File

@@ -3,6 +3,7 @@ using MareSynchronos.API.Data;
using MareSynchronos.API.Dto.Files;
using MareSynchronos.API.Routes;
using MareSynchronos.FileCache;
using MareSynchronos.MareConfiguration;
using MareSynchronos.Services.Mediator;
using MareSynchronos.Services.ServerConfiguration;
using MareSynchronos.UI;
@@ -16,16 +17,19 @@ namespace MareSynchronos.WebAPI.Files;
public sealed class FileUploadManager : DisposableMediatorSubscriberBase
{
private readonly FileCacheManager _fileDbManager;
private readonly MareConfigService _mareConfigService;
private readonly FileTransferOrchestrator _orchestrator;
private readonly ServerConfigurationManager _serverManager;
private readonly Dictionary<string, DateTime> _verifiedUploadedHashes = new(StringComparer.Ordinal);
private CancellationTokenSource? _uploadCancellationTokenSource = new();
public FileUploadManager(ILogger<FileUploadManager> logger, MareMediator mediator,
MareConfigService mareConfigService,
FileTransferOrchestrator orchestrator,
FileCacheManager fileDbManager,
ServerConfigurationManager serverManager) : base(logger, mediator)
{
_mareConfigService = mareConfigService;
_orchestrator = orchestrator;
_fileDbManager = fileDbManager;
_serverManager = serverManager;
@@ -142,10 +146,39 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
{
if (!_orchestrator.IsInitialized) throw new InvalidOperationException("FileTransferManager is not initialized");
Logger.LogInformation("Uploading {file}, {size}", fileHash, UiSharedService.ByteToString(compressedFile.Length));
Logger.LogInformation("[{hash}] Uploading {size}", fileHash, UiSharedService.ByteToString(compressedFile.Length));
if (uploadToken.IsCancellationRequested) return;
try
{
if (!_mareConfigService.Current.UseAlternativeFileUpload)
await UploadFileStream(compressedFile, fileHash, uploadToken).ConfigureAwait(false);
else
await UploadFileFull(compressedFile, fileHash, uploadToken).ConfigureAwait(false);
}
catch (Exception ex)
{
if (!_mareConfigService.Current.UseAlternativeFileUpload)
{
Logger.LogWarning(ex, "[{hash}] Error during file upload, trying alternative file upload", fileHash);
await UploadFileFull(compressedFile, fileHash, uploadToken).ConfigureAwait(false);
}
}
}
private async Task UploadFileFull(byte[] compressedFile, string fileHash, CancellationToken uploadToken)
{
using var content = new ByteArrayContent(compressedFile);
content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
var response = await _orchestrator.SendRequestAsync(HttpMethod.Post, MareFiles.ServerFilesUploadFullPath(_orchestrator.FilesCdnUri!, fileHash), content, uploadToken).ConfigureAwait(false);
Logger.LogDebug("[{hash}] Upload Status: {status}", fileHash, response.StatusCode);
CurrentUploads.Single(f => string.Equals(f.Hash, fileHash, StringComparison.Ordinal)).Transferred = compressedFile.Length;
}
private async Task UploadFileStream(byte[] compressedFile, string fileHash, CancellationToken uploadToken)
{
using var ms = new MemoryStream(compressedFile);
Progress<UploadProgress> prog = new((prog) =>
@@ -156,7 +189,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
var response = await _orchestrator.SendRequestStreamAsync(HttpMethod.Post, MareFiles.ServerFilesUploadFullPath(_orchestrator.FilesCdnUri!, fileHash), streamContent, uploadToken).ConfigureAwait(false);
Logger.LogDebug("Upload Status: {status}", response.StatusCode);
Logger.LogDebug("[{hash}] Upload Status: {status}", fileHash, response.StatusCode);
}
private async Task UploadUnverifiedFiles(HashSet<string> unverifiedUploadHashes, List<UserData> visiblePlayers, CancellationToken uploadToken)
@@ -199,9 +232,10 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
Task uploadTask = Task.CompletedTask;
foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList())
{
Logger.LogDebug("Compressing {file}", file);
Logger.LogDebug("[{hash}] Compressing", file);
var data = await GetCompressedFileData(file.Hash, uploadToken).ConfigureAwait(false);
CurrentUploads.Single(e => string.Equals(e.Hash, data.Item1, StringComparison.Ordinal)).Total = data.Item2.Length;
Logger.LogDebug("[{hash}] Starting upload for {filePath}", data.Item1, _fileDbManager.GetFileCacheByHash(data.Item1)!.ResolvedFilepath);
await uploadTask.ConfigureAwait(false);
uploadTask = UploadFile(data.Item2, file.Hash, uploadToken);
uploadToken.ThrowIfCancellationRequested();