From 5a16a15e8b7a61f1dce3564065a2ebfe113ebb95 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Sun, 15 Jan 2023 16:00:10 +0100 Subject: [PATCH] Download rework (#36) * handle download ready from signalr * add cancellation to the server * adjust queue for removal and cancellation * adjust api to main Co-authored-by: rootdarkarchon --- MareAPI | 2 +- MareSynchronos/Managers/CachedPlayer.cs | 4 +- MareSynchronos/UI/CompactUI.cs | 3 +- .../WebAPI/ApIController.Functions.Files.cs | 123 ++++++++---------- .../ApiController.Functions.Callbacks.cs | 13 ++ MareSynchronos/WebAPI/ApiController.cs | 1 + 6 files changed, 74 insertions(+), 72 deletions(-) diff --git a/MareAPI b/MareAPI index f1c0fc7..5ac8a75 160000 --- a/MareAPI +++ b/MareAPI @@ -1 +1 @@ -Subproject commit f1c0fc76a9195fba84f8e709bc9297e97d7c9214 +Subproject commit 5ac8a753dd1e54ab9ab321119ece462562981e82 diff --git a/MareSynchronos/Managers/CachedPlayer.cs b/MareSynchronos/Managers/CachedPlayer.cs index 5d0bf73..13ef9a4 100644 --- a/MareSynchronos/Managers/CachedPlayer.cs +++ b/MareSynchronos/Managers/CachedPlayer.cs @@ -168,8 +168,10 @@ public class CachedPlayer } _downloadCancellationTokenSource?.Cancel(); + _downloadCancellationTokenSource?.Dispose(); _downloadCancellationTokenSource = new CancellationTokenSource(); var downloadToken = _downloadCancellationTokenSource.Token; + var downloadId = _apiController.GetDownloadId(); Task.Run(async () => { @@ -218,8 +220,6 @@ public class CachedPlayer Logger.Debug("Download Task was cancelled"); _apiController.CancelDownload(downloadId); }); - - _downloadCancellationTokenSource = null; } private List TryCalculateModdedDictionary(out Dictionary moddedDictionary) diff --git a/MareSynchronos/UI/CompactUI.cs b/MareSynchronos/UI/CompactUI.cs index 75bec50..de10068 100644 --- a/MareSynchronos/UI/CompactUI.cs +++ b/MareSynchronos/UI/CompactUI.cs @@ -63,7 +63,8 @@ public class CompactUi : Window, IDisposable this.WindowName = "Mare Synchronos " + dateTime + "###MareSynchronosMainUI"; Toggle(); #else - this.WindowName = "Mare Synchronos " + Assembly.GetExecutingAssembly().GetName().Version + "###MareSynchronosMainUI"; + var ver = Assembly.GetExecutingAssembly().GetName().Version; + this.WindowName = "Mare Synchronos " + ver.Major + "." + ver.Minor + "." + ver.Revision + "###MareSynchronosMainUI"; #endif Logger.Verbose("Creating " + nameof(CompactUi)); diff --git a/MareSynchronos/WebAPI/ApIController.Functions.Files.cs b/MareSynchronos/WebAPI/ApIController.Functions.Files.cs index 9425215..72e133d 100644 --- a/MareSynchronos/WebAPI/ApIController.Functions.Files.cs +++ b/MareSynchronos/WebAPI/ApIController.Functions.Files.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -15,13 +16,13 @@ using MareSynchronos.API; using MareSynchronos.Utils; using MareSynchronos.WebAPI.Utils; using Microsoft.AspNetCore.SignalR.Client; -using Newtonsoft.Json; namespace MareSynchronos.WebAPI; public partial class ApiController { private readonly Dictionary _verifiedUploadedHashes; + private readonly ConcurrentDictionary _downloadReady = new(); private int _downloadId = 0; public async void CancelUpload() @@ -46,94 +47,80 @@ public partial class ApiController await _mareHub!.SendAsync(nameof(FilesDeleteAll)).ConfigureAwait(false); } - private async Task GetQueueRequestDto(DownloadFileTransfer downloadFileTransfer) + private async Task GetQueueRequest(DownloadFileTransfer downloadFileTransfer, CancellationToken ct) { - var response = await SendRequestAsync(HttpMethod.Get, MareFiles.RequestRequestFileFullPath(downloadFileTransfer.DownloadUri, downloadFileTransfer.Hash)).ConfigureAwait(false); - return JsonConvert.DeserializeObject(await response.Content.ReadAsStringAsync().ConfigureAwait(false))!; + var response = await SendRequestAsync(HttpMethod.Get, MareFiles.RequestRequestFileFullPath(downloadFileTransfer.DownloadUri, downloadFileTransfer.Hash), ct: ct).ConfigureAwait(false); + var responseString = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); + var requestId = Guid.Parse(responseString.Trim('"')); + if (!_downloadReady.ContainsKey(requestId)) + { + _downloadReady[requestId] = false; + } + return requestId; } - private async Task WaitForQueue(DownloadFileTransfer fileTransfer, Guid requestId, CancellationToken ct) + private async Task WaitForDownloadReady(DownloadFileTransfer downloadFileTransfer, Guid requestId, CancellationToken ct) { - while (!ct.IsCancellationRequested) + bool alreadyCancelled = false; + try + { + while (!ct.IsCancellationRequested && _downloadReady.TryGetValue(requestId, out bool isReady) && !isReady) + { + Logger.Verbose($"Waiting for {requestId} to become ready for download"); + await Task.Delay(250, ct).ConfigureAwait(false); + } + + Logger.Debug($"Download {requestId} ready"); + } + catch (TaskCanceledException) { - await Task.Delay(500, ct).ConfigureAwait(false); - var queueResponse = await SendRequestAsync(HttpMethod.Get, MareFiles.RequestCheckQueueFullPath(fileTransfer.DownloadUri, requestId)).ConfigureAwait(false); try { - queueResponse.EnsureSuccessStatusCode(); - Logger.Debug($"Starting download for file {fileTransfer.Hash} ({requestId})"); - break; + await SendRequestAsync(HttpMethod.Get, MareFiles.RequestCancelFullPath(downloadFileTransfer.DownloadUri, requestId), ct).ConfigureAwait(false); + alreadyCancelled = true; } - catch (HttpRequestException ex) - { - switch (ex.StatusCode) - { - case HttpStatusCode.Conflict: - Logger.Debug($"In queue for file {fileTransfer.Hash} ({requestId})"); - // still in queue - break; - case HttpStatusCode.BadRequest: - // rerequest queue - Logger.Debug($"Rerequesting {fileTransfer.Hash}"); - var dto = await GetQueueRequestDto(fileTransfer).ConfigureAwait(false); - requestId = dto.RequestId; - break; - default: - Logger.Warn($"Unclear response from server: {fileTransfer.Hash} ({requestId}): {ex.StatusCode}"); - break; - } + catch { } - await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false); - } + throw; + } + finally + { + if (ct.IsCancellationRequested && !alreadyCancelled) + { + try + { + await SendRequestAsync(HttpMethod.Get, MareFiles.RequestCancelFullPath(downloadFileTransfer.DownloadUri, requestId), ct).ConfigureAwait(false); + } + catch { } + } + _downloadReady.Remove(requestId, out _); } - - return requestId; } private async Task DownloadFileHttpClient(DownloadFileTransfer fileTransfer, IProgress progress, CancellationToken ct) { - var queueRequest = await GetQueueRequestDto(fileTransfer).ConfigureAwait(false); + var requestId = await GetQueueRequest(fileTransfer, ct).ConfigureAwait(false); - Logger.Debug($"GUID {queueRequest.RequestId} for file {fileTransfer.Hash}, queue status {queueRequest.QueueStatus}"); + Logger.Debug($"GUID {requestId} for file {fileTransfer.Hash} on server {fileTransfer.DownloadUri}"); - var requestId = queueRequest.QueueStatus == QueueStatus.Ready - ? queueRequest.RequestId - : await WaitForQueue(fileTransfer, queueRequest.RequestId, ct).ConfigureAwait(false); - - int attempts = 1; - bool failed = true; - const int maxAttempts = 16; + await WaitForDownloadReady(fileTransfer, requestId, ct).ConfigureAwait(false); HttpResponseMessage response = null!; - HttpStatusCode? lastError = HttpStatusCode.OK; - var requestUrl = MareFiles.CacheGetFullPath(fileTransfer.DownloadUri, requestId); Logger.Debug($"Downloading {requestUrl} for file {fileTransfer.Hash}"); - while (failed && attempts < maxAttempts && !ct.IsCancellationRequested) + try { - try - { - response = await SendRequestAsync(HttpMethod.Get, requestUrl, ct: ct).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - failed = false; - } - catch (HttpRequestException ex) - { - Logger.Warn($"Attempt {attempts}: Error during download of {requestUrl}, HttpStatusCode: {ex.StatusCode}"); - lastError = ex.StatusCode; - if (ex.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Unauthorized) - { - break; - } - attempts++; - await Task.Delay(TimeSpan.FromSeconds(new Random().NextDouble() * 2), ct).ConfigureAwait(false); - } + response = await SendRequestAsync(HttpMethod.Get, requestUrl, ct: ct).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); } - - if (failed) + catch (HttpRequestException ex) { - throw new Exception($"Http error {lastError} after {maxAttempts} attempts (cancelled: {ct.IsCancellationRequested}): {requestUrl}"); + Logger.Warn($"Error during download of {requestUrl}, HttpStatusCode: {ex.StatusCode}"); + if (ex.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Unauthorized) + { + throw new Exception($"Http error {ex.StatusCode} (cancelled: {ct.IsCancellationRequested}): {requestUrl}", ex); + } } var fileName = Path.GetTempFileName(); @@ -259,16 +246,16 @@ public partial class ApiController if (token.IsCancellationRequested) { File.Delete(tempFile); - Logger.Debug("Detected cancellation, removing " + currentDownloadId); + Logger.Debug("Detetokened cancellation, removing " + currentDownloadId); CancelDownload(currentDownloadId); return; } var tempFileData = await File.ReadAllBytesAsync(tempFile, token).ConfigureAwait(false); - var extractedFile = LZ4Codec.Unwrap(tempFileData); + var extratokenedFile = LZ4Codec.Unwrap(tempFileData); File.Delete(tempFile); var filePath = Path.Combine(_pluginConfiguration.CacheFolder, file.Hash); - await File.WriteAllBytesAsync(filePath, extractedFile, token).ConfigureAwait(false); + await File.WriteAllBytesAsync(filePath, extratokenedFile, token).ConfigureAwait(false); var fi = new FileInfo(filePath); Func RandomDayInThePast() { diff --git a/MareSynchronos/WebAPI/ApiController.Functions.Callbacks.cs b/MareSynchronos/WebAPI/ApiController.Functions.Callbacks.cs index 8f2bd71..f4be627 100644 --- a/MareSynchronos/WebAPI/ApiController.Functions.Callbacks.cs +++ b/MareSynchronos/WebAPI/ApiController.Functions.Callbacks.cs @@ -84,6 +84,12 @@ public partial class ApiController _mareHub!.On(nameof(Client_ReceiveServerMessage), act); } + public void OnDownloadReady(Action act) + { + if (_initialized) return; + _mareHub!.On(nameof(Client_DownloadReady), act); + } + public Task Client_UserUpdateClientPairs(ClientPairDto dto) { var entry = PairedClients.SingleOrDefault(e => string.Equals(e.OtherUID, dto.OtherUID, System.StringComparison.Ordinal)); @@ -243,4 +249,11 @@ public partial class ApiController return Task.CompletedTask; } + + public Task Client_DownloadReady(Guid requestId) + { + Logger.Debug($"Server sent {requestId} ready"); + _downloadReady[requestId] = true; + return Task.CompletedTask; + } } diff --git a/MareSynchronos/WebAPI/ApiController.cs b/MareSynchronos/WebAPI/ApiController.cs index cc17700..b48cdc6 100644 --- a/MareSynchronos/WebAPI/ApiController.cs +++ b/MareSynchronos/WebAPI/ApiController.cs @@ -299,6 +299,7 @@ public partial class ApiController : IDisposable, IMareHubClient OnUserReceiveCharacterData((dto, ident) => Client_UserReceiveCharacterData(dto, ident)); OnGroupChange(async (dto) => await Client_GroupChange(dto).ConfigureAwait(false)); OnGroupUserChange((dto) => Client_GroupUserChange(dto)); + OnDownloadReady((guid) => Client_DownloadReady(guid)); OnAdminForcedReconnect(() => Client_AdminForcedReconnect());