From 1ed831f36096e085fca2b7a2b6eac67839b65c08 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Sun, 2 Apr 2023 14:10:02 +0200 Subject: [PATCH] add alternative upload, ignore NIN pets, fix progress crashing --- .../Configurations/MareConfig.cs | 1 + MareSynchronos/MareSynchronos.csproj | 2 +- .../PlayerData/Handlers/GameObjectHandler.cs | 14 +++++-- MareSynchronos/Services/DalamudUtilService.cs | 4 +- .../Services/Mediator/MareMediator.cs | 10 ++--- MareSynchronos/Services/Mediator/Messages.cs | 2 +- MareSynchronos/UI/SettingsUi.cs | 8 ++++ .../WebAPI/Files/FileDownloadManager.cs | 37 ++++++++++------- .../WebAPI/Files/FileTransferOrchestrator.cs | 7 +++- .../WebAPI/Files/FileUploadManager.cs | 40 +++++++++++++++++-- 10 files changed, 92 insertions(+), 33 deletions(-) diff --git a/MareSynchronos/MareConfiguration/Configurations/MareConfig.cs b/MareSynchronos/MareConfiguration/Configurations/MareConfig.cs index 09714d5..4dabf69 100644 --- a/MareSynchronos/MareConfiguration/Configurations/MareConfig.cs +++ b/MareSynchronos/MareConfiguration/Configurations/MareConfig.cs @@ -40,6 +40,7 @@ public class MareConfig : IMareConfiguration public int TransferBarsHeight { get; set; } = 12; public bool TransferBarsShowText { get; set; } = true; public int TransferBarsWidth { get; set; } = 250; + public bool UseAlternativeFileUpload { get; set; } = false; public int Version { get; set; } = 1; public NotificationLocation WarningNotification { get; set; } = NotificationLocation.Both; } \ No newline at end of file diff --git a/MareSynchronos/MareSynchronos.csproj b/MareSynchronos/MareSynchronos.csproj index b8a9c2e..90c44bf 100644 --- a/MareSynchronos/MareSynchronos.csproj +++ b/MareSynchronos/MareSynchronos.csproj @@ -3,7 +3,7 @@ - 0.8.16 + 0.8.17 https://github.com/Penumbra-Sync/client diff --git a/MareSynchronos/PlayerData/Handlers/GameObjectHandler.cs b/MareSynchronos/PlayerData/Handlers/GameObjectHandler.cs index 9c6a5a8..f802bc2 100644 --- a/MareSynchronos/PlayerData/Handlers/GameObjectHandler.cs +++ b/MareSynchronos/PlayerData/Handlers/GameObjectHandler.cs @@ -268,11 +268,17 @@ public sealed class GameObjectHandler : DisposableMediatorSubscriberBase private unsafe bool IsBeingDrawn(IntPtr drawObj, IntPtr curPtr) { - Logger.LogTrace("IsBeingDrawn for ptr {curPtr} : {drawObj}", curPtr.ToString("X"), drawObj.ToString("X")); + Logger.LogTrace("IsBeingDrawn for {kind} ptr {curPtr} : {drawObj}", ObjectKind, curPtr.ToString("X"), drawObj.ToString("X")); + if (ObjectKind == ObjectKind.Player) + { + return drawObj == IntPtr.Zero + || (((CharacterBase*)drawObj)->HasModelInSlotLoaded != 0) + || (((CharacterBase*)drawObj)->HasModelFilesInSlotLoaded != 0) + || (((GameObject*)curPtr)->RenderFlags & 0b100000000000) == 0b100000000000; + } + return drawObj == IntPtr.Zero - || (((CharacterBase*)drawObj)->HasModelInSlotLoaded != 0) - || (((CharacterBase*)drawObj)->HasModelFilesInSlotLoaded != 0) - || (((GameObject*)curPtr)->RenderFlags & 0b100000000000) == 0b100000000000; + || ((GameObject*)curPtr)->RenderFlags != 0x0; } private void ZoneSwitchEnd() diff --git a/MareSynchronos/Services/DalamudUtilService.cs b/MareSynchronos/Services/DalamudUtilService.cs index 857f970..32251ac 100644 --- a/MareSynchronos/Services/DalamudUtilService.cs +++ b/MareSynchronos/Services/DalamudUtilService.cs @@ -26,6 +26,7 @@ public class DalamudUtilService : IHostedService private readonly MareMediator _mediator; private readonly ObjectTable _objectTable; private readonly PerformanceCollectorService _performanceCollector; + private readonly List ClassJobIdsIgnoredForPets = new() { 30 }; private uint? _classJobId = 0; private DateTime _delayedFrameworkUpdateCheck = DateTime.Now; private bool _sentBetweenAreas = false; @@ -107,6 +108,7 @@ public class DalamudUtilService : IHostedService public unsafe IntPtr GetPet(IntPtr? playerPointer = null) { + if (ClassJobIdsIgnoredForPets.Contains(_classJobId ?? 0)) return IntPtr.Zero; var mgr = CharacterManager.Instance(); playerPointer ??= PlayerPointer; if (playerPointer == IntPtr.Zero) return IntPtr.Zero; @@ -320,7 +322,7 @@ public class DalamudUtilService : IHostedService if (_classJobId != newclassJobId) { _classJobId = newclassJobId; - _mediator.Publish(new ClassJobChangedMessage()); + _mediator.Publish(new ClassJobChangedMessage(_classJobId)); } } diff --git a/MareSynchronos/Services/Mediator/MareMediator.cs b/MareSynchronos/Services/Mediator/MareMediator.cs index 3ac9a0a..8f2e540 100644 --- a/MareSynchronos/Services/Mediator/MareMediator.cs +++ b/MareSynchronos/Services/Mediator/MareMediator.cs @@ -102,16 +102,12 @@ public sealed class MareMediator : IDisposable { lock (_addRemoveLock) { - foreach (KeyValuePair> kvp in _subscriberDict) + foreach (Type kvp in _subscriberDict.Select(k => k.Key)) { - int unSubbed = _subscriberDict[kvp.Key]?.RemoveWhere(p => p.Subscriber == subscriber) ?? 0; + int unSubbed = _subscriberDict[kvp]?.RemoveWhere(p => p.Subscriber == subscriber) ?? 0; if (unSubbed > 0) { - _logger.LogDebug("{sub} unsubscribed from {msg}", subscriber.GetType().Name, kvp.Key.Name); - if (_subscriberDict[kvp.Key].Any()) - { - _logger.LogTrace("Remaining Subscribers: {item}", string.Join(", ", _subscriberDict[kvp.Key].Select(k => k.Subscriber.GetType().Name))); - } + _logger.LogDebug("{sub} unsubscribed from {msg}", subscriber.GetType().Name, kvp.Name); } } } diff --git a/MareSynchronos/Services/Mediator/Messages.cs b/MareSynchronos/Services/Mediator/Messages.cs index 0b06fed..2d993ff 100644 --- a/MareSynchronos/Services/Mediator/Messages.cs +++ b/MareSynchronos/Services/Mediator/Messages.cs @@ -16,7 +16,7 @@ public record OpenSettingsUiMessage : IMessage; public record DalamudLoginMessage : IMessage; public record DalamudLogoutMessage : IMessage; public record FrameworkUpdateMessage : IMessage; -public record ClassJobChangedMessage : IMessage; +public record ClassJobChangedMessage(uint? ClassJob) : IMessage; public record DelayedFrameworkUpdateMessage : IMessage; public record ZoneSwitchStartMessage : IMessage; public record ZoneSwitchEndMessage : IMessage; diff --git a/MareSynchronos/UI/SettingsUi.cs b/MareSynchronos/UI/SettingsUi.cs index cd96ebf..42629e2 100644 --- a/MareSynchronos/UI/SettingsUi.cs +++ b/MareSynchronos/UI/SettingsUi.cs @@ -132,12 +132,20 @@ public class SettingsUi : WindowMediatorSubscriberBase UiSharedService.FontText("Transfer Settings", _uiShared.UidFont); int maxParallelDownloads = _configService.Current.ParallelDownloads; + bool useAlternativeUpload = _configService.Current.UseAlternativeFileUpload; if (ImGui.SliderInt("Maximum Parallel Downloads", ref maxParallelDownloads, 1, 10)) { _configService.Current.ParallelDownloads = maxParallelDownloads; _configService.Save(); } + if (ImGui.Checkbox("Use Alternative Upload Method", ref useAlternativeUpload)) + { + _configService.Current.UseAlternativeFileUpload = useAlternativeUpload; + _configService.Save(); + } + UiSharedService.DrawHelpText("This will attempt to upload files in one go instead of a stream. Typically not necessary to enable. Use if you have upload issues."); + ImGui.Separator(); UiSharedService.FontText("Transfer UI", _uiShared.UidFont); diff --git a/MareSynchronos/WebAPI/Files/FileDownloadManager.cs b/MareSynchronos/WebAPI/Files/FileDownloadManager.cs index 992ecf0..67585f1 100644 --- a/MareSynchronos/WebAPI/Files/FileDownloadManager.cs +++ b/MareSynchronos/WebAPI/Files/FileDownloadManager.cs @@ -30,19 +30,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase _fileDbManager = fileCacheManager; } - public void Initialize() - { - Mediator.Subscribe(this, (msg) => - { - if (_downloadReady.ContainsKey(msg.RequestId)) - { - _downloadReady[msg.RequestId] = true; - } - }); - } - public List CurrentDownloads { get; private set; } = new(); + public List ForbiddenTransfers => _orchestrator.ForbiddenTransfers; + public bool IsDownloading => !CurrentDownloads.Any(); public void CancelDownload() @@ -69,6 +60,17 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase } } + public void Initialize() + { + Mediator.Subscribe(this, (msg) => + { + if (_downloadReady.ContainsKey(msg.RequestId)) + { + _downloadReady[msg.RequestId] = true; + } + }); + } + protected override void Dispose(bool disposing) { CancelDownload(); @@ -195,9 +197,16 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase var tempPath = _fileDbManager.GetCacheFilePath(file.Hash, isTemporaryFile: true); Progress progress = new((bytesDownloaded) => { - if (!_downloadStatus.ContainsKey(fileGroup.Key)) return; - _downloadStatus[fileGroup.Key].TransferredBytes += bytesDownloaded; - file.Transferred += bytesDownloaded; + try + { + if (!_downloadStatus.ContainsKey(fileGroup.Key)) return; + _downloadStatus[fileGroup.Key].TransferredBytes += bytesDownloaded; + file.Transferred += bytesDownloaded; + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Could not set download progress"); + } }); try diff --git a/MareSynchronos/WebAPI/Files/FileTransferOrchestrator.cs b/MareSynchronos/WebAPI/Files/FileTransferOrchestrator.cs index e169267..7bf997f 100644 --- a/MareSynchronos/WebAPI/Files/FileTransferOrchestrator.cs +++ b/MareSynchronos/WebAPI/Files/FileTransferOrchestrator.cs @@ -55,7 +55,10 @@ public class FileTransferOrchestrator : DisposableMediatorSubscriberBase public async Task SendRequestAsync(HttpMethod method, Uri uri, T content, CancellationToken ct) where T : class { using var requestMessage = new HttpRequestMessage(method, uri); - requestMessage.Content = JsonContent.Create(content); + if (content is not ByteArrayContent) + requestMessage.Content = JsonContent.Create(content); + else + requestMessage.Content = content as ByteArrayContent; return await SendRequestInternalAsync(requestMessage, ct).ConfigureAwait(false); } @@ -84,7 +87,7 @@ public class FileTransferOrchestrator : DisposableMediatorSubscriberBase { requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _serverManager.GetToken()); - if (requestMessage.Content != null && requestMessage.Content is not StreamContent) + if (requestMessage.Content != null && requestMessage.Content is not StreamContent && requestMessage.Content is not ByteArrayContent) { var content = await ((JsonContent)requestMessage.Content).ReadAsStringAsync().ConfigureAwait(false); Logger.LogDebug("Sending {method} to {uri} (Content: {content})", requestMessage.Method, requestMessage.RequestUri, content); diff --git a/MareSynchronos/WebAPI/Files/FileUploadManager.cs b/MareSynchronos/WebAPI/Files/FileUploadManager.cs index dc4b1b1..d27f352 100644 --- a/MareSynchronos/WebAPI/Files/FileUploadManager.cs +++ b/MareSynchronos/WebAPI/Files/FileUploadManager.cs @@ -3,6 +3,7 @@ using MareSynchronos.API.Data; using MareSynchronos.API.Dto.Files; using MareSynchronos.API.Routes; using MareSynchronos.FileCache; +using MareSynchronos.MareConfiguration; using MareSynchronos.Services.Mediator; using MareSynchronos.Services.ServerConfiguration; using MareSynchronos.UI; @@ -16,16 +17,19 @@ namespace MareSynchronos.WebAPI.Files; public sealed class FileUploadManager : DisposableMediatorSubscriberBase { private readonly FileCacheManager _fileDbManager; + private readonly MareConfigService _mareConfigService; private readonly FileTransferOrchestrator _orchestrator; private readonly ServerConfigurationManager _serverManager; private readonly Dictionary _verifiedUploadedHashes = new(StringComparer.Ordinal); private CancellationTokenSource? _uploadCancellationTokenSource = new(); public FileUploadManager(ILogger logger, MareMediator mediator, + MareConfigService mareConfigService, FileTransferOrchestrator orchestrator, FileCacheManager fileDbManager, ServerConfigurationManager serverManager) : base(logger, mediator) { + _mareConfigService = mareConfigService; _orchestrator = orchestrator; _fileDbManager = fileDbManager; _serverManager = serverManager; @@ -142,10 +146,39 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase { if (!_orchestrator.IsInitialized) throw new InvalidOperationException("FileTransferManager is not initialized"); - Logger.LogInformation("Uploading {file}, {size}", fileHash, UiSharedService.ByteToString(compressedFile.Length)); + Logger.LogInformation("[{hash}] Uploading {size}", fileHash, UiSharedService.ByteToString(compressedFile.Length)); if (uploadToken.IsCancellationRequested) return; + try + { + if (!_mareConfigService.Current.UseAlternativeFileUpload) + await UploadFileStream(compressedFile, fileHash, uploadToken).ConfigureAwait(false); + else + await UploadFileFull(compressedFile, fileHash, uploadToken).ConfigureAwait(false); + } + catch (Exception ex) + { + if (!_mareConfigService.Current.UseAlternativeFileUpload) + { + Logger.LogWarning(ex, "[{hash}] Error during file upload, trying alternative file upload", fileHash); + await UploadFileFull(compressedFile, fileHash, uploadToken).ConfigureAwait(false); + } + } + } + + private async Task UploadFileFull(byte[] compressedFile, string fileHash, CancellationToken uploadToken) + { + using var content = new ByteArrayContent(compressedFile); + content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + + var response = await _orchestrator.SendRequestAsync(HttpMethod.Post, MareFiles.ServerFilesUploadFullPath(_orchestrator.FilesCdnUri!, fileHash), content, uploadToken).ConfigureAwait(false); + Logger.LogDebug("[{hash}] Upload Status: {status}", fileHash, response.StatusCode); + CurrentUploads.Single(f => string.Equals(f.Hash, fileHash, StringComparison.Ordinal)).Transferred = compressedFile.Length; + } + + private async Task UploadFileStream(byte[] compressedFile, string fileHash, CancellationToken uploadToken) + { using var ms = new MemoryStream(compressedFile); Progress prog = new((prog) => @@ -156,7 +189,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); var response = await _orchestrator.SendRequestStreamAsync(HttpMethod.Post, MareFiles.ServerFilesUploadFullPath(_orchestrator.FilesCdnUri!, fileHash), streamContent, uploadToken).ConfigureAwait(false); - Logger.LogDebug("Upload Status: {status}", response.StatusCode); + Logger.LogDebug("[{hash}] Upload Status: {status}", fileHash, response.StatusCode); } private async Task UploadUnverifiedFiles(HashSet unverifiedUploadHashes, List visiblePlayers, CancellationToken uploadToken) @@ -199,9 +232,10 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase Task uploadTask = Task.CompletedTask; foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList()) { - Logger.LogDebug("Compressing {file}", file); + Logger.LogDebug("[{hash}] Compressing", file); var data = await GetCompressedFileData(file.Hash, uploadToken).ConfigureAwait(false); CurrentUploads.Single(e => string.Equals(e.Hash, data.Item1, StringComparison.Ordinal)).Total = data.Item2.Length; + Logger.LogDebug("[{hash}] Starting upload for {filePath}", data.Item1, _fileDbManager.GetFileCacheByHash(data.Item1)!.ResolvedFilepath); await uploadTask.ConfigureAwait(false); uploadTask = UploadFile(data.Item2, file.Hash, uploadToken); uploadToken.ThrowIfCancellationRequested();