From 3634c06ee59e3616473cb714d7a0bdbc09cbd393 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Sun, 5 Feb 2023 15:52:27 +0100 Subject: [PATCH] fixes for pet handling, uploads, clear storage button, remove button while reconnecting, put paused to online/paused, put visible to online --- .../Factories/CharacterDataFactory.cs | 31 +-- MareSynchronos/FileCache/FileCacheManager.cs | 4 +- .../Managers/CacheCreationService.cs | 15 +- .../Managers/TransientResourceManager.cs | 1 + MareSynchronos/MareSynchronos.csproj | 2 +- MareSynchronos/Mediator/Messages.cs | 1 + MareSynchronos/Models/CharacterData.cs | 19 +- MareSynchronos/Models/FileReplacement.cs | 6 +- MareSynchronos/Models/GameObjectHandler.cs | 33 ++- MareSynchronos/UI/CompactUI.cs | 22 +- MareSynchronos/UI/Components/PairGroupsUi.cs | 6 +- MareSynchronos/UI/SettingsUi.cs | 23 +- MareSynchronos/Utils/DalamudUtil.cs | 6 +- .../Utils/FileReplacementComparer.cs | 4 +- MareSynchronos/Utils/Logger.cs | 8 +- .../WebAPI/ApIController.Functions.Files.cs | 215 ++++++++++-------- MareSynchronos/WebAPI/ApiController.cs | 24 +- 17 files changed, 241 insertions(+), 179 deletions(-) diff --git a/MareSynchronos/Factories/CharacterDataFactory.cs b/MareSynchronos/Factories/CharacterDataFactory.cs index b973567..29a714b 100644 --- a/MareSynchronos/Factories/CharacterDataFactory.cs +++ b/MareSynchronos/Factories/CharacterDataFactory.cs @@ -253,19 +253,9 @@ public class CharacterDataFactory AddPlayerSpecificReplacements(objectKind, charaPointer, human); } - if (objectKind == ObjectKind.Pet) - { - foreach (var item in previousData.FileReplacements[objectKind]) - { - _transientResourceManager.AddSemiTransientResource(objectKind, item.GamePaths.First()); - } - - previousData.FileReplacements[objectKind].Clear(); - } - - Dictionary> resolvedPaths = GetFileReplacementsFromPaths(); - previousData.FileReplacements[objectKind] = new HashSet(resolvedPaths.Select(c => new FileReplacement(c.Value, c.Key, _fileCacheManager))); + previousData.FileReplacements[objectKind] = new HashSet(resolvedPaths.Select(c => new FileReplacement(c.Value, c.Key, _fileCacheManager)), FileReplacementComparer.Instance) + .Where(p => p.HasFileReplacement).ToHashSet(); previousData.ManipulationString = _ipcManager.PenumbraGetMetaManipulations(); previousData.GlamourerString[objectKind] = _ipcManager.GlamourerGetCharacterCustomization(charaPointer); @@ -274,11 +264,19 @@ public class CharacterDataFactory previousData.PalettePlusPalette = _ipcManager.PalettePlusBuildPalette(); Logger.Debug("== Static Replacements =="); - foreach (var item in previousData.FileReplacements[objectKind]) + foreach (var item in previousData.FileReplacements[objectKind].Where(i => i.HasFileReplacement).OrderBy(i => i.GamePaths.First(), StringComparer.OrdinalIgnoreCase)) { Logger.Debug(item.ToString()); } + if (objectKind == ObjectKind.Pet) + { + foreach (var item in previousData.FileReplacements[objectKind].Where(i => i.HasFileReplacement).SelectMany(p => p.GamePaths)) + { + _transientResourceManager.AddSemiTransientResource(objectKind, item); + } + } + Logger.Debug("Handling transient update for " + objectKind); _transientResourceManager.ClearTransientPaths(charaPointer, previousData.FileReplacements[objectKind].SelectMany(c => c.GamePaths).ToList()); @@ -289,7 +287,7 @@ public class CharacterDataFactory var resolvedTransientPaths = GetFileReplacementsFromPaths(); Logger.Debug("== Transient Replacements =="); - foreach (var replacement in resolvedTransientPaths.Select(c => new FileReplacement(c.Value, c.Key, _fileCacheManager))) + foreach (var replacement in resolvedTransientPaths.Select(c => new FileReplacement(c.Value, c.Key, _fileCacheManager)).OrderBy(f => f.ResolvedPath, StringComparer.Ordinal)) { Logger.Debug(replacement.ToString()); previousData.FileReplacements[objectKind].Add(replacement); @@ -297,7 +295,10 @@ public class CharacterDataFactory _transientResourceManager.CleanSemiTransientResources(objectKind, previousData.FileReplacements[objectKind].ToList()); - + foreach (var item in previousData.FileReplacements) + { + previousData.FileReplacements[item.Key] = new HashSet(item.Value.Where(v => v.HasFileReplacement).OrderBy(v => v.ResolvedPath, StringComparer.Ordinal), FileReplacementComparer.Instance); + } st.Stop(); Logger.Info("Building character data for " + objectKind + " took " + st.ElapsedMilliseconds + "ms"); diff --git a/MareSynchronos/FileCache/FileCacheManager.cs b/MareSynchronos/FileCache/FileCacheManager.cs index 2a39c7d..8b79373 100644 --- a/MareSynchronos/FileCache/FileCacheManager.cs +++ b/MareSynchronos/FileCache/FileCacheManager.cs @@ -85,7 +85,9 @@ public class FileCacheManager : IDisposable { if (_fileCaches.Any(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal))) { - return GetValidatedFileCache(_fileCaches.FirstOrDefault(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal)).Value); + return GetValidatedFileCache(_fileCaches.Where(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal)) + .OrderByDescending(f => f.Value.PrefixedFilePath.Length) + .FirstOrDefault(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal)).Value); } return null; diff --git a/MareSynchronos/Managers/CacheCreationService.cs b/MareSynchronos/Managers/CacheCreationService.cs index 7991883..aaeb422 100644 --- a/MareSynchronos/Managers/CacheCreationService.cs +++ b/MareSynchronos/Managers/CacheCreationService.cs @@ -26,6 +26,14 @@ public class CacheCreationService : MediatorSubscriberBase, IDisposable var actualMsg = (CreateCacheForObjectMessage)msg; _cachesToCreate[actualMsg.ObjectToCreateFor.ObjectKind] = actualMsg.ObjectToCreateFor; }); + Mediator.Subscribe(this, (msg) => + { + var actualMsg = (ClearCacheForObjectMessage)msg; + _lastCreatedData.FileReplacements.Remove(actualMsg.ObjectToCreateFor.ObjectKind); + _lastCreatedData.GlamourerString.Remove(actualMsg.ObjectToCreateFor.ObjectKind); + Mediator.Publish(new CharacterDataCreatedMessage(_lastCreatedData)); + }); + Mediator.Subscribe(this, (msg) => UpdatePointers()); Mediator.Subscribe(this, (msg) => ProcessCacheCreation()); Mediator.Subscribe(this, (msg) => CustomizePlusChanged((CustomizePlusMessage)msg)); Mediator.Subscribe(this, (msg) => HeelsOffsetChanged((HeelsOffsetMessage)msg)); @@ -78,6 +86,11 @@ public class CacheCreationService : MediatorSubscriberBase, IDisposable } } + private void UpdatePointers() + { + Mediator.Publish(new PlayerRelatedObjectPointerUpdateMessage(_playerRelatedObjects.Select(f => f.CurrentAddress).ToArray())); + } + private void ProcessCacheCreation() { if (_cachesToCreate.Any() && (_cacheCreationTask?.IsCompleted ?? true)) @@ -109,8 +122,6 @@ public class CacheCreationService : MediatorSubscriberBase, IDisposable { Logger.Debug("Cache Creation stored until previous creation finished"); } - - Mediator.Publish(new PlayerRelatedObjectPointerUpdateMessage(_playerRelatedObjects.Select(f => f.CurrentAddress).ToArray())); } public override void Dispose() diff --git a/MareSynchronos/Managers/TransientResourceManager.cs b/MareSynchronos/Managers/TransientResourceManager.cs index a129f3f..4c183f2 100644 --- a/MareSynchronos/Managers/TransientResourceManager.cs +++ b/MareSynchronos/Managers/TransientResourceManager.cs @@ -133,6 +133,7 @@ public class TransientResourceManager : MediatorSubscriberBase, IDisposable } if (!PlayerRelatedPointers.Contains(gameObject)) { + Logger.Debug("Got resource " + gamePath + " for ptr " + gameObject.ToString("X")); return; } diff --git a/MareSynchronos/MareSynchronos.csproj b/MareSynchronos/MareSynchronos.csproj index 8656a3a..8f292bb 100644 --- a/MareSynchronos/MareSynchronos.csproj +++ b/MareSynchronos/MareSynchronos.csproj @@ -3,7 +3,7 @@ - 0.7.19 + 0.7.20 https://github.com/Penumbra-Sync/client diff --git a/MareSynchronos/Mediator/Messages.cs b/MareSynchronos/Mediator/Messages.cs index 45094d9..676a9fe 100644 --- a/MareSynchronos/Mediator/Messages.cs +++ b/MareSynchronos/Mediator/Messages.cs @@ -37,6 +37,7 @@ public record ResumeScanMessage(string Source) : IMessage; public record NotificationMessage (string Title, string Message, NotificationType Type, uint TimeShownOnScreen = 3000) : IMessage; public record CreateCacheForObjectMessage(GameObjectHandler ObjectToCreateFor) : IMessage; +public record ClearCacheForObjectMessage(GameObjectHandler ObjectToCreateFor) : IMessage; public record CharacterDataCreatedMessage(CharacterData CharacterData) : IMessage; #pragma warning restore MA0048 // File name must match type name \ No newline at end of file diff --git a/MareSynchronos/Models/CharacterData.cs b/MareSynchronos/Models/CharacterData.cs index 8f6c948..32c482e 100644 --- a/MareSynchronos/Models/CharacterData.cs +++ b/MareSynchronos/Models/CharacterData.cs @@ -28,23 +28,6 @@ public class CharacterData [JsonProperty] public string PalettePlusPalette { get; set; } = string.Empty; - public void AddFileReplacement(ObjectKind objectKind, FileReplacement fileReplacement) - { - if (!fileReplacement.HasFileReplacement) return; - - if (!FileReplacements.ContainsKey(objectKind)) FileReplacements.Add(objectKind, new HashSet()); - - var existingReplacement = FileReplacements[objectKind].SingleOrDefault(f => string.Equals(f.ResolvedPath, fileReplacement.ResolvedPath, StringComparison.OrdinalIgnoreCase)); - if (existingReplacement != null) - { - existingReplacement.GamePaths.AddRange(fileReplacement.GamePaths.Where(e => !existingReplacement.GamePaths.Contains(e, StringComparer.OrdinalIgnoreCase))); - } - else - { - FileReplacements[objectKind].Add(fileReplacement); - } - } - public API.Data.CharacterData ToAPI() { var fileReplacements = FileReplacements.ToDictionary(k => k.Key, k => k.Value.Where(f => f.HasFileReplacement && !f.IsFileSwap).GroupBy(f => f.Hash, StringComparer.OrdinalIgnoreCase).Select(g => @@ -76,7 +59,7 @@ public class CharacterData public override string ToString() { StringBuilder stringBuilder = new(); - foreach (var fileReplacement in FileReplacements.SelectMany(k => k.Value).OrderBy(a => a.GamePaths[0], StringComparer.Ordinal)) + foreach (var fileReplacement in FileReplacements.SelectMany(k => k.Value).OrderBy(a => a.GamePaths.First(), StringComparer.Ordinal)) { stringBuilder.AppendLine(fileReplacement.ToString()); } diff --git a/MareSynchronos/Models/FileReplacement.cs b/MareSynchronos/Models/FileReplacement.cs index cb6d97f..e9be801 100644 --- a/MareSynchronos/Models/FileReplacement.cs +++ b/MareSynchronos/Models/FileReplacement.cs @@ -8,18 +8,18 @@ public class FileReplacement { public FileReplacement(List gamePaths, string filePath, FileCacheManager fileDbManager) { - GamePaths = gamePaths.Select(g => g.Replace('\\', '/')).ToList(); + GamePaths = gamePaths.Select(g => g.Replace('\\', '/')).ToHashSet(StringComparer.Ordinal); ResolvedPath = filePath.Replace('\\', '/'); HashLazy = new(() => !IsFileSwap ? fileDbManager.GetFileCacheByPath(ResolvedPath)?.Hash ?? string.Empty : string.Empty); } public bool Computed => IsFileSwap || !HasFileReplacement || !string.IsNullOrEmpty(Hash); - public List GamePaths { get; init; } = new(); + public HashSet GamePaths { get; init; } = new(); public bool HasFileReplacement => GamePaths.Count >= 1 && GamePaths.Any(p => !string.Equals(p, ResolvedPath, StringComparison.Ordinal)); - public bool IsFileSwap => !Regex.IsMatch(ResolvedPath, @"^[a-zA-Z]:(/|\\)", RegexOptions.ECMAScript) && !string.Equals(GamePaths[0], ResolvedPath, StringComparison.Ordinal); + public bool IsFileSwap => !Regex.IsMatch(ResolvedPath, @"^[a-zA-Z]:(/|\\)", RegexOptions.ECMAScript) && !string.Equals(GamePaths.First(), ResolvedPath, StringComparison.Ordinal); public string Hash => HashLazy.Value; diff --git a/MareSynchronos/Models/GameObjectHandler.cs b/MareSynchronos/Models/GameObjectHandler.cs index 7f398cd..98da88a 100644 --- a/MareSynchronos/Models/GameObjectHandler.cs +++ b/MareSynchronos/Models/GameObjectHandler.cs @@ -4,6 +4,8 @@ using MareSynchronos.Utils; using Penumbra.String; using MareSynchronos.API.Data.Enum; using MareSynchronos.Mediator; +using System; +using Microsoft.Extensions.Logging.Abstractions; namespace MareSynchronos.Models; @@ -57,6 +59,8 @@ public class GameObjectHandler : MediatorSubscriberBase public byte[] EquipSlotData { get; set; } = new byte[40]; public byte[] CustomizeData { get; set; } = new byte[26]; + private Task? _petClearTask; + private CancellationTokenSource? _petCts = new(); public byte? HatState { get; set; } public byte? VisorWeaponState { get; set; } private bool _doNotSendUpdate; @@ -64,8 +68,14 @@ public class GameObjectHandler : MediatorSubscriberBase public unsafe bool CheckAndUpdateObject() { var curPtr = CurrentAddress; - if (curPtr != IntPtr.Zero) + if (curPtr != IntPtr.Zero && (IntPtr)((Character*)curPtr)->GameObject.DrawObject != IntPtr.Zero) { + if (ObjectKind == ObjectKind.Pet && _petCts != null) + { + Logger.Debug("Cancelling PetClearTask for " + ObjectKind); + _petCts?.Cancel(); + _petCts = null; + } var chara = (Character*)curPtr; bool addr = Address == IntPtr.Zero || Address != curPtr; bool equip = CompareAndUpdateByteData(chara->EquipSlotData, chara->CustomizeData); @@ -93,11 +103,32 @@ public class GameObjectHandler : MediatorSubscriberBase Address = IntPtr.Zero; DrawObjectAddress = IntPtr.Zero; Logger.Verbose(ObjectKind + " Changed: " + _name + ", now: " + Address + ", " + DrawObjectAddress); + if (_sendUpdates && ObjectKind == ObjectKind.Pet) + { + _petCts?.Cancel(); + _petCts?.Dispose(); + _petCts = new(); + var token = _petCts.Token; + _petClearTask = Task.Run(() => PetClearTask(token), token); + } + else if (_sendUpdates) + { + Logger.Debug("Sending ClearCachedForObjectMessage for " + ObjectKind); + Mediator.Publish(new ClearCacheForObjectMessage(this)); + } } return false; } + private async Task PetClearTask(CancellationToken token) + { + Logger.Debug("Running PetClearTask for " + ObjectKind); + await Task.Delay(TimeSpan.FromSeconds(1), token).ConfigureAwait(false); + Logger.Debug("Sending ClearCachedForObjectMessage for " + ObjectKind); + Mediator.Publish(new ClearCacheForObjectMessage(this)); + } + private unsafe bool CompareAndUpdateByteData(byte* equipSlotData, byte* customizeData) { bool hasChanges = false; diff --git a/MareSynchronos/UI/CompactUI.cs b/MareSynchronos/UI/CompactUI.cs index 8e18efc..6f8416c 100644 --- a/MareSynchronos/UI/CompactUI.cs +++ b/MareSynchronos/UI/CompactUI.cs @@ -535,8 +535,9 @@ public class CompactUi : WindowMediatorSubscriberBase, IDisposable ImGui.BeginChild("list", new Vector2(WindowContentWidth, ySize), border: false); var visibleUsers = users.Where(u => u.IsVisible && u.UserPair!.OtherPermissions.IsPaired()).OrderBy(u => _configService.Current.ShowCharacterNameInsteadOfNotesForVisible ? u.PlayerName : (u.GetNote() ?? u.UserData.AliasOrUID), StringComparer.OrdinalIgnoreCase).ToList(); - var onlineUsers = users.Where(u => u.IsOnline && !u.IsVisible && u.UserPair!.OtherPermissions.IsPaired()).OrderBy(u => u.GetNote() ?? u.UserData.AliasOrUID, StringComparer.OrdinalIgnoreCase).ToList(); - var offlineUsers = users.Where(u => !u.IsOnline && !u.IsVisible || !u.UserPair!.OtherPermissions.IsPaired()).OrderBy(u => u.GetNote() ?? u.UserData.AliasOrUID, StringComparer.OrdinalIgnoreCase).ToList(); + var onlineUsers = users.Where(u => u.IsOnline || (u.UserPair.OwnPermissions.IsPaused() || u.UserPair.OtherPermissions.IsPaused())).OrderBy(u => u.GetNote() ?? u.UserData.AliasOrUID, StringComparer.OrdinalIgnoreCase).ToList(); + var offlineUsers = users.Where(u => (!u.IsOnline && !u.IsVisible || !u.UserPair!.OtherPermissions.IsPaired()) && !(u.UserPair.OwnPermissions.IsPaused() || u.UserPair.OtherPermissions.IsPaused())) + .OrderBy(u => u.GetNote() ?? u.UserData.AliasOrUID, StringComparer.OrdinalIgnoreCase).ToList(); if (_configService.Current.ReverseUserSort) { @@ -604,15 +605,18 @@ public class CompactUi : WindowMediatorSubscriberBase, IDisposable var color = UiShared.GetBoolColor(!_serverManager.CurrentServer!.FullPause); var connectedIcon = !_serverManager.CurrentServer.FullPause ? FontAwesomeIcon.Link : FontAwesomeIcon.Unlink; - ImGui.PushStyleColor(ImGuiCol.Text, color); - if (ImGuiComponents.IconButton(connectedIcon)) + if (_apiController.ServerState != ServerState.Reconnecting) { - _serverManager.CurrentServer.FullPause = !_serverManager.CurrentServer.FullPause; - _serverManager.Save(); - _ = _apiController.CreateConnections(); + ImGui.PushStyleColor(ImGuiCol.Text, color); + if (ImGuiComponents.IconButton(connectedIcon)) + { + _serverManager.CurrentServer.FullPause = !_serverManager.CurrentServer.FullPause; + _serverManager.Save(); + _ = _apiController.CreateConnections(); + } + ImGui.PopStyleColor(); + UiShared.AttachToolTip(!_serverManager.CurrentServer.FullPause ? "Disconnect from " + _serverManager.CurrentServer.ServerName : "Connect to " + _serverManager.CurrentServer.ServerName); } - ImGui.PopStyleColor(); - UiShared.AttachToolTip(!_serverManager.CurrentServer.FullPause ? "Disconnect from " + _serverManager.CurrentServer.ServerName : "Connect to " + _serverManager.CurrentServer.ServerName); } private void DrawTransfers() diff --git a/MareSynchronos/UI/Components/PairGroupsUi.cs b/MareSynchronos/UI/Components/PairGroupsUi.cs index eefb7a3..8dc295d 100644 --- a/MareSynchronos/UI/Components/PairGroupsUi.cs +++ b/MareSynchronos/UI/Components/PairGroupsUi.cs @@ -75,8 +75,8 @@ namespace MareSynchronos.UI.Components { string displayedName = tag switch { - TagHandler.CustomOfflineTag => "Offline/Unknown", - TagHandler.CustomOnlineTag => "Online", + TagHandler.CustomOfflineTag => "Offline/Unpaired", + TagHandler.CustomOnlineTag => "Online/Paused", TagHandler.CustomVisibleTag => "Visible", _ => tag }; @@ -103,7 +103,7 @@ namespace MareSynchronos.UI.Components ImGui.TextUnformatted($"Group {tag}"); ImGui.Separator(); ImGui.TextUnformatted($"{visible} Pairs visible"); - ImGui.TextUnformatted($"{online} Pairs online"); + ImGui.TextUnformatted($"{online} Pairs online/paused"); ImGui.TextUnformatted($"{total} Pairs total"); ImGui.EndTooltip(); } diff --git a/MareSynchronos/UI/SettingsUi.cs b/MareSynchronos/UI/SettingsUi.cs index 6b435c4..7e9ee49 100644 --- a/MareSynchronos/UI/SettingsUi.cs +++ b/MareSynchronos/UI/SettingsUi.cs @@ -784,11 +784,20 @@ public class SettingsUi : WindowMediatorSubscriberBase, IDisposable _uiShared.DrawFileScanState(); _uiShared.DrawTimeSpanBetweenScansSetting(); _uiShared.DrawCacheDirectorySetting(); - ImGui.Text($"Local storage size: {UiShared.ByteToString(_uiShared.FileCacheSize)}"); + ImGui.Text($"Currently utilized local storage: {UiShared.ByteToString(_uiShared.FileCacheSize)}"); + ImGui.Dummy(new Vector2(10, 10)); + ImGui.Text("To clear the local storage accept the following disclaimer"); + ImGui.Indent(); + ImGui.Checkbox("##readClearCache", ref _readClearCache); ImGui.SameLine(); - if (ImGui.Button("Clear local storage")) + UiShared.TextWrapped("I understand that: " + Environment.NewLine + "- By clearing the local storage I put the file servers of my connected service under extra strain by having to redownload all data." + + Environment.NewLine + "- This is not a step to try to fix sync issues." + + Environment.NewLine + "- This can make the situation of not getting other players data worse in situations of heavy file server load."); + if (!_readClearCache) + ImGui.BeginDisabled(); + if (UiShared.IconTextButton(FontAwesomeIcon.Trash, "Clear local storage")) { - if (UiShared.CtrlPressed()) + if (UiShared.CtrlPressed() && _readClearCache) { Task.Run(() => { @@ -801,11 +810,17 @@ public class SettingsUi : WindowMediatorSubscriberBase, IDisposable }); } } - UiShared.AttachToolTip("You normally do not need to do this. This will solely remove all downloaded data from all players and will require you to re-download everything again." + Environment.NewLine + UiShared.AttachToolTip("You normally do not need to do this. THIS IS NOT SOMETHING YOU SHOULD BE DOING TO TRY TO FIX SYNC ISSUES." + Environment.NewLine + + "This will solely remove all downloaded data from all players and will require you to re-download everything again." + Environment.NewLine + "Mares storage is self-clearing and will not surpass the limit you have set it to." + Environment.NewLine + "If you still think you need to do this hold CTRL while pressing the button."); + if (!_readClearCache) + ImGui.EndDisabled(); + ImGui.Unindent(); } + private bool _readClearCache = false; + public override void OnClose() { _uiShared.EditTrackerPosition = false; diff --git a/MareSynchronos/Utils/DalamudUtil.cs b/MareSynchronos/Utils/DalamudUtil.cs index 30dc83c..2b0eecf 100644 --- a/MareSynchronos/Utils/DalamudUtil.cs +++ b/MareSynchronos/Utils/DalamudUtil.cs @@ -237,7 +237,7 @@ public class DalamudUtil : IDisposable while ((!ct?.IsCancellationRequested ?? true) && curWaitTime < timeOut && (((obj->GetDrawObject() == null - || ((CharacterBase*)obj->GetDrawObject())->HasModelFilesInSlotLoaded != 0 + || ((CharacterBase*)obj->GetDrawObject())->HasModelInSlotLoaded != 0 || ((CharacterBase*)obj->GetDrawObject())->HasModelFilesInSlotLoaded != 0)) || ((obj->RenderFlags & 0b100000000000) == 0b100000000000))) // 0b100000000000 is "still rendering" or something { @@ -246,6 +246,10 @@ public class DalamudUtil : IDisposable Thread.Sleep(tick); } } + catch (NullReferenceException ex) + { + Logger.Warn("Error accessing " + characterAddress.ToString("X") + ", object does not exist anymore?", ex); + } catch (AccessViolationException ex) { Logger.Warn("Error accessing " + characterAddress.ToString("X") + ", object does not exist anymore?", ex); diff --git a/MareSynchronos/Utils/FileReplacementComparer.cs b/MareSynchronos/Utils/FileReplacementComparer.cs index a2be7dc..e0a614d 100644 --- a/MareSynchronos/Utils/FileReplacementComparer.cs +++ b/MareSynchronos/Utils/FileReplacementComparer.cs @@ -29,14 +29,14 @@ public class FileReplacementComparer : IEqualityComparer return hash; } - private bool CompareLists(List list1, List list2) + private bool CompareLists(HashSet list1, HashSet list2) { if (list1.Count != list2.Count) return false; for (int i = 0; i < list1.Count; i++) { - if (!string.Equals(list1[i], list2[i], StringComparison.OrdinalIgnoreCase)) + if (!string.Equals(list1.ElementAt(i), list2.ElementAt(i), StringComparison.OrdinalIgnoreCase)) return false; } diff --git a/MareSynchronos/Utils/Logger.cs b/MareSynchronos/Utils/Logger.cs index 8f8142e..47fd25c 100644 --- a/MareSynchronos/Utils/Logger.cs +++ b/MareSynchronos/Utils/Logger.cs @@ -74,17 +74,17 @@ internal class Logger : ILogger switch (logLevel) { case LogLevel.Debug: - PluginLog.Debug($"[{_name}] [{eventId}] {formatter(state, exception)}"); + Debug($"[{_name}] [{eventId}] {formatter(state, exception)}"); break; case LogLevel.Error: case LogLevel.Critical: - PluginLog.Error($"[{_name}] [{eventId}] {formatter(state, exception)}"); + Error($"[{_name}] [{eventId}] {formatter(state, exception)}", exception); break; case LogLevel.Information: - PluginLog.Information($"[{_name}] [{eventId}] {formatter(state, exception)}"); + Info($"[{_name}] [{eventId}] {formatter(state, exception)}"); break; case LogLevel.Warning: - PluginLog.Warning($"[{_name}] [{eventId}] {formatter(state, exception)}"); + Warn($"[{_name}] [{eventId}] {formatter(state, exception)}", exception); break; case LogLevel.Trace: default: diff --git a/MareSynchronos/WebAPI/ApIController.Functions.Files.cs b/MareSynchronos/WebAPI/ApIController.Functions.Files.cs index 1225c47..d2975c4 100644 --- a/MareSynchronos/WebAPI/ApIController.Functions.Files.cs +++ b/MareSynchronos/WebAPI/ApIController.Functions.Files.cs @@ -10,6 +10,7 @@ using MareSynchronos.API.Data; using MareSynchronos.API.Dto.Files; using MareSynchronos.API.Routes; using MareSynchronos.Mediator; +using MareSynchronos.UI; using MareSynchronos.Utils; using MareSynchronos.WebAPI.Utils; using Microsoft.AspNetCore.SignalR.Client; @@ -20,17 +21,23 @@ public partial class ApiController { private readonly Dictionary _verifiedUploadedHashes; private readonly ConcurrentDictionary _downloadReady = new(); + private bool currentUploadCancelled = false; private int _downloadId = 0; - public async void CancelUpload() + public async Task CancelUpload() { - if (_uploadCancellationTokenSource != null) + if (CurrentUploads.Any()) { - Logger.Debug("Cancelling upload"); + Logger.Debug("Cancelling current upload"); _uploadCancellationTokenSource?.Cancel(); + _uploadCancellationTokenSource?.Dispose(); + _uploadCancellationTokenSource = null; CurrentUploads.Clear(); await FilesAbortUpload().ConfigureAwait(false); + return true; } + + return false; } public async Task FilesAbortUpload() @@ -333,19 +340,45 @@ public partial class ApiController CancelDownload(currentDownloadId); } - public async Task PushCharacterData(API.Data.CharacterData character, List visibleCharacters) + public async Task PushCharacterData(CharacterData data, List visibleCharacters) { if (!IsConnected) return; - Logger.Debug("Sending Character data to service " + _serverManager.CurrentApiUrl); - visibleCharacters = visibleCharacters.ToList(); - CancelUpload(); - _uploadCancellationTokenSource = new CancellationTokenSource(); - var uploadToken = _uploadCancellationTokenSource.Token; - Logger.Verbose("New Token Created"); + try + { + currentUploadCancelled = await CancelUpload().ConfigureAwait(false); - List unverifiedUploadHashes = new(); - foreach (var item in character.FileReplacements.SelectMany(c => c.Value.Where(f => string.IsNullOrEmpty(f.FileSwapPath)).Select(v => v.Hash).Distinct(StringComparer.Ordinal)).Distinct(StringComparer.Ordinal).ToList()) + _uploadCancellationTokenSource = new CancellationTokenSource(); + var uploadToken = _uploadCancellationTokenSource.Token; + Logger.Debug($"Sending Character data {data.DataHash.Value} to service {_serverManager.CurrentApiUrl}"); + + HashSet unverifiedUploads = VerifyFiles(data); + if (unverifiedUploads.Any()) + { + await UploadMissingFiles(unverifiedUploads, uploadToken).ConfigureAwait(false); + Logger.Info("Upload complete for " + data.DataHash.Value); + } + await PushCharacterDataInternal(data, visibleCharacters.ToList()).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + Logger.Debug("Upload operation was cancelled"); + } + catch (Exception ex) + { + Logger.Warn("Error during upload of files", ex); + } + finally + { + if (!currentUploadCancelled) + currentUploadCancelled = await CancelUpload().ConfigureAwait(false); + } + } + + private HashSet VerifyFiles(CharacterData data) + { + HashSet unverifiedUploadHashes = new(StringComparer.Ordinal); + foreach (var item in data.FileReplacements.SelectMany(c => c.Value.Where(f => string.IsNullOrEmpty(f.FileSwapPath)).Select(v => v.Hash).Distinct(StringComparer.Ordinal)).Distinct(StringComparer.Ordinal).ToList()) { if (!_verifiedUploadedHashes.TryGetValue(item, out var verifiedTime)) { @@ -359,103 +392,93 @@ public partial class ApiController } } - if (unverifiedUploadHashes.Any()) + return unverifiedUploadHashes; + } + + private async Task UploadMissingFiles(HashSet unverifiedUploadHashes, CancellationToken uploadToken) + { + unverifiedUploadHashes = unverifiedUploadHashes.Where(h => _fileDbManager.GetFileCacheByHash(h) != null).ToHashSet(StringComparer.Ordinal); + + Logger.Debug("Verifying " + unverifiedUploadHashes.Count + " files"); + var filesToUpload = await FilesSend(unverifiedUploadHashes.ToList()).ConfigureAwait(false); + + foreach (var file in filesToUpload.Where(f => !f.IsForbidden)) { - unverifiedUploadHashes = unverifiedUploadHashes.Where(h => _fileDbManager.GetFileCacheByHash(h) != null).ToList(); - - Logger.Debug("Verifying " + unverifiedUploadHashes.Count + " files"); - var filesToUpload = await FilesSend(unverifiedUploadHashes).ConfigureAwait(false); - - foreach (var file in filesToUpload.Where(f => !f.IsForbidden)) + try { - try + CurrentUploads.Add(new UploadFileTransfer(file) { - CurrentUploads.Add(new UploadFileTransfer(file) - { - Total = new FileInfo(_fileDbManager.GetFileCacheByHash(file.Hash)!.ResolvedFilepath).Length, - }); - } - catch (Exception ex) - { - Logger.Warn("Tried to request file " + file.Hash + " but file was not present"); - Logger.Warn(ex.StackTrace!); - } + Total = new FileInfo(_fileDbManager.GetFileCacheByHash(file.Hash)!.ResolvedFilepath).Length, + }); } - - foreach (var file in filesToUpload.Where(c => c.IsForbidden)) + catch (Exception ex) { - if (ForbiddenTransfers.All(f => !string.Equals(f.Hash, file.Hash, StringComparison.Ordinal))) - { - ForbiddenTransfers.Add(new UploadFileTransfer(file) - { - LocalFile = _fileDbManager.GetFileCacheByHash(file.Hash)?.ResolvedFilepath ?? string.Empty, - }); - } + Logger.Warn("Tried to request file " + file.Hash + " but file was not present", ex); } - - var totalSize = CurrentUploads.Sum(c => c.Total); - Logger.Debug("Compressing and uploading files"); - foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList()) - { - Logger.Debug("Compressing and uploading " + file); - var data = await GetCompressedFileData(file.Hash, uploadToken).ConfigureAwait(false); - CurrentUploads.Single(e => string.Equals(e.Hash, data.Item1, StringComparison.Ordinal)).Total = data.Item2.Length; - await UploadFile(data.Item2, file.Hash, uploadToken).ConfigureAwait(false); - if (!uploadToken.IsCancellationRequested) continue; - Logger.Warn("Cancel in filesToUpload loop detected"); - CurrentUploads.Clear(); - break; - } - - if (CurrentUploads.Any()) - { - var compressedSize = CurrentUploads.Sum(c => c.Total); - Logger.Debug($"Compressed {totalSize} to {compressedSize} ({(compressedSize / (double)totalSize):P2})"); - - Logger.Debug("Upload tasks complete, waiting for server to confirm"); - Logger.Debug("Uploads open: " + CurrentUploads.Any(c => c.IsInTransfer)); - const double waitStep = 1.0d; - while (CurrentUploads.Any(c => c.IsInTransfer) && !uploadToken.IsCancellationRequested) - { - await Task.Delay(TimeSpan.FromSeconds(waitStep), uploadToken).ConfigureAwait(false); - Logger.Debug("Waiting for uploads to finish"); - } - } - - foreach (var item in unverifiedUploadHashes) - { - _verifiedUploadedHashes[item] = DateTime.UtcNow; - } - - CurrentUploads.Clear(); - } - else - { - Logger.Debug("All files already verified"); } - if (!uploadToken.IsCancellationRequested) + foreach (var file in filesToUpload.Where(c => c.IsForbidden)) { - Logger.Info("Pushing character data for " + character.GetHashCode() + " to " + string.Join(", ", visibleCharacters.Select(c => c.AliasOrUID))); - StringBuilder sb = new(); - foreach (var item in character.FileReplacements) + if (ForbiddenTransfers.All(f => !string.Equals(f.Hash, file.Hash, StringComparison.Ordinal))) { - sb.AppendLine($"FileReplacements for {item.Key}: {item.Value.Count}"); + ForbiddenTransfers.Add(new UploadFileTransfer(file) + { + LocalFile = _fileDbManager.GetFileCacheByHash(file.Hash)?.ResolvedFilepath ?? string.Empty, + }); } - foreach (var item in character.GlamourerData) - { - sb.AppendLine($"GlamourerData for {item.Key}: {!string.IsNullOrEmpty(item.Value)}"); - } - Logger.Debug("Chara data contained: " + Environment.NewLine + sb.ToString()); - await UserPushData(new(visibleCharacters, character)).ConfigureAwait(false); - } - else - { - Logger.Warn("=== Upload operation was cancelled ==="); + + _verifiedUploadedHashes[file.Hash] = DateTime.UtcNow; } - Logger.Verbose("Upload complete for " + character.DataHash); - _uploadCancellationTokenSource = null; + var totalSize = CurrentUploads.Sum(c => c.Total); + Logger.Debug("Compressing and uploading files"); + foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList()) + { + Logger.Debug("Compressing and uploading " + file); + var data = await GetCompressedFileData(file.Hash, uploadToken).ConfigureAwait(false); + CurrentUploads.Single(e => string.Equals(e.Hash, data.Item1, StringComparison.Ordinal)).Total = data.Item2.Length; + await UploadFile(data.Item2, file.Hash, uploadToken).ConfigureAwait(false); + _verifiedUploadedHashes[file.Hash] = DateTime.UtcNow; + uploadToken.ThrowIfCancellationRequested(); + } + + if (CurrentUploads.Any()) + { + var compressedSize = CurrentUploads.Sum(c => c.Total); + Logger.Debug($"Compressed {UiShared.ByteToString(totalSize)} to {UiShared.ByteToString(compressedSize)} ({(compressedSize / (double)totalSize):P2})"); + + Logger.Debug("Upload tasks complete, waiting for server to confirm"); + Logger.Debug("Uploads open: " + CurrentUploads.Any(c => c.IsInTransfer)); + const double waitStep = 1.0d; + while (CurrentUploads.Any(c => c.IsInTransfer) && !uploadToken.IsCancellationRequested) + { + await Task.Delay(TimeSpan.FromSeconds(waitStep), uploadToken).ConfigureAwait(false); + Logger.Debug("Waiting for uploads to finish"); + } + } + + foreach(var file in unverifiedUploadHashes.Where(c=>!CurrentUploads.Any(u=> string.Equals(u.Hash, c, StringComparison.Ordinal)))) + { + _verifiedUploadedHashes[file] = DateTime.UtcNow; + } + + CurrentUploads.Clear(); + } + + private async Task PushCharacterDataInternal(CharacterData character, List visibleCharacters) + { + Logger.Info("Pushing character data for " + character.DataHash.Value + " to " + string.Join(", ", visibleCharacters.Select(c => c.AliasOrUID))); + StringBuilder sb = new(); + foreach (var item in character.FileReplacements) + { + sb.AppendLine($"FileReplacements for {item.Key}: {item.Value.Count}"); + } + foreach (var item in character.GlamourerData) + { + sb.AppendLine($"GlamourerData for {item.Key}: {!string.IsNullOrEmpty(item.Value)}"); + } + Logger.Debug("Chara data contained: " + Environment.NewLine + sb.ToString()); + await UserPushData(new(visibleCharacters, character)).ConfigureAwait(false); } private async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken) diff --git a/MareSynchronos/WebAPI/ApiController.cs b/MareSynchronos/WebAPI/ApiController.cs index 7511480..602cee0 100644 --- a/MareSynchronos/WebAPI/ApiController.cs +++ b/MareSynchronos/WebAPI/ApiController.cs @@ -204,9 +204,7 @@ public partial class ApiController : MediatorSubscriberBase, IDisposable, IMareH } catch (HttpRequestException ex) { - Logger.Warn(ex.GetType().ToString()); - Logger.Warn(ex.Message); - Logger.Warn(ex.StackTrace ?? string.Empty); + Logger.Warn("HttpRequestException on Connection", ex); if (ex.StatusCode == System.Net.HttpStatusCode.Unauthorized) { @@ -222,9 +220,8 @@ public partial class ApiController : MediatorSubscriberBase, IDisposable, IMareH } catch (Exception ex) { - Logger.Warn(ex.GetType().ToString()); - Logger.Warn(ex.Message); - Logger.Warn(ex.StackTrace ?? string.Empty); + Logger.Warn("Exception on Connection", ex); + Logger.Info("Failed to establish connection, retrying"); await Task.Delay(TimeSpan.FromSeconds(new Random().Next(5, 20)), token).ConfigureAwait(false); } @@ -236,14 +233,8 @@ public partial class ApiController : MediatorSubscriberBase, IDisposable, IMareH while (!ct.IsCancellationRequested && _mareHub != null) { await Task.Delay(TimeSpan.FromSeconds(30), ct).ConfigureAwait(false); - if (ct.IsCancellationRequested) break; var needsRestart = await CheckClientHealth().ConfigureAwait(false); - Logger.Debug("Checked Client Health State, healthy: " + !needsRestart); - if (needsRestart) - { - ServerState = ServerState.Offline; - _ = CreateConnections(); - } + Logger.Debug("Checked Client Health State"); } } @@ -328,7 +319,7 @@ public partial class ApiController : MediatorSubscriberBase, IDisposable, IMareH .ConfigureLogging(a => { a.ClearProviders().AddProvider(new DalamudLoggingProvider()); - a.SetMinimumLevel(LogLevel.Warning); + a.SetMinimumLevel(LogLevel.Information); }) .Build(); } @@ -387,11 +378,6 @@ public partial class ApiController : MediatorSubscriberBase, IDisposable, IMareH } } - public async Task Heartbeat(string characterIdentification) - { - return await _mareHub!.InvokeAsync(nameof(Heartbeat), characterIdentification).ConfigureAwait(false); - } - public async Task GetConnectionDto() { return await _mareHub!.InvokeAsync(nameof(GetConnectionDto)).ConfigureAwait(false);