t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixes for pet handling, uploads, clear storage button, remove button while reconnecting, put paused to online/paused, put visible to online

Browse Source
This commit is contained in:
rootdarkarchon
2023-02-05 15:52:27 +01:00
parent 673d098827
commit 3634c06ee5
17 changed files with 241 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
MareSynchronos/Factories/CharacterDataFactory.cs
View File

@@ -253,19 +253,9 @@ public class CharacterDataFactory
AddPlayerSpecificReplacements(objectKind, charaPointer, human);
}
if (objectKind == ObjectKind.Pet)
{
foreach (var item in previousData.FileReplacements[objectKind])
{
_transientResourceManager.AddSemiTransientResource(objectKind, item.GamePaths.First());
}
previousData.FileReplacements[objectKind].Clear();
}
Dictionary<string, List<string>> resolvedPaths = GetFileReplacementsFromPaths();
previousData.FileReplacements[objectKind] = new HashSet<FileReplacement>(resolvedPaths.Select(c => new FileReplacement(c.Value, c.Key, _fileCacheManager)));
previousData.FileReplacements[objectKind] = new HashSet<FileReplacement>(resolvedPaths.Select(c => new FileReplacement(c.Value, c.Key, _fileCacheManager)), FileReplacementComparer.Instance)
.Where(p => p.HasFileReplacement).ToHashSet();
previousData.ManipulationString = _ipcManager.PenumbraGetMetaManipulations();
previousData.GlamourerString[objectKind] = _ipcManager.GlamourerGetCharacterCustomization(charaPointer);
@@ -274,11 +264,19 @@ public class CharacterDataFactory
previousData.PalettePlusPalette = _ipcManager.PalettePlusBuildPalette();
Logger.Debug("== Static Replacements ==");
foreach (var item in previousData.FileReplacements[objectKind])
foreach (var item in previousData.FileReplacements[objectKind].Where(i => i.HasFileReplacement).OrderBy(i => i.GamePaths.First(), StringComparer.OrdinalIgnoreCase))
{
Logger.Debug(item.ToString());
}
if (objectKind == ObjectKind.Pet)
{
foreach (var item in previousData.FileReplacements[objectKind].Where(i => i.HasFileReplacement).SelectMany(p => p.GamePaths))
{
_transientResourceManager.AddSemiTransientResource(objectKind, item);
}
}
Logger.Debug("Handling transient update for " + objectKind);
_transientResourceManager.ClearTransientPaths(charaPointer, previousData.FileReplacements[objectKind].SelectMany(c => c.GamePaths).ToList());
@@ -289,7 +287,7 @@ public class CharacterDataFactory
var resolvedTransientPaths = GetFileReplacementsFromPaths();
Logger.Debug("== Transient Replacements ==");
foreach (var replacement in resolvedTransientPaths.Select(c => new FileReplacement(c.Value, c.Key, _fileCacheManager)))
foreach (var replacement in resolvedTransientPaths.Select(c => new FileReplacement(c.Value, c.Key, _fileCacheManager)).OrderBy(f => f.ResolvedPath, StringComparer.Ordinal))
{
Logger.Debug(replacement.ToString());
previousData.FileReplacements[objectKind].Add(replacement);
@@ -297,7 +295,10 @@ public class CharacterDataFactory
_transientResourceManager.CleanSemiTransientResources(objectKind, previousData.FileReplacements[objectKind].ToList());
foreach (var item in previousData.FileReplacements)
{
previousData.FileReplacements[item.Key] = new HashSet<FileReplacement>(item.Value.Where(v => v.HasFileReplacement).OrderBy(v => v.ResolvedPath, StringComparer.Ordinal), FileReplacementComparer.Instance);
}
st.Stop();
Logger.Info("Building character data for " + objectKind + " took " + st.ElapsedMilliseconds + "ms");

4
MareSynchronos/FileCache/FileCacheManager.cs
View File

@@ -85,7 +85,9 @@ public class FileCacheManager : IDisposable
{
if (_fileCaches.Any(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal)))
{
return GetValidatedFileCache(_fileCaches.FirstOrDefault(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal)).Value);
return GetValidatedFileCache(_fileCaches.Where(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal))
.OrderByDescending(f => f.Value.PrefixedFilePath.Length)
.FirstOrDefault(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal)).Value);
}
return null;

15
MareSynchronos/Managers/CacheCreationService.cs
View File

@@ -26,6 +26,14 @@ public class CacheCreationService : MediatorSubscriberBase, IDisposable
var actualMsg = (CreateCacheForObjectMessage)msg;
_cachesToCreate[actualMsg.ObjectToCreateFor.ObjectKind] = actualMsg.ObjectToCreateFor;
});
Mediator.Subscribe<ClearCacheForObjectMessage>(this, (msg) =>
{
var actualMsg = (ClearCacheForObjectMessage)msg;
_lastCreatedData.FileReplacements.Remove(actualMsg.ObjectToCreateFor.ObjectKind);
_lastCreatedData.GlamourerString.Remove(actualMsg.ObjectToCreateFor.ObjectKind);
Mediator.Publish(new CharacterDataCreatedMessage(_lastCreatedData));
});
Mediator.Subscribe<FrameworkUpdateMessage>(this, (msg) => UpdatePointers());
Mediator.Subscribe<DelayedFrameworkUpdateMessage>(this, (msg) => ProcessCacheCreation());
Mediator.Subscribe<CustomizePlusMessage>(this, (msg) => CustomizePlusChanged((CustomizePlusMessage)msg));
Mediator.Subscribe<HeelsOffsetMessage>(this, (msg) => HeelsOffsetChanged((HeelsOffsetMessage)msg));
@@ -78,6 +86,11 @@ public class CacheCreationService : MediatorSubscriberBase, IDisposable
}
}
private void UpdatePointers()
{
Mediator.Publish(new PlayerRelatedObjectPointerUpdateMessage(_playerRelatedObjects.Select(f => f.CurrentAddress).ToArray()));
}
private void ProcessCacheCreation()
{
if (_cachesToCreate.Any() && (_cacheCreationTask?.IsCompleted ?? true))
@@ -109,8 +122,6 @@ public class CacheCreationService : MediatorSubscriberBase, IDisposable
{
Logger.Debug("Cache Creation stored until previous creation finished");
}
Mediator.Publish(new PlayerRelatedObjectPointerUpdateMessage(_playerRelatedObjects.Select(f => f.CurrentAddress).ToArray()));
}
public override void Dispose()

1
MareSynchronos/Managers/TransientResourceManager.cs
View File

@@ -133,6 +133,7 @@ public class TransientResourceManager : MediatorSubscriberBase, IDisposable
}
if (!PlayerRelatedPointers.Contains(gameObject))
{
Logger.Debug("Got resource " + gamePath + " for ptr " + gameObject.ToString("X"));
return;
}

2
MareSynchronos/MareSynchronos.csproj
View File

@@ -3,7 +3,7 @@
<PropertyGroup>
<Authors></Authors>
<Company></Company>
<Version>0.7.19</Version>
<Version>0.7.20</Version>
<Description></Description>
<Copyright></Copyright>
<PackageProjectUrl>https://github.com/Penumbra-Sync/client</PackageProjectUrl>

1
MareSynchronos/Mediator/Messages.cs
View File

@@ -37,6 +37,7 @@ public record ResumeScanMessage(string Source) : IMessage;
public record NotificationMessage
(string Title, string Message, NotificationType Type, uint TimeShownOnScreen = 3000) : IMessage;
public record CreateCacheForObjectMessage(GameObjectHandler ObjectToCreateFor) : IMessage;
public record ClearCacheForObjectMessage(GameObjectHandler ObjectToCreateFor) : IMessage;
public record CharacterDataCreatedMessage(CharacterData CharacterData) : IMessage;
#pragma warning restore MA0048 // File name must match type name

19
MareSynchronos/Models/CharacterData.cs
View File

@@ -28,23 +28,6 @@ public class CharacterData
[JsonProperty]
public string PalettePlusPalette { get; set; } = string.Empty;
public void AddFileReplacement(ObjectKind objectKind, FileReplacement fileReplacement)
{
if (!fileReplacement.HasFileReplacement) return;
if (!FileReplacements.ContainsKey(objectKind)) FileReplacements.Add(objectKind, new HashSet<FileReplacement>());
var existingReplacement = FileReplacements[objectKind].SingleOrDefault(f => string.Equals(f.ResolvedPath, fileReplacement.ResolvedPath, StringComparison.OrdinalIgnoreCase));
if (existingReplacement != null)
{
existingReplacement.GamePaths.AddRange(fileReplacement.GamePaths.Where(e => !existingReplacement.GamePaths.Contains(e, StringComparer.OrdinalIgnoreCase)));
}
else
{
FileReplacements[objectKind].Add(fileReplacement);
}
}
public API.Data.CharacterData ToAPI()
{
var fileReplacements = FileReplacements.ToDictionary(k => k.Key, k => k.Value.Where(f => f.HasFileReplacement && !f.IsFileSwap).GroupBy(f => f.Hash, StringComparer.OrdinalIgnoreCase).Select(g =>
@@ -76,7 +59,7 @@ public class CharacterData
public override string ToString()
{
StringBuilder stringBuilder = new();
foreach (var fileReplacement in FileReplacements.SelectMany(k => k.Value).OrderBy(a => a.GamePaths[0], StringComparer.Ordinal))
foreach (var fileReplacement in FileReplacements.SelectMany(k => k.Value).OrderBy(a => a.GamePaths.First(), StringComparer.Ordinal))
{
stringBuilder.AppendLine(fileReplacement.ToString());
}

6
MareSynchronos/Models/FileReplacement.cs
View File

@@ -8,18 +8,18 @@ public class FileReplacement
{
public FileReplacement(List<string> gamePaths, string filePath, FileCacheManager fileDbManager)
{
GamePaths = gamePaths.Select(g => g.Replace('\\', '/')).ToList();
GamePaths = gamePaths.Select(g => g.Replace('\\', '/')).ToHashSet(StringComparer.Ordinal);
ResolvedPath = filePath.Replace('\\', '/');
HashLazy = new(() => !IsFileSwap ? fileDbManager.GetFileCacheByPath(ResolvedPath)?.Hash ?? string.Empty : string.Empty);
}
public bool Computed => IsFileSwap || !HasFileReplacement || !string.IsNullOrEmpty(Hash);
public List<string> GamePaths { get; init; } = new();
public HashSet<string> GamePaths { get; init; } = new();
public bool HasFileReplacement => GamePaths.Count >= 1 && GamePaths.Any(p => !string.Equals(p, ResolvedPath, StringComparison.Ordinal));
public bool IsFileSwap => !Regex.IsMatch(ResolvedPath, @"^[a-zA-Z]:(/|\\)", RegexOptions.ECMAScript) && !string.Equals(GamePaths[0], ResolvedPath, StringComparison.Ordinal);
public bool IsFileSwap => !Regex.IsMatch(ResolvedPath, @"^[a-zA-Z]:(/|\\)", RegexOptions.ECMAScript) && !string.Equals(GamePaths.First(), ResolvedPath, StringComparison.Ordinal);
public string Hash => HashLazy.Value;

33
MareSynchronos/Models/GameObjectHandler.cs
View File

@@ -4,6 +4,8 @@ using MareSynchronos.Utils;
using Penumbra.String;
using MareSynchronos.API.Data.Enum;
using MareSynchronos.Mediator;
using System;
using Microsoft.Extensions.Logging.Abstractions;
namespace MareSynchronos.Models;
@@ -57,6 +59,8 @@ public class GameObjectHandler : MediatorSubscriberBase
public byte[] EquipSlotData { get; set; } = new byte[40];
public byte[] CustomizeData { get; set; } = new byte[26];
private Task? _petClearTask;
private CancellationTokenSource? _petCts = new();
public byte? HatState { get; set; }
public byte? VisorWeaponState { get; set; }
private bool _doNotSendUpdate;
@@ -64,8 +68,14 @@ public class GameObjectHandler : MediatorSubscriberBase
public unsafe bool CheckAndUpdateObject()
{
var curPtr = CurrentAddress;
if (curPtr != IntPtr.Zero)
if (curPtr != IntPtr.Zero && (IntPtr)((Character*)curPtr)->GameObject.DrawObject != IntPtr.Zero)
{
if (ObjectKind == ObjectKind.Pet && _petCts != null)
{
Logger.Debug("Cancelling PetClearTask for " + ObjectKind);
_petCts?.Cancel();
_petCts = null;
}
var chara = (Character*)curPtr;
bool addr = Address == IntPtr.Zero || Address != curPtr;
bool equip = CompareAndUpdateByteData(chara->EquipSlotData, chara->CustomizeData);
@@ -93,11 +103,32 @@ public class GameObjectHandler : MediatorSubscriberBase
Address = IntPtr.Zero;
DrawObjectAddress = IntPtr.Zero;
Logger.Verbose(ObjectKind + " Changed: " + _name + ", now: " + Address + ", " + DrawObjectAddress);
if (_sendUpdates && ObjectKind == ObjectKind.Pet)
{
_petCts?.Cancel();
_petCts?.Dispose();
_petCts = new();
var token = _petCts.Token;
_petClearTask = Task.Run(() => PetClearTask(token), token);
}
else if (_sendUpdates)
{
Logger.Debug("Sending ClearCachedForObjectMessage for " + ObjectKind);
Mediator.Publish(new ClearCacheForObjectMessage(this));
}
}
return false;
}
private async Task PetClearTask(CancellationToken token)
{
Logger.Debug("Running PetClearTask for " + ObjectKind);
await Task.Delay(TimeSpan.FromSeconds(1), token).ConfigureAwait(false);
Logger.Debug("Sending ClearCachedForObjectMessage for " + ObjectKind);
Mediator.Publish(new ClearCacheForObjectMessage(this));
}
private unsafe bool CompareAndUpdateByteData(byte* equipSlotData, byte* customizeData)
{
bool hasChanges = false;

22
MareSynchronos/UI/CompactUI.cs
View File

@@ -535,8 +535,9 @@ public class CompactUi : WindowMediatorSubscriberBase, IDisposable
ImGui.BeginChild("list", new Vector2(WindowContentWidth, ySize), border: false);
var visibleUsers = users.Where(u => u.IsVisible && u.UserPair!.OtherPermissions.IsPaired()).OrderBy(u => _configService.Current.ShowCharacterNameInsteadOfNotesForVisible ?
u.PlayerName : (u.GetNote() ?? u.UserData.AliasOrUID), StringComparer.OrdinalIgnoreCase).ToList();
var onlineUsers = users.Where(u => u.IsOnline && !u.IsVisible && u.UserPair!.OtherPermissions.IsPaired()).OrderBy(u => u.GetNote() ?? u.UserData.AliasOrUID, StringComparer.OrdinalIgnoreCase).ToList();
var offlineUsers = users.Where(u => !u.IsOnline && !u.IsVisible || !u.UserPair!.OtherPermissions.IsPaired()).OrderBy(u => u.GetNote() ?? u.UserData.AliasOrUID, StringComparer.OrdinalIgnoreCase).ToList();
var onlineUsers = users.Where(u => u.IsOnline || (u.UserPair.OwnPermissions.IsPaused() || u.UserPair.OtherPermissions.IsPaused())).OrderBy(u => u.GetNote() ?? u.UserData.AliasOrUID, StringComparer.OrdinalIgnoreCase).ToList();
var offlineUsers = users.Where(u => (!u.IsOnline && !u.IsVisible || !u.UserPair!.OtherPermissions.IsPaired()) && !(u.UserPair.OwnPermissions.IsPaused() || u.UserPair.OtherPermissions.IsPaused()))
.OrderBy(u => u.GetNote() ?? u.UserData.AliasOrUID, StringComparer.OrdinalIgnoreCase).ToList();
if (_configService.Current.ReverseUserSort)
{
@@ -604,15 +605,18 @@ public class CompactUi : WindowMediatorSubscriberBase, IDisposable
var color = UiShared.GetBoolColor(!_serverManager.CurrentServer!.FullPause);
var connectedIcon = !_serverManager.CurrentServer.FullPause ? FontAwesomeIcon.Link : FontAwesomeIcon.Unlink;
ImGui.PushStyleColor(ImGuiCol.Text, color);
if (ImGuiComponents.IconButton(connectedIcon))
if (_apiController.ServerState != ServerState.Reconnecting)
{
_serverManager.CurrentServer.FullPause = !_serverManager.CurrentServer.FullPause;
_serverManager.Save();
_ = _apiController.CreateConnections();
ImGui.PushStyleColor(ImGuiCol.Text, color);
if (ImGuiComponents.IconButton(connectedIcon))
{
_serverManager.CurrentServer.FullPause = !_serverManager.CurrentServer.FullPause;
_serverManager.Save();
_ = _apiController.CreateConnections();
}
ImGui.PopStyleColor();
UiShared.AttachToolTip(!_serverManager.CurrentServer.FullPause ? "Disconnect from " + _serverManager.CurrentServer.ServerName : "Connect to " + _serverManager.CurrentServer.ServerName);
}
ImGui.PopStyleColor();
UiShared.AttachToolTip(!_serverManager.CurrentServer.FullPause ? "Disconnect from " + _serverManager.CurrentServer.ServerName : "Connect to " + _serverManager.CurrentServer.ServerName);
}
private void DrawTransfers()

6
MareSynchronos/UI/Components/PairGroupsUi.cs
View File

@@ -75,8 +75,8 @@ namespace MareSynchronos.UI.Components
{
string displayedName = tag switch
{
TagHandler.CustomOfflineTag => "Offline/Unknown",
TagHandler.CustomOnlineTag => "Online",
TagHandler.CustomOfflineTag => "Offline/Unpaired",
TagHandler.CustomOnlineTag => "Online/Paused",
TagHandler.CustomVisibleTag => "Visible",
_ => tag
};
@@ -103,7 +103,7 @@ namespace MareSynchronos.UI.Components
ImGui.TextUnformatted($"Group {tag}");
ImGui.Separator();
ImGui.TextUnformatted($"{visible} Pairs visible");
ImGui.TextUnformatted($"{online} Pairs online");
ImGui.TextUnformatted($"{online} Pairs online/paused");
ImGui.TextUnformatted($"{total} Pairs total");
ImGui.EndTooltip();
}

23
MareSynchronos/UI/SettingsUi.cs
View File

@@ -784,11 +784,20 @@ public class SettingsUi : WindowMediatorSubscriberBase, IDisposable
_uiShared.DrawFileScanState();
_uiShared.DrawTimeSpanBetweenScansSetting();
_uiShared.DrawCacheDirectorySetting();
ImGui.Text($"Local storage size: {UiShared.ByteToString(_uiShared.FileCacheSize)}");
ImGui.Text($"Currently utilized local storage: {UiShared.ByteToString(_uiShared.FileCacheSize)}");
ImGui.Dummy(new Vector2(10, 10));
ImGui.Text("To clear the local storage accept the following disclaimer");
ImGui.Indent();
ImGui.Checkbox("##readClearCache", ref _readClearCache);
ImGui.SameLine();
if (ImGui.Button("Clear local storage"))
UiShared.TextWrapped("I understand that: " + Environment.NewLine + "- By clearing the local storage I put the file servers of my connected service under extra strain by having to redownload all data."
+ Environment.NewLine + "- This is not a step to try to fix sync issues."
+ Environment.NewLine + "- This can make the situation of not getting other players data worse in situations of heavy file server load.");
if (!_readClearCache)
ImGui.BeginDisabled();
if (UiShared.IconTextButton(FontAwesomeIcon.Trash, "Clear local storage"))
{
if (UiShared.CtrlPressed())
if (UiShared.CtrlPressed() && _readClearCache)
{
Task.Run(() =>
{
@@ -801,11 +810,17 @@ public class SettingsUi : WindowMediatorSubscriberBase, IDisposable
});
}
}
UiShared.AttachToolTip("You normally do not need to do this. This will solely remove all downloaded data from all players and will require you to re-download everything again." + Environment.NewLine
UiShared.AttachToolTip("You normally do not need to do this. THIS IS NOT SOMETHING YOU SHOULD BE DOING TO TRY TO FIX SYNC ISSUES." + Environment.NewLine
+ "This will solely remove all downloaded data from all players and will require you to re-download everything again." + Environment.NewLine
+ "Mares storage is self-clearing and will not surpass the limit you have set it to." + Environment.NewLine
+ "If you still think you need to do this hold CTRL while pressing the button.");
if (!_readClearCache)
ImGui.EndDisabled();
ImGui.Unindent();
}
private bool _readClearCache = false;
public override void OnClose()
{
_uiShared.EditTrackerPosition = false;

6
MareSynchronos/Utils/DalamudUtil.cs
View File

@@ -237,7 +237,7 @@ public class DalamudUtil : IDisposable
while ((!ct?.IsCancellationRequested ?? true)
&& curWaitTime < timeOut
&& (((obj->GetDrawObject() == null
|| ((CharacterBase*)obj->GetDrawObject())->HasModelFilesInSlotLoaded != 0
|| ((CharacterBase*)obj->GetDrawObject())->HasModelInSlotLoaded != 0
|| ((CharacterBase*)obj->GetDrawObject())->HasModelFilesInSlotLoaded != 0))
|| ((obj->RenderFlags & 0b100000000000) == 0b100000000000))) // 0b100000000000 is "still rendering" or something
{
@@ -246,6 +246,10 @@ public class DalamudUtil : IDisposable
Thread.Sleep(tick);
}
}
catch (NullReferenceException ex)
{
Logger.Warn("Error accessing " + characterAddress.ToString("X") + ", object does not exist anymore?", ex);
}
catch (AccessViolationException ex)
{
Logger.Warn("Error accessing " + characterAddress.ToString("X") + ", object does not exist anymore?", ex);

4
MareSynchronos/Utils/FileReplacementComparer.cs
View File

@@ -29,14 +29,14 @@ public class FileReplacementComparer : IEqualityComparer<FileReplacement>
return hash;
}
private bool CompareLists(List<string> list1, List<string> list2)
private bool CompareLists(HashSet<string> list1, HashSet<string> list2)
{
if (list1.Count != list2.Count)
return false;
for (int i = 0; i < list1.Count; i++)
{
if (!string.Equals(list1[i], list2[i], StringComparison.OrdinalIgnoreCase))
if (!string.Equals(list1.ElementAt(i), list2.ElementAt(i), StringComparison.OrdinalIgnoreCase))
return false;
}

8
MareSynchronos/Utils/Logger.cs
View File

@@ -74,17 +74,17 @@ internal class Logger : ILogger
switch (logLevel)
{
case LogLevel.Debug:
PluginLog.Debug($"[{_name}] [{eventId}] {formatter(state, exception)}");
Debug($"[{_name}] [{eventId}] {formatter(state, exception)}");
break;
case LogLevel.Error:
case LogLevel.Critical:
PluginLog.Error($"[{_name}] [{eventId}] {formatter(state, exception)}");
Error($"[{_name}] [{eventId}] {formatter(state, exception)}", exception);
break;
case LogLevel.Information:
PluginLog.Information($"[{_name}] [{eventId}] {formatter(state, exception)}");
Info($"[{_name}] [{eventId}] {formatter(state, exception)}");
break;
case LogLevel.Warning:
PluginLog.Warning($"[{_name}] [{eventId}] {formatter(state, exception)}");
Warn($"[{_name}] [{eventId}] {formatter(state, exception)}", exception);
break;
case LogLevel.Trace:
default:

215
MareSynchronos/WebAPI/ApIController.Functions.Files.cs
View File

@@ -10,6 +10,7 @@ using MareSynchronos.API.Data;
using MareSynchronos.API.Dto.Files;
using MareSynchronos.API.Routes;
using MareSynchronos.Mediator;
using MareSynchronos.UI;
using MareSynchronos.Utils;
using MareSynchronos.WebAPI.Utils;
using Microsoft.AspNetCore.SignalR.Client;
@@ -20,17 +21,23 @@ public partial class ApiController
{
private readonly Dictionary<string, DateTime> _verifiedUploadedHashes;
private readonly ConcurrentDictionary<Guid, bool> _downloadReady = new();
private bool currentUploadCancelled = false;
private int _downloadId = 0;
public async void CancelUpload()
public async Task<bool> CancelUpload()
{
if (_uploadCancellationTokenSource != null)
if (CurrentUploads.Any())
{
Logger.Debug("Cancelling upload");
Logger.Debug("Cancelling current upload");
_uploadCancellationTokenSource?.Cancel();
_uploadCancellationTokenSource?.Dispose();
_uploadCancellationTokenSource = null;
CurrentUploads.Clear();
await FilesAbortUpload().ConfigureAwait(false);
return true;
}
return false;
}
public async Task FilesAbortUpload()
@@ -333,19 +340,45 @@ public partial class ApiController
CancelDownload(currentDownloadId);
}
public async Task PushCharacterData(API.Data.CharacterData character, List<UserData> visibleCharacters)
public async Task PushCharacterData(CharacterData data, List<UserData> visibleCharacters)
{
if (!IsConnected) return;
Logger.Debug("Sending Character data to service " + _serverManager.CurrentApiUrl);
visibleCharacters = visibleCharacters.ToList();
CancelUpload();
_uploadCancellationTokenSource = new CancellationTokenSource();
var uploadToken = _uploadCancellationTokenSource.Token;
Logger.Verbose("New Token Created");
try
{
currentUploadCancelled = await CancelUpload().ConfigureAwait(false);
List<string> unverifiedUploadHashes = new();
foreach (var item in character.FileReplacements.SelectMany(c => c.Value.Where(f => string.IsNullOrEmpty(f.FileSwapPath)).Select(v => v.Hash).Distinct(StringComparer.Ordinal)).Distinct(StringComparer.Ordinal).ToList())
_uploadCancellationTokenSource = new CancellationTokenSource();
var uploadToken = _uploadCancellationTokenSource.Token;
Logger.Debug($"Sending Character data {data.DataHash.Value} to service {_serverManager.CurrentApiUrl}");
HashSet<string> unverifiedUploads = VerifyFiles(data);
if (unverifiedUploads.Any())
{
await UploadMissingFiles(unverifiedUploads, uploadToken).ConfigureAwait(false);
Logger.Info("Upload complete for " + data.DataHash.Value);
}
await PushCharacterDataInternal(data, visibleCharacters.ToList()).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
Logger.Debug("Upload operation was cancelled");
}
catch (Exception ex)
{
Logger.Warn("Error during upload of files", ex);
}
finally
{
if (!currentUploadCancelled)
currentUploadCancelled = await CancelUpload().ConfigureAwait(false);
}
}
private HashSet<string> VerifyFiles(CharacterData data)
{
HashSet<string> unverifiedUploadHashes = new(StringComparer.Ordinal);
foreach (var item in data.FileReplacements.SelectMany(c => c.Value.Where(f => string.IsNullOrEmpty(f.FileSwapPath)).Select(v => v.Hash).Distinct(StringComparer.Ordinal)).Distinct(StringComparer.Ordinal).ToList())
{
if (!_verifiedUploadedHashes.TryGetValue(item, out var verifiedTime))
{
@@ -359,103 +392,93 @@ public partial class ApiController
}
}
if (unverifiedUploadHashes.Any())
return unverifiedUploadHashes;
}
private async Task UploadMissingFiles(HashSet<string> unverifiedUploadHashes, CancellationToken uploadToken)
{
unverifiedUploadHashes = unverifiedUploadHashes.Where(h => _fileDbManager.GetFileCacheByHash(h) != null).ToHashSet(StringComparer.Ordinal);
Logger.Debug("Verifying " + unverifiedUploadHashes.Count + " files");
var filesToUpload = await FilesSend(unverifiedUploadHashes.ToList()).ConfigureAwait(false);
foreach (var file in filesToUpload.Where(f => !f.IsForbidden))
{
unverifiedUploadHashes = unverifiedUploadHashes.Where(h => _fileDbManager.GetFileCacheByHash(h) != null).ToList();
Logger.Debug("Verifying " + unverifiedUploadHashes.Count + " files");
var filesToUpload = await FilesSend(unverifiedUploadHashes).ConfigureAwait(false);
foreach (var file in filesToUpload.Where(f => !f.IsForbidden))
try
{
try
CurrentUploads.Add(new UploadFileTransfer(file)
{
CurrentUploads.Add(new UploadFileTransfer(file)
{
Total = new FileInfo(_fileDbManager.GetFileCacheByHash(file.Hash)!.ResolvedFilepath).Length,
});
}
catch (Exception ex)
{
Logger.Warn("Tried to request file " + file.Hash + " but file was not present");
Logger.Warn(ex.StackTrace!);
}
Total = new FileInfo(_fileDbManager.GetFileCacheByHash(file.Hash)!.ResolvedFilepath).Length,
});
}
foreach (var file in filesToUpload.Where(c => c.IsForbidden))
catch (Exception ex)
{
if (ForbiddenTransfers.All(f => !string.Equals(f.Hash, file.Hash, StringComparison.Ordinal)))
{
ForbiddenTransfers.Add(new UploadFileTransfer(file)
{
LocalFile = _fileDbManager.GetFileCacheByHash(file.Hash)?.ResolvedFilepath ?? string.Empty,
});
}
Logger.Warn("Tried to request file " + file.Hash + " but file was not present", ex);
}
var totalSize = CurrentUploads.Sum(c => c.Total);
Logger.Debug("Compressing and uploading files");
foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList())
{
Logger.Debug("Compressing and uploading " + file);
var data = await GetCompressedFileData(file.Hash, uploadToken).ConfigureAwait(false);
CurrentUploads.Single(e => string.Equals(e.Hash, data.Item1, StringComparison.Ordinal)).Total = data.Item2.Length;
await UploadFile(data.Item2, file.Hash, uploadToken).ConfigureAwait(false);
if (!uploadToken.IsCancellationRequested) continue;
Logger.Warn("Cancel in filesToUpload loop detected");
CurrentUploads.Clear();
break;
}
if (CurrentUploads.Any())
{
var compressedSize = CurrentUploads.Sum(c => c.Total);
Logger.Debug($"Compressed {totalSize} to {compressedSize} ({(compressedSize / (double)totalSize):P2})");
Logger.Debug("Upload tasks complete, waiting for server to confirm");
Logger.Debug("Uploads open: " + CurrentUploads.Any(c => c.IsInTransfer));
const double waitStep = 1.0d;
while (CurrentUploads.Any(c => c.IsInTransfer) && !uploadToken.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(waitStep), uploadToken).ConfigureAwait(false);
Logger.Debug("Waiting for uploads to finish");
}
}
foreach (var item in unverifiedUploadHashes)
{
_verifiedUploadedHashes[item] = DateTime.UtcNow;
}
CurrentUploads.Clear();
}
else
{
Logger.Debug("All files already verified");
}
if (!uploadToken.IsCancellationRequested)
foreach (var file in filesToUpload.Where(c => c.IsForbidden))
{
Logger.Info("Pushing character data for " + character.GetHashCode() + " to " + string.Join(", ", visibleCharacters.Select(c => c.AliasOrUID)));
StringBuilder sb = new();
foreach (var item in character.FileReplacements)
if (ForbiddenTransfers.All(f => !string.Equals(f.Hash, file.Hash, StringComparison.Ordinal)))
{
sb.AppendLine($"FileReplacements for {item.Key}: {item.Value.Count}");
ForbiddenTransfers.Add(new UploadFileTransfer(file)
{
LocalFile = _fileDbManager.GetFileCacheByHash(file.Hash)?.ResolvedFilepath ?? string.Empty,
});
}
foreach (var item in character.GlamourerData)
{
sb.AppendLine($"GlamourerData for {item.Key}: {!string.IsNullOrEmpty(item.Value)}");
}
Logger.Debug("Chara data contained: " + Environment.NewLine + sb.ToString());
await UserPushData(new(visibleCharacters, character)).ConfigureAwait(false);
}
else
{
Logger.Warn("=== Upload operation was cancelled ===");
_verifiedUploadedHashes[file.Hash] = DateTime.UtcNow;
}
Logger.Verbose("Upload complete for " + character.DataHash);
_uploadCancellationTokenSource = null;
var totalSize = CurrentUploads.Sum(c => c.Total);
Logger.Debug("Compressing and uploading files");
foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList())
{
Logger.Debug("Compressing and uploading " + file);
var data = await GetCompressedFileData(file.Hash, uploadToken).ConfigureAwait(false);
CurrentUploads.Single(e => string.Equals(e.Hash, data.Item1, StringComparison.Ordinal)).Total = data.Item2.Length;
await UploadFile(data.Item2, file.Hash, uploadToken).ConfigureAwait(false);
_verifiedUploadedHashes[file.Hash] = DateTime.UtcNow;
uploadToken.ThrowIfCancellationRequested();
}
if (CurrentUploads.Any())
{
var compressedSize = CurrentUploads.Sum(c => c.Total);
Logger.Debug($"Compressed {UiShared.ByteToString(totalSize)} to {UiShared.ByteToString(compressedSize)} ({(compressedSize / (double)totalSize):P2})");
Logger.Debug("Upload tasks complete, waiting for server to confirm");
Logger.Debug("Uploads open: " + CurrentUploads.Any(c => c.IsInTransfer));
const double waitStep = 1.0d;
while (CurrentUploads.Any(c => c.IsInTransfer) && !uploadToken.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(waitStep), uploadToken).ConfigureAwait(false);
Logger.Debug("Waiting for uploads to finish");
}
}
foreach(var file in unverifiedUploadHashes.Where(c=>!CurrentUploads.Any(u=> string.Equals(u.Hash, c, StringComparison.Ordinal))))
{
_verifiedUploadedHashes[file] = DateTime.UtcNow;
}
CurrentUploads.Clear();
}
private async Task PushCharacterDataInternal(CharacterData character, List<UserData> visibleCharacters)
{
Logger.Info("Pushing character data for " + character.DataHash.Value + " to " + string.Join(", ", visibleCharacters.Select(c => c.AliasOrUID)));
StringBuilder sb = new();
foreach (var item in character.FileReplacements)
{
sb.AppendLine($"FileReplacements for {item.Key}: {item.Value.Count}");
}
foreach (var item in character.GlamourerData)
{
sb.AppendLine($"GlamourerData for {item.Key}: {!string.IsNullOrEmpty(item.Value)}");
}
Logger.Debug("Chara data contained: " + Environment.NewLine + sb.ToString());
await UserPushData(new(visibleCharacters, character)).ConfigureAwait(false);
}
private async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken)

24
MareSynchronos/WebAPI/ApiController.cs
View File

@@ -204,9 +204,7 @@ public partial class ApiController : MediatorSubscriberBase, IDisposable, IMareH
}
catch (HttpRequestException ex)
{
Logger.Warn(ex.GetType().ToString());
Logger.Warn(ex.Message);
Logger.Warn(ex.StackTrace ?? string.Empty);
Logger.Warn("HttpRequestException on Connection", ex);
if (ex.StatusCode == System.Net.HttpStatusCode.Unauthorized)
{
@@ -222,9 +220,8 @@ public partial class ApiController : MediatorSubscriberBase, IDisposable, IMareH
}
catch (Exception ex)
{
Logger.Warn(ex.GetType().ToString());
Logger.Warn(ex.Message);
Logger.Warn(ex.StackTrace ?? string.Empty);
Logger.Warn("Exception on Connection", ex);
Logger.Info("Failed to establish connection, retrying");
await Task.Delay(TimeSpan.FromSeconds(new Random().Next(5, 20)), token).ConfigureAwait(false);
}
@@ -236,14 +233,8 @@ public partial class ApiController : MediatorSubscriberBase, IDisposable, IMareH
while (!ct.IsCancellationRequested && _mareHub != null)
{
await Task.Delay(TimeSpan.FromSeconds(30), ct).ConfigureAwait(false);
if (ct.IsCancellationRequested) break;
var needsRestart = await CheckClientHealth().ConfigureAwait(false);
Logger.Debug("Checked Client Health State, healthy: " + !needsRestart);
if (needsRestart)
{
ServerState = ServerState.Offline;
_ = CreateConnections();
}
Logger.Debug("Checked Client Health State");
}
}
@@ -328,7 +319,7 @@ public partial class ApiController : MediatorSubscriberBase, IDisposable, IMareH
.ConfigureLogging(a =>
{
a.ClearProviders().AddProvider(new DalamudLoggingProvider());
a.SetMinimumLevel(LogLevel.Warning);
a.SetMinimumLevel(LogLevel.Information);
})
.Build();
}
@@ -387,11 +378,6 @@ public partial class ApiController : MediatorSubscriberBase, IDisposable, IMareH
}
}
public async Task<ConnectionDto> Heartbeat(string characterIdentification)
{
return await _mareHub!.InvokeAsync<ConnectionDto>(nameof(Heartbeat), characterIdentification).ConfigureAwait(false);
}
public async Task<ConnectionDto> GetConnectionDto()
{
return await _mareHub!.InvokeAsync<ConnectionDto>(nameof(GetConnectionDto)).ConfigureAwait(false);