From ffd896168cc314da49b32112a3a3e8893af14116 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Sun, 18 Dec 2022 13:00:34 +0100 Subject: [PATCH] remove Penumbra dependency and use nuget, minor internal refactoring of structs, more resilent downloads --- .gitmodules | 3 - MareSynchronos.sln | 10 --- .../Factories/CharacterDataFactory.cs | 14 ++--- MareSynchronos/Interop/Material.cs | 12 ---- MareSynchronos/Interop/MtrlResource.cs | 1 + MareSynchronos/Interop/RenderModel.cs | 1 + MareSynchronos/Interop/ResourceHandle.cs | 63 ------------------- MareSynchronos/Interop/Weapon.cs | 5 +- MareSynchronos/MareSynchronos.csproj | 3 +- MareSynchronos/Models/PlayerRelatedObject.cs | 4 +- MareSynchronos/Utils/Logger.cs | 7 +++ .../WebAPI/ApIController.Functions.Files.cs | 39 ++++++++++-- Penumbra | 1 - 13 files changed, 58 insertions(+), 105 deletions(-) delete mode 160000 Penumbra diff --git a/.gitmodules b/.gitmodules index a9931b0..938d8ea 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "MareAPI"] path = MareAPI url = https://github.com/Penumbra-Sync/api.git -[submodule "Penumbra"] - path = Penumbra - url = https://github.com/xivdev/Penumbra.git diff --git a/MareSynchronos.sln b/MareSynchronos.sln index f38743f..ec5ad79 100644 --- a/MareSynchronos.sln +++ b/MareSynchronos.sln @@ -7,8 +7,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MareSynchronos", "MareSynch EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MareSynchronos.API", "MareAPI\MareSynchronosAPI\MareSynchronos.API.csproj", "{5A0B7434-8D89-4E90-B55C-B4A7AE1A6ADE}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Penumbra.GameData", "Penumbra\Penumbra.GameData\Penumbra.GameData.csproj", "{89DD407C-B2B7-4BB3-BF26-C550BA1841F8}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{585B740D-BA2C-429B-9CF3-B2D223423748}" ProjectSection(SolutionItems) = preProject .editorconfig = .editorconfig @@ -38,14 +36,6 @@ Global {5A0B7434-8D89-4E90-B55C-B4A7AE1A6ADE}.Release|Any CPU.Build.0 = Release|Any CPU {5A0B7434-8D89-4E90-B55C-B4A7AE1A6ADE}.Release|x64.ActiveCfg = Release|Any CPU {5A0B7434-8D89-4E90-B55C-B4A7AE1A6ADE}.Release|x64.Build.0 = Release|Any CPU - {89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Debug|Any CPU.Build.0 = Debug|Any CPU - {89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Debug|x64.ActiveCfg = Debug|Any CPU - {89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Debug|x64.Build.0 = Debug|Any CPU - {89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Release|Any CPU.ActiveCfg = Release|Any CPU - {89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Release|Any CPU.Build.0 = Release|Any CPU - {89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Release|x64.ActiveCfg = Release|Any CPU - {89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/MareSynchronos/Factories/CharacterDataFactory.cs b/MareSynchronos/Factories/CharacterDataFactory.cs index 3678393..4529427 100644 --- a/MareSynchronos/Factories/CharacterDataFactory.cs +++ b/MareSynchronos/Factories/CharacterDataFactory.cs @@ -12,9 +12,9 @@ using MareSynchronos.Interop; using MareSynchronos.Managers; using MareSynchronos.Models; using MareSynchronos.Utils; -using Penumbra.GameData.ByteString; using Penumbra.Interop.Structs; using Object = FFXIVClientStructs.FFXIV.Client.Graphics.Scene.Object; +using Penumbra.String; using Weapon = MareSynchronos.Interop.Weapon; namespace MareSynchronos.Factories; @@ -116,7 +116,7 @@ public class CharacterDataFactory string mdlPath; try { - mdlPath = new Utf8String(mdl->ResourceHandle->FileName()).ToString(); + mdlPath = new ByteString(mdl->ResourceHandle->FileName()).ToString(); } catch { @@ -144,7 +144,7 @@ public class CharacterDataFactory string fileName; try { - fileName = new Utf8String(mtrl->ResourceHandle->FileName()).ToString(); + fileName = new ByteString(mtrl->ResourceHandle->FileName()).ToString(); } catch @@ -175,7 +175,7 @@ public class CharacterDataFactory string? texPath = null; try { - texPath = new Utf8String(mtrlResourceHandle->TexString(resIdx)).ToString(); + texPath = new ByteString(mtrlResourceHandle->TexString(resIdx)).ToString(); } catch { @@ -191,7 +191,7 @@ public class CharacterDataFactory try { - var shpkPath = "shader/sm5/shpk/" + new Utf8String(mtrlResourceHandle->ShpkString).ToString(); + var shpkPath = "shader/sm5/shpk/" + new ByteString(mtrlResourceHandle->ShpkString).ToString(); Logger.Verbose("Checking File Replacement for Shader " + shpkPath); AddReplacementsFromShader(shpkPath, objectKind, cache, inheritanceLevel + 1); } @@ -412,7 +412,7 @@ public class CharacterDataFactory AddReplacementSkeleton(((HumanExt*)human)->Human.RaceSexId, objectKind, previousData); try { - AddReplacementsFromTexture(new Utf8String(((HumanExt*)human)->Decal->FileName()).ToString(), objectKind, previousData, 0, false); + AddReplacementsFromTexture(new ByteString(((HumanExt*)human)->Decal->FileName()).ToString(), objectKind, previousData, 0, false); } catch { @@ -420,7 +420,7 @@ public class CharacterDataFactory } try { - AddReplacementsFromTexture(new Utf8String(((HumanExt*)human)->LegacyBodyDecal->FileName()).ToString(), objectKind, previousData, 0, false); + AddReplacementsFromTexture(new ByteString(((HumanExt*)human)->LegacyBodyDecal->FileName()).ToString(), objectKind, previousData, 0, false); } catch { diff --git a/MareSynchronos/Interop/Material.cs b/MareSynchronos/Interop/Material.cs index 963b569..3be3212 100644 --- a/MareSynchronos/Interop/Material.cs +++ b/MareSynchronos/Interop/Material.cs @@ -8,18 +8,6 @@ public unsafe struct Material { [FieldOffset( 0x10 )] public ResourceHandle* ResourceHandle; - - [FieldOffset( 0x28 )] - public MaterialData* MaterialData; - - [FieldOffset( 0x48 )] - public Texture* Tex1; - - [FieldOffset( 0x60 )] - public Texture* Tex2; - - [FieldOffset( 0x78 )] - public Texture* Tex3; } [StructLayout( LayoutKind.Explicit )] diff --git a/MareSynchronos/Interop/MtrlResource.cs b/MareSynchronos/Interop/MtrlResource.cs index ff5b6ab..21ff22a 100644 --- a/MareSynchronos/Interop/MtrlResource.cs +++ b/MareSynchronos/Interop/MtrlResource.cs @@ -1,3 +1,4 @@ +using FFXIVClientStructs.FFXIV.Client.System.Resource.Handle; using System.Runtime.InteropServices; namespace Penumbra.Interop.Structs; diff --git a/MareSynchronos/Interop/RenderModel.cs b/MareSynchronos/Interop/RenderModel.cs index b9e0490..db84aa7 100644 --- a/MareSynchronos/Interop/RenderModel.cs +++ b/MareSynchronos/Interop/RenderModel.cs @@ -1,5 +1,6 @@ using System.Runtime.InteropServices; using FFXIVClientStructs.FFXIV.Client.Graphics.Render; +using FFXIVClientStructs.FFXIV.Client.System.Resource.Handle; namespace Penumbra.Interop.Structs; diff --git a/MareSynchronos/Interop/ResourceHandle.cs b/MareSynchronos/Interop/ResourceHandle.cs index 6b3a8a7..d4284ba 100644 --- a/MareSynchronos/Interop/ResourceHandle.cs +++ b/MareSynchronos/Interop/ResourceHandle.cs @@ -1,26 +1,12 @@ using System; using System.Runtime.InteropServices; using FFXIVClientStructs.FFXIV.Client.System.Resource; -using Penumbra.GameData.Enums; namespace Penumbra.Interop.Structs; [StructLayout( LayoutKind.Explicit )] public unsafe struct ResourceHandle { - [StructLayout( LayoutKind.Explicit )] - public struct DataIndirection - { - [FieldOffset( 0x00 )] - public void** VTable; - - [FieldOffset( 0x10 )] - public byte* DataPtr; - - [FieldOffset( 0x28 )] - public ulong DataLength; - } - public const int SsoSize = 15; public byte* FileName() @@ -36,61 +22,12 @@ public unsafe struct ResourceHandle } } - public ReadOnlySpan< byte > FileNameSpan() - => new(FileName(), FileNameLength); - - [FieldOffset( 0x00 )] - public void** VTable; - [FieldOffset( 0x08 )] public ResourceCategory Category; - [FieldOffset( 0x0C )] - public ResourceType FileType; - - [FieldOffset( 0x10 )] - public uint Id; - [FieldOffset( 0x48 )] public byte* FileNameData; [FieldOffset( 0x58 )] public int FileNameLength; - - [FieldOffset( 0xAC )] - public uint RefCount; - - // May return null. - public static byte* GetData( ResourceHandle* handle ) - => ( ( delegate* unmanaged< ResourceHandle*, byte* > )handle->VTable[ 23 ] )( handle ); - - public static ulong GetLength( ResourceHandle* handle ) - => ( ( delegate* unmanaged< ResourceHandle*, ulong > )handle->VTable[ 17 ] )( handle ); - - - // Only use these if you know what you are doing. - // Those are actually only sure to be accessible for DefaultResourceHandles. - [FieldOffset( 0xB0 )] - public DataIndirection* Data; - - [FieldOffset( 0xB8 )] - public uint DataLength; - - public (IntPtr Data, int Length) GetData() - => Data != null - ? ( ( IntPtr )Data->DataPtr, ( int )Data->DataLength ) - : ( IntPtr.Zero, 0 ); - - public bool SetData( IntPtr data, int length ) - { - if( Data == null ) - { - return false; - } - - Data->DataPtr = length != 0 ? ( byte* )data : null; - Data->DataLength = ( ulong )length; - DataLength = ( uint )length; - return true; - } } \ No newline at end of file diff --git a/MareSynchronos/Interop/Weapon.cs b/MareSynchronos/Interop/Weapon.cs index b02cbf2..4c84aeb 100644 --- a/MareSynchronos/Interop/Weapon.cs +++ b/MareSynchronos/Interop/Weapon.cs @@ -2,6 +2,7 @@ using System.Runtime.InteropServices; using FFXIVClientStructs.FFXIV.Client.Game.Character; using FFXIVClientStructs.FFXIV.Client.Graphics.Scene; +using FFXIVClientStructs.FFXIV.Client.System.Resource.Handle; using Penumbra.Interop.Structs; namespace MareSynchronos.Interop; @@ -25,8 +26,8 @@ public unsafe struct WeaponDrawObject public unsafe struct HumanExt { [FieldOffset(0x0)] public Human Human; - [FieldOffset(0x9E8)] public ResourceHandle* Decal; - [FieldOffset(0x9F0)] public ResourceHandle* LegacyBodyDecal; + [FieldOffset(0x9E8)] public Penumbra.Interop.Structs.ResourceHandle* Decal; + [FieldOffset(0x9F0)] public Penumbra.Interop.Structs.ResourceHandle* LegacyBodyDecal; } [StructLayout(LayoutKind.Explicit)] diff --git a/MareSynchronos/MareSynchronos.csproj b/MareSynchronos/MareSynchronos.csproj index 8e3b123..438aa45 100644 --- a/MareSynchronos/MareSynchronos.csproj +++ b/MareSynchronos/MareSynchronos.csproj @@ -33,6 +33,8 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + + @@ -41,7 +43,6 @@ - diff --git a/MareSynchronos/Models/PlayerRelatedObject.cs b/MareSynchronos/Models/PlayerRelatedObject.cs index 50ad1b8..db375d1 100644 --- a/MareSynchronos/Models/PlayerRelatedObject.cs +++ b/MareSynchronos/Models/PlayerRelatedObject.cs @@ -3,7 +3,7 @@ using MareSynchronos.API; using FFXIVClientStructs.FFXIV.Client.Game.Character; using System.Runtime.InteropServices; using MareSynchronos.Utils; -using Penumbra.GameData.ByteString; +using Penumbra.String; namespace MareSynchronos.Models; @@ -60,7 +60,7 @@ public class PlayerRelatedObject bool addr = Address == IntPtr.Zero || Address != curPtr; bool equip = CompareAndUpdateByteData(chara->EquipSlotData, chara->CustomizeData); bool drawObj = (IntPtr)chara->GameObject.DrawObject != DrawObjectAddress; - var name = new Utf8String(chara->GameObject.Name).ToString(); + var name = new ByteString(chara->GameObject.Name).ToString(); bool nameChange = (!string.Equals(name, _name, StringComparison.Ordinal)); if (addr || equip || drawObj || nameChange) { diff --git a/MareSynchronos/Utils/Logger.cs b/MareSynchronos/Utils/Logger.cs index f816a5f..9384a54 100644 --- a/MareSynchronos/Utils/Logger.cs +++ b/MareSynchronos/Utils/Logger.cs @@ -2,6 +2,7 @@ using System.Diagnostics; using Dalamud.Logging; using Dalamud.Utility; +using Lumina.Excel.GeneratedSheets; using Microsoft.Extensions.Logging; namespace MareSynchronos.Utils; @@ -29,6 +30,12 @@ internal class Logger : ILogger } } + public static void Warn(string msg, Exception ex) + { + var caller = new StackTrace().GetFrame(1)?.GetMethod()?.ReflectedType?.Name ?? "Unknown"; + PluginLog.Warning($"[{caller}] {msg} {Environment.NewLine} Exception: {ex.Message} {Environment.NewLine} {ex.StackTrace}"); + } + public static void Warn(string warn) { var caller = new StackTrace().GetFrame(1)?.GetMethod()?.ReflectedType?.Name ?? "Unknown"; diff --git a/MareSynchronos/WebAPI/ApIController.Functions.Files.cs b/MareSynchronos/WebAPI/ApIController.Functions.Files.cs index 2ba127c..c00c2cb 100644 --- a/MareSynchronos/WebAPI/ApIController.Functions.Files.cs +++ b/MareSynchronos/WebAPI/ApIController.Functions.Files.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net; using System.Net.Http; using System.Runtime.CompilerServices; using System.Text; @@ -45,9 +46,37 @@ public partial class ApiController { using var client = new HttpClient(); client.DefaultRequestHeaders.Add("Authorization", SecretKey); + int attempts = 0; + bool failed = true; + const int maxAttempts = 10; - var response = await client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); + HttpResponseMessage response = null!; + HttpStatusCode? lastError = HttpStatusCode.OK; + while (failed && attempts < maxAttempts && !ct.IsCancellationRequested) + { + try + { + response = await client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + failed = false; + } + catch (HttpRequestException ex) + { + Logger.Warn($"Attempt {attempts}: Error during download of {url}, HttpStatusCode: {ex.StatusCode}"); + lastError = ex.StatusCode; + if (ex.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Unauthorized) + { + break; + } + attempts++; + await Task.Delay(TimeSpan.FromSeconds(new Random().Next(1, 5)), ct).ConfigureAwait(false); + } + } + + if (failed) + { + throw new Exception($"Http error {lastError} after {maxAttempts} attempts (cancelled: {ct.IsCancellationRequested}): {url}"); + } var fileName = Path.GetTempFileName(); try @@ -68,11 +97,13 @@ public partial class ApiController progress.Report(bytesRead); } + Logger.Debug($"{url} downloaded to {fileName}"); return fileName; } } - catch + catch (Exception ex) { + Logger.Warn($"Error during file download of {url}", ex); try { File.Delete(fileName); @@ -123,7 +154,7 @@ public partial class ApiController await Parallel.ForEachAsync(CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred), new ParallelOptions() { - MaxDegreeOfParallelism = 5, + MaxDegreeOfParallelism = 2, CancellationToken = ct }, async (file, token) => diff --git a/Penumbra b/Penumbra deleted file mode 160000 index e66ca7c..0000000 --- a/Penumbra +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e66ca7c5805bf23a82b51649e7f5f75baabe4bc7