remove Penumbra dependency and use nuget, minor internal refactoring of structs, more resilent downloads

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "MareAPI"]
 	path = MareAPI
 	url = https://github.com/Penumbra-Sync/api.git
-[submodule "Penumbra"]
-	path = Penumbra
-	url = https://github.com/xivdev/Penumbra.git

MareSynchronos.sln

@@ -7,8 +7,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MareSynchronos", "MareSynch
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MareSynchronos.API", "MareAPI\MareSynchronosAPI\MareSynchronos.API.csproj", "{5A0B7434-8D89-4E90-B55C-B4A7AE1A6ADE}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Penumbra.GameData", "Penumbra\Penumbra.GameData\Penumbra.GameData.csproj", "{89DD407C-B2B7-4BB3-BF26-C550BA1841F8}"
-EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{585B740D-BA2C-429B-9CF3-B2D223423748}"
 	ProjectSection(SolutionItems) = preProject
 		.editorconfig = .editorconfig
@@ -38,14 +36,6 @@ Global
 		{5A0B7434-8D89-4E90-B55C-B4A7AE1A6ADE}.Release|Any CPU.Build.0 = Release|Any CPU
 		{5A0B7434-8D89-4E90-B55C-B4A7AE1A6ADE}.Release|x64.ActiveCfg = Release|Any CPU
 		{5A0B7434-8D89-4E90-B55C-B4A7AE1A6ADE}.Release|x64.Build.0 = Release|Any CPU
-		{89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Debug|x64.Build.0 = Debug|Any CPU
-		{89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Release|Any CPU.Build.0 = Release|Any CPU
-		{89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Release|x64.ActiveCfg = Release|Any CPU
-		{89DD407C-B2B7-4BB3-BF26-C550BA1841F8}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

MareSynchronos/Factories/CharacterDataFactory.cs

@@ -12,9 +12,9 @@ using MareSynchronos.Interop;
 using MareSynchronos.Managers;
 using MareSynchronos.Models;
 using MareSynchronos.Utils;
-using Penumbra.GameData.ByteString;
 using Penumbra.Interop.Structs;
 using Object = FFXIVClientStructs.FFXIV.Client.Graphics.Scene.Object;
+using Penumbra.String;
 using Weapon = MareSynchronos.Interop.Weapon;
 
 namespace MareSynchronos.Factories;
@@ -116,7 +116,7 @@ public class CharacterDataFactory
         string mdlPath;
         try
         {
-            mdlPath = new Utf8String(mdl->ResourceHandle->FileName()).ToString();
+            mdlPath = new ByteString(mdl->ResourceHandle->FileName()).ToString();
         }
         catch
         {
@@ -144,7 +144,7 @@ public class CharacterDataFactory
         string fileName;
         try
         {
-            fileName = new Utf8String(mtrl->ResourceHandle->FileName()).ToString();
+            fileName = new ByteString(mtrl->ResourceHandle->FileName()).ToString();
 
         }
         catch
@@ -175,7 +175,7 @@ public class CharacterDataFactory
             string? texPath = null;
             try
             {
-                texPath = new Utf8String(mtrlResourceHandle->TexString(resIdx)).ToString();
+                texPath = new ByteString(mtrlResourceHandle->TexString(resIdx)).ToString();
             }
             catch
             {
@@ -191,7 +191,7 @@ public class CharacterDataFactory
 
         try
         {
-            var shpkPath = "shader/sm5/shpk/" + new Utf8String(mtrlResourceHandle->ShpkString).ToString();
+            var shpkPath = "shader/sm5/shpk/" + new ByteString(mtrlResourceHandle->ShpkString).ToString();
             Logger.Verbose("Checking File Replacement for Shader " + shpkPath);
             AddReplacementsFromShader(shpkPath, objectKind, cache, inheritanceLevel + 1);
         }
@@ -412,7 +412,7 @@ public class CharacterDataFactory
         AddReplacementSkeleton(((HumanExt*)human)->Human.RaceSexId, objectKind, previousData);
         try
         {
-            AddReplacementsFromTexture(new Utf8String(((HumanExt*)human)->Decal->FileName()).ToString(), objectKind, previousData, 0, false);
+            AddReplacementsFromTexture(new ByteString(((HumanExt*)human)->Decal->FileName()).ToString(), objectKind, previousData, 0, false);
         }
         catch
         {
@@ -420,7 +420,7 @@ public class CharacterDataFactory
         }
         try
         {
-            AddReplacementsFromTexture(new Utf8String(((HumanExt*)human)->LegacyBodyDecal->FileName()).ToString(), objectKind, previousData, 0, false);
+            AddReplacementsFromTexture(new ByteString(((HumanExt*)human)->LegacyBodyDecal->FileName()).ToString(), objectKind, previousData, 0, false);
         }
         catch
         {

MareSynchronos/Interop/Material.cs

@@ -8,18 +8,6 @@ public unsafe struct Material
 {
     [FieldOffset( 0x10 )]
     public ResourceHandle* ResourceHandle;
-
-    [FieldOffset( 0x28 )]
-    public MaterialData* MaterialData;
-
-    [FieldOffset( 0x48 )]
-    public Texture* Tex1;
-
-    [FieldOffset( 0x60 )]
-    public Texture* Tex2;
-
-    [FieldOffset( 0x78 )]
-    public Texture* Tex3;
 }
 
 [StructLayout( LayoutKind.Explicit )]

MareSynchronos/Interop/MtrlResource.cs

@@ -1,3 +1,4 @@
+using FFXIVClientStructs.FFXIV.Client.System.Resource.Handle;
 using System.Runtime.InteropServices;
 
 namespace Penumbra.Interop.Structs;

MareSynchronos/Interop/RenderModel.cs

@@ -1,5 +1,6 @@
 using System.Runtime.InteropServices;
 using FFXIVClientStructs.FFXIV.Client.Graphics.Render;
+using FFXIVClientStructs.FFXIV.Client.System.Resource.Handle;
 
 namespace Penumbra.Interop.Structs;
 

MareSynchronos/Interop/ResourceHandle.cs

@@ -1,26 +1,12 @@
 using System;
 using System.Runtime.InteropServices;
 using FFXIVClientStructs.FFXIV.Client.System.Resource;
-using Penumbra.GameData.Enums;
 
 namespace Penumbra.Interop.Structs;
 
 [StructLayout( LayoutKind.Explicit )]
 public unsafe struct ResourceHandle
 {
-    [StructLayout( LayoutKind.Explicit )]
-    public struct DataIndirection
-    {
-        [FieldOffset( 0x00 )]
-        public void** VTable;
-
-        [FieldOffset( 0x10 )]
-        public byte* DataPtr;
-
-        [FieldOffset( 0x28 )]
-        public ulong DataLength;
-    }
-
     public const int SsoSize = 15;
 
     public byte* FileName()
@@ -36,61 +22,12 @@ public unsafe struct ResourceHandle
         }
     }
 
-    public ReadOnlySpan< byte > FileNameSpan()
-        => new(FileName(), FileNameLength);
-
-    [FieldOffset( 0x00 )]
-    public void** VTable;
-
     [FieldOffset( 0x08 )]
     public ResourceCategory Category;
 
-    [FieldOffset( 0x0C )]
-    public ResourceType FileType;
-
-    [FieldOffset( 0x10 )]
-    public uint Id;
-
     [FieldOffset( 0x48 )]
     public byte* FileNameData;
 
     [FieldOffset( 0x58 )]
     public int FileNameLength;
-
-    [FieldOffset( 0xAC )]
-    public uint RefCount;
-
-    // May return null.
-    public static byte* GetData( ResourceHandle* handle )
-        => ( ( delegate* unmanaged< ResourceHandle*, byte* > )handle->VTable[ 23 ] )( handle );
-
-    public static ulong GetLength( ResourceHandle* handle )
-        => ( ( delegate* unmanaged< ResourceHandle*, ulong > )handle->VTable[ 17 ] )( handle );
-
-
-    // Only use these if you know what you are doing.
-    // Those are actually only sure to be accessible for DefaultResourceHandles.
-    [FieldOffset( 0xB0 )]
-    public DataIndirection* Data;
-
-    [FieldOffset( 0xB8 )]
-    public uint DataLength;
-
-    public (IntPtr Data, int Length) GetData()
-        => Data != null
-            ? ( ( IntPtr )Data->DataPtr, ( int )Data->DataLength )
-            : ( IntPtr.Zero, 0 );
-
-    public bool SetData( IntPtr data, int length )
-    {
-        if( Data == null )
-        {
-            return false;
-        }
-
-        Data->DataPtr    = length != 0 ? ( byte* )data : null;
-        Data->DataLength = ( ulong )length;
-        DataLength       = ( uint )length;
-        return true;
-    }
 }

MareSynchronos/Interop/Weapon.cs

@@ -2,6 +2,7 @@
 using System.Runtime.InteropServices;
 using FFXIVClientStructs.FFXIV.Client.Game.Character;
 using FFXIVClientStructs.FFXIV.Client.Graphics.Scene;
+using FFXIVClientStructs.FFXIV.Client.System.Resource.Handle;
 using Penumbra.Interop.Structs;
 
 namespace MareSynchronos.Interop;
@@ -25,8 +26,8 @@ public unsafe struct WeaponDrawObject
 public unsafe struct HumanExt
 {
     [FieldOffset(0x0)] public Human Human;
-    [FieldOffset(0x9E8)] public ResourceHandle* Decal;
+    [FieldOffset(0x9E8)] public Penumbra.Interop.Structs.ResourceHandle* Decal;
-    [FieldOffset(0x9F0)] public ResourceHandle* LegacyBodyDecal;
+    [FieldOffset(0x9F0)] public Penumbra.Interop.Structs.ResourceHandle* LegacyBodyDecal;
 }
 
 [StructLayout(LayoutKind.Explicit)]

MareSynchronos/MareSynchronos.csproj

@@ -33,6 +33,8 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="6.0.8" />
+    <PackageReference Include="Penumbra.Api" Version="1.0.3" />
+    <PackageReference Include="Penumbra.String" Version="1.0.0" />
   </ItemGroup>
 
   <PropertyGroup>
@@ -41,7 +43,6 @@
 
   <ItemGroup>
     <ProjectReference Include="..\MareAPI\MareSynchronosAPI\MareSynchronos.API.csproj" />
-    <ProjectReference Include="..\Penumbra\Penumbra.GameData\Penumbra.GameData.csproj" />
   </ItemGroup>
 
   <ItemGroup>

MareSynchronos/Models/PlayerRelatedObject.cs

@@ -3,7 +3,7 @@ using MareSynchronos.API;
 using FFXIVClientStructs.FFXIV.Client.Game.Character;
 using System.Runtime.InteropServices;
 using MareSynchronos.Utils;
-using Penumbra.GameData.ByteString;
+using Penumbra.String;
 
 namespace MareSynchronos.Models;
 
@@ -60,7 +60,7 @@ public class PlayerRelatedObject
             bool addr = Address == IntPtr.Zero || Address != curPtr;
             bool equip = CompareAndUpdateByteData(chara->EquipSlotData, chara->CustomizeData);
             bool drawObj = (IntPtr)chara->GameObject.DrawObject != DrawObjectAddress;
-            var name = new Utf8String(chara->GameObject.Name).ToString();
+            var name = new ByteString(chara->GameObject.Name).ToString();
             bool nameChange = (!string.Equals(name, _name, StringComparison.Ordinal));
             if (addr || equip || drawObj || nameChange)
             {

MareSynchronos/Utils/Logger.cs

@@ -2,6 +2,7 @@
 using System.Diagnostics;
 using Dalamud.Logging;
 using Dalamud.Utility;
+using Lumina.Excel.GeneratedSheets;
 using Microsoft.Extensions.Logging;
 
 namespace MareSynchronos.Utils;
@@ -29,6 +30,12 @@ internal class Logger : ILogger
         }
     }
 
+    public static void Warn(string msg, Exception ex)
+    {
+        var caller = new StackTrace().GetFrame(1)?.GetMethod()?.ReflectedType?.Name ?? "Unknown";
+        PluginLog.Warning($"[{caller}] {msg} {Environment.NewLine} Exception: {ex.Message} {Environment.NewLine} {ex.StackTrace}");
+    }
+
     public static void Warn(string warn)
     {
         var caller = new StackTrace().GetFrame(1)?.GetMethod()?.ReflectedType?.Name ?? "Unknown";

MareSynchronos/WebAPI/ApIController.Functions.Files.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Net;
 using System.Net.Http;
 using System.Runtime.CompilerServices;
 using System.Text;
@@ -45,9 +46,37 @@ public partial class ApiController
     {
         using var client = new HttpClient();
         client.DefaultRequestHeaders.Add("Authorization", SecretKey);
+        int attempts = 0;
+        bool failed = true;
+        const int maxAttempts = 10;
 
-        var response = await client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
+        HttpResponseMessage response = null!;
-        response.EnsureSuccessStatusCode();
+        HttpStatusCode? lastError = HttpStatusCode.OK;
+        while (failed && attempts < maxAttempts && !ct.IsCancellationRequested)
+        {
+            try
+            {
+                response = await client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
+                response.EnsureSuccessStatusCode();
+                failed = false;
+            }
+            catch (HttpRequestException ex)
+            {
+                Logger.Warn($"Attempt {attempts}: Error during download of {url}, HttpStatusCode: {ex.StatusCode}");
+                lastError = ex.StatusCode;
+                if (ex.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Unauthorized)
+                {
+                    break;
+                }
+                attempts++;
+                await Task.Delay(TimeSpan.FromSeconds(new Random().Next(1, 5)), ct).ConfigureAwait(false);
+            }
+        }
+
+        if (failed)
+        {
+            throw new Exception($"Http error {lastError} after {maxAttempts} attempts (cancelled: {ct.IsCancellationRequested}): {url}");
+        }
 
         var fileName = Path.GetTempFileName();
         try
@@ -68,11 +97,13 @@ public partial class ApiController
                     progress.Report(bytesRead);
                 }
 
+                Logger.Debug($"{url} downloaded to {fileName}");
                 return fileName;
             }
         }
-        catch
+        catch (Exception ex)
         {
+            Logger.Warn($"Error during file download of {url}", ex);
             try
             {
                 File.Delete(fileName);
@@ -123,7 +154,7 @@ public partial class ApiController
 
         await Parallel.ForEachAsync(CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred), new ParallelOptions()
         {
-            MaxDegreeOfParallelism = 5,
+            MaxDegreeOfParallelism = 2,
             CancellationToken = ct
         },
         async (file, token) =>