Update for 6.3 and change to File Service Queue (#34)

* add request queue processing on clientside, switch to net7

* reuse httpclient

* dispose and renew httpclient on building connection

* add handling of QueueRequestDto to downloads

* group download by host and port

* refactor SendAsync to SendRequestAsync for readability

* add holding ctrl before clearing cache, add tooltip

* update pathing

* fixes to api usage in client

* fix rendermodel offsets

* fixes for downloads

Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>

MareSynchronos/Interop/RenderModel.cs

@@ -25,18 +25,18 @@ public unsafe struct RenderModel
     [FieldOffset( 0x60 )]
     public int BoneListCount;
 
-    [FieldOffset( 0x68 )]
+    [FieldOffset( 0x70 )]
     private void* UnkDXBuffer1;
 
-    [FieldOffset( 0x70 )]
+    [FieldOffset( 0x78 )]
     private void* UnkDXBuffer2;
 
-    [FieldOffset( 0x78 )]
+    [FieldOffset( 0x80 )]
     private void* UnkDXBuffer3;
 
-    [FieldOffset( 0x90 )]
+    [FieldOffset( 0x98 )]
     public void** Materials;
 
-    [FieldOffset( 0x98 )]
+    [FieldOffset( 0xA0 )]
     public int MaterialCount;
 }

MareSynchronos/MareSynchronos.csproj

@@ -10,7 +10,7 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <TargetFramework>net6.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
     <Platforms>x64</Platforms>
     <Nullable>enable</Nullable>
     <LangVersion>latest</LangVersion>
@@ -26,15 +26,15 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="DalamudPackager" Version="2.1.8" />
+    <PackageReference Include="DalamudPackager" Version="2.1.10" />
     <PackageReference Include="lz4net" Version="1.0.15.93" />
-    <PackageReference Include="Meziantou.Analyzer" Version="1.0.733">
+    <PackageReference Include="Meziantou.Analyzer" Version="2.0.4">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="6.0.8" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="7.0.1" />
-    <PackageReference Include="Penumbra.Api" Version="1.0.3" />
+    <PackageReference Include="Penumbra.Api" Version="1.0.5" />
-    <PackageReference Include="Penumbra.String" Version="1.0.0" />
+    <PackageReference Include="Penumbra.String" Version="1.0.1" />
   </ItemGroup>
 
   <PropertyGroup>

MareSynchronos/UI/SettingsUi.cs

@@ -669,16 +669,22 @@ public class SettingsUi : Window, IDisposable
         ImGui.SameLine();
         if (ImGui.Button("Clear local cache"))
         {
-            Task.Run(() =>
+            if (UiShared.CtrlPressed())
             {
-                foreach (var file in Directory.GetFiles(_configuration.CacheFolder))
+                Task.Run(() =>
                 {
-                    File.Delete(file);
+                    foreach (var file in Directory.GetFiles(_configuration.CacheFolder))
-                }
+                    {
+                        File.Delete(file);
+                    }
 
-                _uiShared.RecalculateFileCacheSize();
+                    _uiShared.RecalculateFileCacheSize();
-            });
+                });
+            }
         }
+        UiShared.AttachToolTip("You normally do not need to do this. This will solely remove all downloaded data from all players and will require you to re-download everything again." + Environment.NewLine
+            + "Mares Cache is self-clearing and will not surpass the limit you have set it to." + Environment.NewLine
+            + "If you still think you need to do this hold CTRL while pressing the button.");
     }
 
     public override void OnClose()

MareSynchronos/Utils/DalamudUtil.cs

@@ -177,21 +177,21 @@ public class DalamudUtil : IDisposable
 
     public unsafe IntPtr GetMinion()
     {
-        return (IntPtr)((FFXIVClientStructs.FFXIV.Client.Game.Character.Character*)PlayerPointer)->CompanionObject;
+        return (IntPtr)((Character*)PlayerPointer)->CompanionObject;
     }
 
     public unsafe IntPtr GetPet(IntPtr? playerPointer = null)
     {
         var mgr = CharacterManager.Instance();
         if (playerPointer == null) playerPointer = PlayerPointer;
-        return (IntPtr)mgr->LookupPetByOwnerObject((FFXIVClientStructs.FFXIV.Client.Game.Character.BattleChara*)playerPointer);
+        return (IntPtr)mgr->LookupPetByOwnerObject((BattleChara*)playerPointer);
     }
 
     public unsafe IntPtr GetCompanion(IntPtr? playerPointer = null)
     {
         var mgr = CharacterManager.Instance();
         if (playerPointer == null) playerPointer = PlayerPointer;
-        return (IntPtr)mgr->LookupBuddyByOwnerObject((FFXIVClientStructs.FFXIV.Client.Game.Character.BattleChara*)playerPointer);
+        return (IntPtr)mgr->LookupBuddyByOwnerObject((BattleChara*)playerPointer);
     }
 
     public string PlayerName => _clientState.LocalPlayer?.Name.ToString() ?? "--";

MareSynchronos/WebAPI/ApIController.Functions.Files.cs

@@ -4,6 +4,8 @@ using System.IO;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
 using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading;
@@ -13,6 +15,7 @@ using MareSynchronos.API;
 using MareSynchronos.Utils;
 using MareSynchronos.WebAPI.Utils;
 using Microsoft.AspNetCore.SignalR.Client;
+using Newtonsoft.Json;
 
 namespace MareSynchronos.WebAPI;
 
@@ -43,30 +46,81 @@ public partial class ApiController
         await _mareHub!.SendAsync(nameof(FilesDeleteAll)).ConfigureAwait(false);
     }
 
-    private async Task<string> DownloadFileHttpClient(Uri url, IProgress<long> progress, CancellationToken ct)
+    private async Task<QueueRequestDto> GetQueueRequestDto(DownloadFileTransfer downloadFileTransfer)
     {
-        using var client = new HttpClient();
+        var response = await SendRequestAsync<object>(HttpMethod.Get, MareFiles.RequestRequestFileFullPath(downloadFileTransfer.DownloadUri, downloadFileTransfer.Hash)).ConfigureAwait(false);
-        client.DefaultRequestHeaders.Add(AuthorizationJwtHeader.Key, AuthorizationJwtHeader.Value);
+        return JsonConvert.DeserializeObject<QueueRequestDto>(await response.Content.ReadAsStringAsync().ConfigureAwait(false))!;
+    }
+
+    private async Task<Guid> WaitForQueue(DownloadFileTransfer fileTransfer, Guid requestId, CancellationToken ct)
+    {
+        while (!ct.IsCancellationRequested)
+        {
+            await Task.Delay(250, ct).ConfigureAwait(false);
+            var queueResponse = await SendRequestAsync<object>(HttpMethod.Get, MareFiles.RequestCheckQueueFullPath(fileTransfer.DownloadUri, requestId)).ConfigureAwait(false);
+            try
+            {
+                queueResponse.EnsureSuccessStatusCode();
+                Logger.Debug($"Starting download for file {fileTransfer.Hash} ({requestId})");
+                break;
+            }
+            catch (HttpRequestException ex)
+            {
+                switch (ex.StatusCode)
+                {
+                    case HttpStatusCode.Conflict:
+                        Logger.Debug($"In queue for file {fileTransfer.Hash} ({requestId})");
+                        // still in queue
+                        break;
+                    case HttpStatusCode.BadRequest:
+                        // rerequest queue
+                        Logger.Debug($"Rerequesting {fileTransfer.Hash}");
+                        var dto = await GetQueueRequestDto(fileTransfer).ConfigureAwait(false);
+                        requestId = dto.RequestId;
+                        break;
+                    default:
+                        Logger.Warn($"Unclear response from server: {fileTransfer.Hash} ({requestId}): {ex.StatusCode}");
+                        break;
+                }
+
+                await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
+            }
+        }
+
+        return requestId;
+    }
+
+    private async Task<string> DownloadFileHttpClient(DownloadFileTransfer fileTransfer, IProgress<long> progress, CancellationToken ct)
+    {
+        var queueRequest = await GetQueueRequestDto(fileTransfer).ConfigureAwait(false);
+
+        Logger.Debug($"GUID {queueRequest.RequestId} for file {fileTransfer.Hash}, queue status {queueRequest.QueueStatus}");
+
+        var requestId = queueRequest.QueueStatus == QueueStatus.Ready
+            ? queueRequest.RequestId
+            : await WaitForQueue(fileTransfer, queueRequest.RequestId, ct).ConfigureAwait(false);
+
         int attempts = 1;
         bool failed = true;
         const int maxAttempts = 16;
 
         HttpResponseMessage response = null!;
         HttpStatusCode? lastError = HttpStatusCode.OK;
-        var bypassUrl = new Uri(url, "?nocache=" + DateTime.UtcNow.Ticks);
 
+        var requestUrl = MareFiles.CacheGetFullPath(fileTransfer.DownloadUri, requestId);
+
+        Logger.Debug($"Downloading {requestUrl} for file {fileTransfer.Hash}");
         while (failed && attempts < maxAttempts && !ct.IsCancellationRequested)
         {
             try
             {
-                response = await client.GetAsync(bypassUrl, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
+                response = await SendRequestAsync<object>(HttpMethod.Get, requestUrl, ct: ct).ConfigureAwait(false);
                 response.EnsureSuccessStatusCode();
                 failed = false;
             }
             catch (HttpRequestException ex)
             {
-                Logger.Warn($"Attempt {attempts}: Error during download of {bypassUrl}, HttpStatusCode: {ex.StatusCode}");
+                Logger.Warn($"Attempt {attempts}: Error during download of {requestUrl}, HttpStatusCode: {ex.StatusCode}");
-                bypassUrl = new Uri(url, "?nocache=" + DateTime.UtcNow.Ticks);
                 lastError = ex.StatusCode;
                 if (ex.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Unauthorized)
                 {
@@ -79,7 +133,7 @@ public partial class ApiController
 
         if (failed)
         {
-            throw new Exception($"Http error {lastError} after {maxAttempts} attempts (cancelled: {ct.IsCancellationRequested}): {url}");
+            throw new Exception($"Http error {lastError} after {maxAttempts} attempts (cancelled: {ct.IsCancellationRequested}): {requestUrl}");
         }
 
         var fileName = Path.GetTempFileName();
@@ -101,13 +155,13 @@ public partial class ApiController
                     progress.Report(bytesRead);
                 }
 
-                Logger.Debug($"{bypassUrl} downloaded to {fileName}");
+                Logger.Debug($"{requestUrl} downloaded to {fileName}");
                 return fileName;
             }
         }
         catch (Exception ex)
         {
-            Logger.Warn($"Error during file download of {bypassUrl}", ex);
+            Logger.Warn($"Error during file download of {requestUrl}", ex);
             try
             {
                 File.Delete(fileName);
@@ -136,6 +190,30 @@ public partial class ApiController
         }
     }
 
+    private async Task<HttpResponseMessage> SendRequestAsync<T>(HttpMethod method, Uri uri, T content = default, CancellationToken? ct = null) where T : class
+    {
+        using var requestMessage = new HttpRequestMessage(method, uri);
+        if (content != default)
+        {
+            requestMessage.Content = JsonContent.Create(content);
+        }
+        requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", Authorization);
+
+        if (content != default)
+        {
+            Logger.Debug("Sending " + method + " to " + uri + " (Content: " + await (((JsonContent)requestMessage.Content).ReadAsStringAsync()) + ")");
+        }
+        else
+        {
+            Logger.Debug("Sending " + method + " to " + uri);
+        }
+
+        if (ct.HasValue)
+            return await _httpClient.SendAsync(requestMessage, ct.Value).ConfigureAwait(false);
+
+        return await _httpClient.SendAsync(requestMessage).ConfigureAwait(false);
+    }
+
     private async Task DownloadFilesInternal(int currentDownloadId, List<FileReplacementDto> fileReplacementDto, CancellationToken ct)
     {
         Logger.Debug("Downloading files (Download ID " + currentDownloadId + ")");
@@ -156,7 +234,7 @@ public partial class ApiController
             }
         }
 
-        var downloadGroups = CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred).GroupBy(f => f.DownloadUri.Host, StringComparer.Ordinal);
+        var downloadGroups = CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred).GroupBy(f => f.DownloadUri.Host + f.DownloadUri.Port, StringComparer.Ordinal);
 
         await Parallel.ForEachAsync(downloadGroups, new ParallelOptions()
         {
@@ -165,16 +243,19 @@ public partial class ApiController
         },
         async (fileGroup, token) =>
         {
+            // let server predownload files
+            await SendRequestAsync(HttpMethod.Post, MareFiles.RequestEnqueueFullPath(fileGroup.First().DownloadUri),
+                fileGroup.Select(c => c.Hash), token).ConfigureAwait(false);
+
             foreach (var file in fileGroup)
             {
-                Logger.Debug($"Downloading {file.DownloadUri}");
                 var hash = file.Hash;
                 Progress<long> progress = new((bytesDownloaded) =>
                 {
                     file.Transferred += bytesDownloaded;
                 });
 
-                var tempFile = await DownloadFileHttpClient(file.DownloadUri, progress, token).ConfigureAwait(false);
+                var tempFile = await DownloadFileHttpClient(file, progress, token).ConfigureAwait(false);
                 if (token.IsCancellationRequested)
                 {
                     File.Delete(tempFile);

MareSynchronos/WebAPI/ApiController.cs

@@ -33,7 +33,7 @@ public partial class ApiController : IDisposable, IMareHubClient
     private readonly FileCacheManager _fileDbManager;
     private CancellationTokenSource _connectionCancellationTokenSource;
     private Dictionary<JwtCache, string> _jwtToken = new();
-    private KeyValuePair<string, string> AuthorizationJwtHeader => new("Authorization", "Bearer " + _jwtToken.GetValueOrDefault(new JwtCache(ApiUri, _dalamudUtil.PlayerNameHashed, SecretKey), string.Empty));
+    private string Authorization => _jwtToken.GetValueOrDefault(new JwtCache(ApiUri, _dalamudUtil.PlayerNameHashed, SecretKey), string.Empty);
 
     private HubConnection? _mareHub;
 
@@ -49,6 +49,8 @@ public partial class ApiController : IDisposable, IMareHubClient
 
     public bool IsAdmin => _connectionDto?.IsAdmin ?? false;
 
+    private HttpClient _httpClient;
+
     public ApiController(Configuration pluginConfiguration, DalamudUtil dalamudUtil, FileCacheManager fileDbManager)
     {
         Logger.Verbose("Creating " + nameof(ApiController));
@@ -144,6 +146,9 @@ public partial class ApiController : IDisposable, IMareHubClient
     {
         Logger.Debug("CreateConnections called");
 
+        _httpClient?.Dispose();
+        _httpClient = new();
+
         if (_pluginConfiguration.FullPause)
         {
             Logger.Info("Not recreating Connection, paused");
@@ -181,9 +186,9 @@ public partial class ApiController : IDisposable, IMareHubClient
                 {
                     Logger.Debug("Requesting new JWT");
                     using HttpClient httpClient = new();
-                    var postUri = new Uri(new Uri(ApiUri
+                    var postUri = MareAuth.AuthFullPath(new Uri(ApiUri
                         .Replace("wss://", "https://", StringComparison.OrdinalIgnoreCase)
-                        .Replace("ws://", "http://", StringComparison.OrdinalIgnoreCase)), MareAuth.AuthFullPath);
+                        .Replace("ws://", "http://", StringComparison.OrdinalIgnoreCase)));
                     using var sha256 = SHA256.Create();
                     var auth = BitConverter.ToString(sha256.ComputeHash(Encoding.UTF8.GetBytes(SecretKey))).Replace("-", "", StringComparison.OrdinalIgnoreCase);
                     var result = await httpClient.PostAsync(postUri, new FormUrlEncodedContent(new[]
@@ -343,7 +348,7 @@ public partial class ApiController : IDisposable, IMareHubClient
         return new HubConnectionBuilder()
             .WithUrl(ApiUri + hubName, options =>
             {
-                options.Headers.Add(AuthorizationJwtHeader);
+                options.Headers.Add("Authorization", "Bearer " + Authorization);
                 options.Transports = HttpTransportType.WebSockets | HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling;
             })
             .WithAutomaticReconnect(new ForeverRetryPolicy())