Update for 6.3 and change to File Service Queue (#34)

* add request queue processing on clientside, switch to net7 * reuse httpclient * dispose and renew httpclient on building connection * add handling of QueueRequestDto to downloads * group download by host and port * refactor SendAsync to SendRequestAsync for readability * add holding ctrl before clearing cache, add tooltip * update pathing * fixes to api usage in client * fix rendermodel offsets * fixes for downloads Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>
2023-01-11 14:41:13 +01:00
parent 964b29b87e
commit 099ac2b3a1
7 changed files with 130 additions and 38 deletions
--- a/2
+++ b/2
--- a/MareSynchronos/Interop/RenderModel.cs
+++ b/MareSynchronos/Interop/RenderModel.cs
@@ -25,18 +25,18 @@ public unsafe struct RenderModel
    [FieldOffset( 0x60 )]
    public int BoneListCount;

-    [FieldOffset( 0x68 )]
+    [FieldOffset( 0x70 )]
    private void* UnkDXBuffer1;

-    [FieldOffset( 0x70 )]
+    [FieldOffset( 0x78 )]
    private void* UnkDXBuffer2;

-    [FieldOffset( 0x78 )]
+    [FieldOffset( 0x80 )]
    private void* UnkDXBuffer3;

-    [FieldOffset( 0x90 )]
+    [FieldOffset( 0x98 )]
    public void** Materials;

-    [FieldOffset( 0x98 )]
+    [FieldOffset( 0xA0 )]
    public int MaterialCount;
 }
--- a/MareSynchronos/MareSynchronos.csproj
+++ b/MareSynchronos/MareSynchronos.csproj
@@ -10,7 +10,7 @@
  </PropertyGroup>

  <PropertyGroup>
-    <TargetFramework>net6.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
    <Platforms>x64</Platforms>
    <Nullable>enable</Nullable>
    <LangVersion>latest</LangVersion>
@@ -26,15 +26,15 @@
  </PropertyGroup>

  <ItemGroup>
-    <PackageReference Include="DalamudPackager" Version="2.1.8" />
+    <PackageReference Include="DalamudPackager" Version="2.1.10" />
    <PackageReference Include="lz4net" Version="1.0.15.93" />
-    <PackageReference Include="Meziantou.Analyzer" Version="1.0.733">
+    <PackageReference Include="Meziantou.Analyzer" Version="2.0.4">
      <PrivateAssets>all</PrivateAssets>
      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
    </PackageReference>
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="6.0.8" />
-    <PackageReference Include="Penumbra.Api" Version="1.0.3" />
-    <PackageReference Include="Penumbra.String" Version="1.0.0" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="7.0.1" />
+    <PackageReference Include="Penumbra.Api" Version="1.0.5" />
+    <PackageReference Include="Penumbra.String" Version="1.0.1" />
  </ItemGroup>

  <PropertyGroup>
--- a/MareSynchronos/UI/SettingsUi.cs
+++ b/MareSynchronos/UI/SettingsUi.cs
@@ -668,6 +668,8 @@ public class SettingsUi : Window, IDisposable
        ImGui.Text($"Local cache size: {UiShared.ByteToString(_uiShared.FileCacheSize)}");
        ImGui.SameLine();
        if (ImGui.Button("Clear local cache"))
+        {
+            if (UiShared.CtrlPressed())
            {
                Task.Run(() =>
                {
@@ -680,6 +682,10 @@ public class SettingsUi : Window, IDisposable
                });
            }
        }
+        UiShared.AttachToolTip("You normally do not need to do this. This will solely remove all downloaded data from all players and will require you to re-download everything again." + Environment.NewLine
+            + "Mares Cache is self-clearing and will not surpass the limit you have set it to." + Environment.NewLine
+            + "If you still think you need to do this hold CTRL while pressing the button.");
+    }

    public override void OnClose()
    {
--- a/MareSynchronos/Utils/DalamudUtil.cs
+++ b/MareSynchronos/Utils/DalamudUtil.cs
@@ -177,21 +177,21 @@ public class DalamudUtil : IDisposable

    public unsafe IntPtr GetMinion()
    {
-        return (IntPtr)((FFXIVClientStructs.FFXIV.Client.Game.Character.Character*)PlayerPointer)->CompanionObject;
+        return (IntPtr)((Character*)PlayerPointer)->CompanionObject;
    }

    public unsafe IntPtr GetPet(IntPtr? playerPointer = null)
    {
        var mgr = CharacterManager.Instance();
        if (playerPointer == null) playerPointer = PlayerPointer;
-        return (IntPtr)mgr->LookupPetByOwnerObject((FFXIVClientStructs.FFXIV.Client.Game.Character.BattleChara*)playerPointer);
+        return (IntPtr)mgr->LookupPetByOwnerObject((BattleChara*)playerPointer);
    }

    public unsafe IntPtr GetCompanion(IntPtr? playerPointer = null)
    {
        var mgr = CharacterManager.Instance();
        if (playerPointer == null) playerPointer = PlayerPointer;
-        return (IntPtr)mgr->LookupBuddyByOwnerObject((FFXIVClientStructs.FFXIV.Client.Game.Character.BattleChara*)playerPointer);
+        return (IntPtr)mgr->LookupBuddyByOwnerObject((BattleChara*)playerPointer);
    }

    public string PlayerName => _clientState.LocalPlayer?.Name.ToString() ?? "--";
--- a/MareSynchronos/WebAPI/ApIController.Functions.Files.cs
+++ b/MareSynchronos/WebAPI/ApIController.Functions.Files.cs
@@ -4,6 +4,8 @@ using System.IO;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
 using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading;
@@ -13,6 +15,7 @@ using MareSynchronos.API;
 using MareSynchronos.Utils;
 using MareSynchronos.WebAPI.Utils;
 using Microsoft.AspNetCore.SignalR.Client;
+using Newtonsoft.Json;

 namespace MareSynchronos.WebAPI;

@@ -43,30 +46,81 @@ public partial class ApiController
        await _mareHub!.SendAsync(nameof(FilesDeleteAll)).ConfigureAwait(false);
    }

-    private async Task<string> DownloadFileHttpClient(Uri url, IProgress<long> progress, CancellationToken ct)
+    private async Task<QueueRequestDto> GetQueueRequestDto(DownloadFileTransfer downloadFileTransfer)
    {
-        using var client = new HttpClient();
-        client.DefaultRequestHeaders.Add(AuthorizationJwtHeader.Key, AuthorizationJwtHeader.Value);
+        var response = await SendRequestAsync<object>(HttpMethod.Get, MareFiles.RequestRequestFileFullPath(downloadFileTransfer.DownloadUri, downloadFileTransfer.Hash)).ConfigureAwait(false);
+        return JsonConvert.DeserializeObject<QueueRequestDto>(await response.Content.ReadAsStringAsync().ConfigureAwait(false))!;
+    }
+
+    private async Task<Guid> WaitForQueue(DownloadFileTransfer fileTransfer, Guid requestId, CancellationToken ct)
+    {
+        while (!ct.IsCancellationRequested)
+        {
+            await Task.Delay(250, ct).ConfigureAwait(false);
+            var queueResponse = await SendRequestAsync<object>(HttpMethod.Get, MareFiles.RequestCheckQueueFullPath(fileTransfer.DownloadUri, requestId)).ConfigureAwait(false);
+            try
+            {
+                queueResponse.EnsureSuccessStatusCode();
+                Logger.Debug($"Starting download for file {fileTransfer.Hash} ({requestId})");
+                break;
+            }
+            catch (HttpRequestException ex)
+            {
+                switch (ex.StatusCode)
+                {
+                    case HttpStatusCode.Conflict:
+                        Logger.Debug($"In queue for file {fileTransfer.Hash} ({requestId})");
+                        // still in queue
+                        break;
+                    case HttpStatusCode.BadRequest:
+                        // rerequest queue
+                        Logger.Debug($"Rerequesting {fileTransfer.Hash}");
+                        var dto = await GetQueueRequestDto(fileTransfer).ConfigureAwait(false);
+                        requestId = dto.RequestId;
+                        break;
+                    default:
+                        Logger.Warn($"Unclear response from server: {fileTransfer.Hash} ({requestId}): {ex.StatusCode}");
+                        break;
+                }
+
+                await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
+            }
+        }
+
+        return requestId;
+    }
+
+    private async Task<string> DownloadFileHttpClient(DownloadFileTransfer fileTransfer, IProgress<long> progress, CancellationToken ct)
+    {
+        var queueRequest = await GetQueueRequestDto(fileTransfer).ConfigureAwait(false);
+
+        Logger.Debug($"GUID {queueRequest.RequestId} for file {fileTransfer.Hash}, queue status {queueRequest.QueueStatus}");
+
+        var requestId = queueRequest.QueueStatus == QueueStatus.Ready
+            ? queueRequest.RequestId
+            : await WaitForQueue(fileTransfer, queueRequest.RequestId, ct).ConfigureAwait(false);
+
        int attempts = 1;
        bool failed = true;
        const int maxAttempts = 16;

        HttpResponseMessage response = null!;
        HttpStatusCode? lastError = HttpStatusCode.OK;
-        var bypassUrl = new Uri(url, "?nocache=" + DateTime.UtcNow.Ticks);

+        var requestUrl = MareFiles.CacheGetFullPath(fileTransfer.DownloadUri, requestId);
+
+        Logger.Debug($"Downloading {requestUrl} for file {fileTransfer.Hash}");
        while (failed && attempts < maxAttempts && !ct.IsCancellationRequested)
        {
            try
            {
-                response = await client.GetAsync(bypassUrl, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
+                response = await SendRequestAsync<object>(HttpMethod.Get, requestUrl, ct: ct).ConfigureAwait(false);
                response.EnsureSuccessStatusCode();
                failed = false;
            }
            catch (HttpRequestException ex)
            {
-                Logger.Warn($"Attempt {attempts}: Error during download of {bypassUrl}, HttpStatusCode: {ex.StatusCode}");
-                bypassUrl = new Uri(url, "?nocache=" + DateTime.UtcNow.Ticks);
+                Logger.Warn($"Attempt {attempts}: Error during download of {requestUrl}, HttpStatusCode: {ex.StatusCode}");
                lastError = ex.StatusCode;
                if (ex.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Unauthorized)
                {
@@ -79,7 +133,7 @@ public partial class ApiController

        if (failed)
        {
-            throw new Exception($"Http error {lastError} after {maxAttempts} attempts (cancelled: {ct.IsCancellationRequested}): {url}");
+            throw new Exception($"Http error {lastError} after {maxAttempts} attempts (cancelled: {ct.IsCancellationRequested}): {requestUrl}");
        }

        var fileName = Path.GetTempFileName();
@@ -101,13 +155,13 @@ public partial class ApiController
                    progress.Report(bytesRead);
                }

-                Logger.Debug($"{bypassUrl} downloaded to {fileName}");
+                Logger.Debug($"{requestUrl} downloaded to {fileName}");
                return fileName;
            }
        }
        catch (Exception ex)
        {
-            Logger.Warn($"Error during file download of {bypassUrl}", ex);
+            Logger.Warn($"Error during file download of {requestUrl}", ex);
            try
            {
                File.Delete(fileName);
@@ -136,6 +190,30 @@ public partial class ApiController
        }
    }

+    private async Task<HttpResponseMessage> SendRequestAsync<T>(HttpMethod method, Uri uri, T content = default, CancellationToken? ct = null) where T : class
+    {
+        using var requestMessage = new HttpRequestMessage(method, uri);
+        if (content != default)
+        {
+            requestMessage.Content = JsonContent.Create(content);
+        }
+        requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", Authorization);
+
+        if (content != default)
+        {
+            Logger.Debug("Sending " + method + " to " + uri + " (Content: " + await (((JsonContent)requestMessage.Content).ReadAsStringAsync()) + ")");
+        }
+        else
+        {
+            Logger.Debug("Sending " + method + " to " + uri);
+        }
+
+        if (ct.HasValue)
+            return await _httpClient.SendAsync(requestMessage, ct.Value).ConfigureAwait(false);
+
+        return await _httpClient.SendAsync(requestMessage).ConfigureAwait(false);
+    }
+
    private async Task DownloadFilesInternal(int currentDownloadId, List<FileReplacementDto> fileReplacementDto, CancellationToken ct)
    {
        Logger.Debug("Downloading files (Download ID " + currentDownloadId + ")");
@@ -156,7 +234,7 @@ public partial class ApiController
            }
        }

-        var downloadGroups = CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred).GroupBy(f => f.DownloadUri.Host, StringComparer.Ordinal);
+        var downloadGroups = CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred).GroupBy(f => f.DownloadUri.Host + f.DownloadUri.Port, StringComparer.Ordinal);

        await Parallel.ForEachAsync(downloadGroups, new ParallelOptions()
        {
@@ -165,16 +243,19 @@ public partial class ApiController
        },
        async (fileGroup, token) =>
        {
+            // let server predownload files
+            await SendRequestAsync(HttpMethod.Post, MareFiles.RequestEnqueueFullPath(fileGroup.First().DownloadUri),
+                fileGroup.Select(c => c.Hash), token).ConfigureAwait(false);
+
            foreach (var file in fileGroup)
            {
-                Logger.Debug($"Downloading {file.DownloadUri}");
                var hash = file.Hash;
                Progress<long> progress = new((bytesDownloaded) =>
                {
                    file.Transferred += bytesDownloaded;
                });

-                var tempFile = await DownloadFileHttpClient(file.DownloadUri, progress, token).ConfigureAwait(false);
+                var tempFile = await DownloadFileHttpClient(file, progress, token).ConfigureAwait(false);
                if (token.IsCancellationRequested)
                {
                    File.Delete(tempFile);
--- a/MareSynchronos/WebAPI/ApiController.cs
+++ b/MareSynchronos/WebAPI/ApiController.cs
@@ -33,7 +33,7 @@ public partial class ApiController : IDisposable, IMareHubClient
    private readonly FileCacheManager _fileDbManager;
    private CancellationTokenSource _connectionCancellationTokenSource;
    private Dictionary<JwtCache, string> _jwtToken = new();
-    private KeyValuePair<string, string> AuthorizationJwtHeader => new("Authorization", "Bearer " + _jwtToken.GetValueOrDefault(new JwtCache(ApiUri, _dalamudUtil.PlayerNameHashed, SecretKey), string.Empty));
+    private string Authorization => _jwtToken.GetValueOrDefault(new JwtCache(ApiUri, _dalamudUtil.PlayerNameHashed, SecretKey), string.Empty);

    private HubConnection? _mareHub;

@@ -49,6 +49,8 @@ public partial class ApiController : IDisposable, IMareHubClient

    public bool IsAdmin => _connectionDto?.IsAdmin ?? false;

+    private HttpClient _httpClient;
+
    public ApiController(Configuration pluginConfiguration, DalamudUtil dalamudUtil, FileCacheManager fileDbManager)
    {
        Logger.Verbose("Creating " + nameof(ApiController));
@@ -144,6 +146,9 @@ public partial class ApiController : IDisposable, IMareHubClient
    {
        Logger.Debug("CreateConnections called");

+        _httpClient?.Dispose();
+        _httpClient = new();
+
        if (_pluginConfiguration.FullPause)
        {
            Logger.Info("Not recreating Connection, paused");
@@ -181,9 +186,9 @@ public partial class ApiController : IDisposable, IMareHubClient
                {
                    Logger.Debug("Requesting new JWT");
                    using HttpClient httpClient = new();
-                    var postUri = new Uri(new Uri(ApiUri
+                    var postUri = MareAuth.AuthFullPath(new Uri(ApiUri
                        .Replace("wss://", "https://", StringComparison.OrdinalIgnoreCase)
-                        .Replace("ws://", "http://", StringComparison.OrdinalIgnoreCase)), MareAuth.AuthFullPath);
+                        .Replace("ws://", "http://", StringComparison.OrdinalIgnoreCase)));
                    using var sha256 = SHA256.Create();
                    var auth = BitConverter.ToString(sha256.ComputeHash(Encoding.UTF8.GetBytes(SecretKey))).Replace("-", "", StringComparison.OrdinalIgnoreCase);
                    var result = await httpClient.PostAsync(postUri, new FormUrlEncodedContent(new[]
@@ -343,7 +348,7 @@ public partial class ApiController : IDisposable, IMareHubClient
        return new HubConnectionBuilder()
            .WithUrl(ApiUri + hubName, options =>
            {
-                options.Headers.Add(AuthorizationJwtHeader);
+                options.Headers.Add("Authorization", "Bearer " + Authorization);
                options.Transports = HttpTransportType.WebSockets | HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling;
            })
            .WithAutomaticReconnect(new ForeverRetryPolicy())