diff --git a/MareAPI b/MareAPI index d361cfa..bd4c360 160000 --- a/MareAPI +++ b/MareAPI @@ -1 +1 @@ -Subproject commit d361cfa3b983e8772c2bd08b3d638542ed57cd0f +Subproject commit bd4c360a4f1d0b1caa0e2c00d19eed7bd209d931 diff --git a/MareSynchronos/Interop/RenderModel.cs b/MareSynchronos/Interop/RenderModel.cs index db84aa7..ca6946f 100644 --- a/MareSynchronos/Interop/RenderModel.cs +++ b/MareSynchronos/Interop/RenderModel.cs @@ -25,18 +25,18 @@ public unsafe struct RenderModel [FieldOffset( 0x60 )] public int BoneListCount; - [FieldOffset( 0x68 )] + [FieldOffset( 0x70 )] private void* UnkDXBuffer1; - [FieldOffset( 0x70 )] + [FieldOffset( 0x78 )] private void* UnkDXBuffer2; - [FieldOffset( 0x78 )] + [FieldOffset( 0x80 )] private void* UnkDXBuffer3; - [FieldOffset( 0x90 )] + [FieldOffset( 0x98 )] public void** Materials; - [FieldOffset( 0x98 )] + [FieldOffset( 0xA0 )] public int MaterialCount; } \ No newline at end of file diff --git a/MareSynchronos/MareSynchronos.csproj b/MareSynchronos/MareSynchronos.csproj index 76f0df6..22f54fc 100644 --- a/MareSynchronos/MareSynchronos.csproj +++ b/MareSynchronos/MareSynchronos.csproj @@ -10,7 +10,7 @@ - net6.0-windows + net7.0-windows x64 enable latest @@ -26,15 +26,15 @@ - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + diff --git a/MareSynchronos/UI/SettingsUi.cs b/MareSynchronos/UI/SettingsUi.cs index e0d2547..db3ec76 100644 --- a/MareSynchronos/UI/SettingsUi.cs +++ b/MareSynchronos/UI/SettingsUi.cs @@ -669,16 +669,22 @@ public class SettingsUi : Window, IDisposable ImGui.SameLine(); if (ImGui.Button("Clear local cache")) { - Task.Run(() => + if (UiShared.CtrlPressed()) { - foreach (var file in Directory.GetFiles(_configuration.CacheFolder)) + Task.Run(() => { - File.Delete(file); - } + foreach (var file in Directory.GetFiles(_configuration.CacheFolder)) + { + File.Delete(file); + } - _uiShared.RecalculateFileCacheSize(); - }); + _uiShared.RecalculateFileCacheSize(); + }); + } } + UiShared.AttachToolTip("You normally do not need to do this. This will solely remove all downloaded data from all players and will require you to re-download everything again." + Environment.NewLine + + "Mares Cache is self-clearing and will not surpass the limit you have set it to." + Environment.NewLine + + "If you still think you need to do this hold CTRL while pressing the button."); } public override void OnClose() diff --git a/MareSynchronos/Utils/DalamudUtil.cs b/MareSynchronos/Utils/DalamudUtil.cs index 6d66c4a..6bd0068 100644 --- a/MareSynchronos/Utils/DalamudUtil.cs +++ b/MareSynchronos/Utils/DalamudUtil.cs @@ -177,21 +177,21 @@ public class DalamudUtil : IDisposable public unsafe IntPtr GetMinion() { - return (IntPtr)((FFXIVClientStructs.FFXIV.Client.Game.Character.Character*)PlayerPointer)->CompanionObject; + return (IntPtr)((Character*)PlayerPointer)->CompanionObject; } public unsafe IntPtr GetPet(IntPtr? playerPointer = null) { var mgr = CharacterManager.Instance(); if (playerPointer == null) playerPointer = PlayerPointer; - return (IntPtr)mgr->LookupPetByOwnerObject((FFXIVClientStructs.FFXIV.Client.Game.Character.BattleChara*)playerPointer); + return (IntPtr)mgr->LookupPetByOwnerObject((BattleChara*)playerPointer); } public unsafe IntPtr GetCompanion(IntPtr? playerPointer = null) { var mgr = CharacterManager.Instance(); if (playerPointer == null) playerPointer = PlayerPointer; - return (IntPtr)mgr->LookupBuddyByOwnerObject((FFXIVClientStructs.FFXIV.Client.Game.Character.BattleChara*)playerPointer); + return (IntPtr)mgr->LookupBuddyByOwnerObject((BattleChara*)playerPointer); } public string PlayerName => _clientState.LocalPlayer?.Name.ToString() ?? "--"; diff --git a/MareSynchronos/WebAPI/ApIController.Functions.Files.cs b/MareSynchronos/WebAPI/ApIController.Functions.Files.cs index 3d162d2..2a9a973 100644 --- a/MareSynchronos/WebAPI/ApIController.Functions.Files.cs +++ b/MareSynchronos/WebAPI/ApIController.Functions.Files.cs @@ -4,6 +4,8 @@ using System.IO; using System.Linq; using System.Net; using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Http.Json; using System.Runtime.CompilerServices; using System.Text; using System.Threading; @@ -13,6 +15,7 @@ using MareSynchronos.API; using MareSynchronos.Utils; using MareSynchronos.WebAPI.Utils; using Microsoft.AspNetCore.SignalR.Client; +using Newtonsoft.Json; namespace MareSynchronos.WebAPI; @@ -43,30 +46,81 @@ public partial class ApiController await _mareHub!.SendAsync(nameof(FilesDeleteAll)).ConfigureAwait(false); } - private async Task DownloadFileHttpClient(Uri url, IProgress progress, CancellationToken ct) + private async Task GetQueueRequestDto(DownloadFileTransfer downloadFileTransfer) { - using var client = new HttpClient(); - client.DefaultRequestHeaders.Add(AuthorizationJwtHeader.Key, AuthorizationJwtHeader.Value); + var response = await SendRequestAsync(HttpMethod.Get, MareFiles.RequestRequestFileFullPath(downloadFileTransfer.DownloadUri, downloadFileTransfer.Hash)).ConfigureAwait(false); + return JsonConvert.DeserializeObject(await response.Content.ReadAsStringAsync().ConfigureAwait(false))!; + } + + private async Task WaitForQueue(DownloadFileTransfer fileTransfer, Guid requestId, CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + await Task.Delay(250, ct).ConfigureAwait(false); + var queueResponse = await SendRequestAsync(HttpMethod.Get, MareFiles.RequestCheckQueueFullPath(fileTransfer.DownloadUri, requestId)).ConfigureAwait(false); + try + { + queueResponse.EnsureSuccessStatusCode(); + Logger.Debug($"Starting download for file {fileTransfer.Hash} ({requestId})"); + break; + } + catch (HttpRequestException ex) + { + switch (ex.StatusCode) + { + case HttpStatusCode.Conflict: + Logger.Debug($"In queue for file {fileTransfer.Hash} ({requestId})"); + // still in queue + break; + case HttpStatusCode.BadRequest: + // rerequest queue + Logger.Debug($"Rerequesting {fileTransfer.Hash}"); + var dto = await GetQueueRequestDto(fileTransfer).ConfigureAwait(false); + requestId = dto.RequestId; + break; + default: + Logger.Warn($"Unclear response from server: {fileTransfer.Hash} ({requestId}): {ex.StatusCode}"); + break; + } + + await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); + } + } + + return requestId; + } + + private async Task DownloadFileHttpClient(DownloadFileTransfer fileTransfer, IProgress progress, CancellationToken ct) + { + var queueRequest = await GetQueueRequestDto(fileTransfer).ConfigureAwait(false); + + Logger.Debug($"GUID {queueRequest.RequestId} for file {fileTransfer.Hash}, queue status {queueRequest.QueueStatus}"); + + var requestId = queueRequest.QueueStatus == QueueStatus.Ready + ? queueRequest.RequestId + : await WaitForQueue(fileTransfer, queueRequest.RequestId, ct).ConfigureAwait(false); + int attempts = 1; bool failed = true; const int maxAttempts = 16; HttpResponseMessage response = null!; HttpStatusCode? lastError = HttpStatusCode.OK; - var bypassUrl = new Uri(url, "?nocache=" + DateTime.UtcNow.Ticks); + var requestUrl = MareFiles.CacheGetFullPath(fileTransfer.DownloadUri, requestId); + + Logger.Debug($"Downloading {requestUrl} for file {fileTransfer.Hash}"); while (failed && attempts < maxAttempts && !ct.IsCancellationRequested) { try { - response = await client.GetAsync(bypassUrl, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false); + response = await SendRequestAsync(HttpMethod.Get, requestUrl, ct: ct).ConfigureAwait(false); response.EnsureSuccessStatusCode(); failed = false; } catch (HttpRequestException ex) { - Logger.Warn($"Attempt {attempts}: Error during download of {bypassUrl}, HttpStatusCode: {ex.StatusCode}"); - bypassUrl = new Uri(url, "?nocache=" + DateTime.UtcNow.Ticks); + Logger.Warn($"Attempt {attempts}: Error during download of {requestUrl}, HttpStatusCode: {ex.StatusCode}"); lastError = ex.StatusCode; if (ex.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Unauthorized) { @@ -79,7 +133,7 @@ public partial class ApiController if (failed) { - throw new Exception($"Http error {lastError} after {maxAttempts} attempts (cancelled: {ct.IsCancellationRequested}): {url}"); + throw new Exception($"Http error {lastError} after {maxAttempts} attempts (cancelled: {ct.IsCancellationRequested}): {requestUrl}"); } var fileName = Path.GetTempFileName(); @@ -101,13 +155,13 @@ public partial class ApiController progress.Report(bytesRead); } - Logger.Debug($"{bypassUrl} downloaded to {fileName}"); + Logger.Debug($"{requestUrl} downloaded to {fileName}"); return fileName; } } catch (Exception ex) { - Logger.Warn($"Error during file download of {bypassUrl}", ex); + Logger.Warn($"Error during file download of {requestUrl}", ex); try { File.Delete(fileName); @@ -136,6 +190,30 @@ public partial class ApiController } } + private async Task SendRequestAsync(HttpMethod method, Uri uri, T content = default, CancellationToken? ct = null) where T : class + { + using var requestMessage = new HttpRequestMessage(method, uri); + if (content != default) + { + requestMessage.Content = JsonContent.Create(content); + } + requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", Authorization); + + if (content != default) + { + Logger.Debug("Sending " + method + " to " + uri + " (Content: " + await (((JsonContent)requestMessage.Content).ReadAsStringAsync()) + ")"); + } + else + { + Logger.Debug("Sending " + method + " to " + uri); + } + + if (ct.HasValue) + return await _httpClient.SendAsync(requestMessage, ct.Value).ConfigureAwait(false); + + return await _httpClient.SendAsync(requestMessage).ConfigureAwait(false); + } + private async Task DownloadFilesInternal(int currentDownloadId, List fileReplacementDto, CancellationToken ct) { Logger.Debug("Downloading files (Download ID " + currentDownloadId + ")"); @@ -156,7 +234,7 @@ public partial class ApiController } } - var downloadGroups = CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred).GroupBy(f => f.DownloadUri.Host, StringComparer.Ordinal); + var downloadGroups = CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred).GroupBy(f => f.DownloadUri.Host + f.DownloadUri.Port, StringComparer.Ordinal); await Parallel.ForEachAsync(downloadGroups, new ParallelOptions() { @@ -165,16 +243,19 @@ public partial class ApiController }, async (fileGroup, token) => { + // let server predownload files + await SendRequestAsync(HttpMethod.Post, MareFiles.RequestEnqueueFullPath(fileGroup.First().DownloadUri), + fileGroup.Select(c => c.Hash), token).ConfigureAwait(false); + foreach (var file in fileGroup) { - Logger.Debug($"Downloading {file.DownloadUri}"); var hash = file.Hash; Progress progress = new((bytesDownloaded) => { file.Transferred += bytesDownloaded; }); - var tempFile = await DownloadFileHttpClient(file.DownloadUri, progress, token).ConfigureAwait(false); + var tempFile = await DownloadFileHttpClient(file, progress, token).ConfigureAwait(false); if (token.IsCancellationRequested) { File.Delete(tempFile); diff --git a/MareSynchronos/WebAPI/ApiController.cs b/MareSynchronos/WebAPI/ApiController.cs index b63cd2d..cc17700 100644 --- a/MareSynchronos/WebAPI/ApiController.cs +++ b/MareSynchronos/WebAPI/ApiController.cs @@ -33,7 +33,7 @@ public partial class ApiController : IDisposable, IMareHubClient private readonly FileCacheManager _fileDbManager; private CancellationTokenSource _connectionCancellationTokenSource; private Dictionary _jwtToken = new(); - private KeyValuePair AuthorizationJwtHeader => new("Authorization", "Bearer " + _jwtToken.GetValueOrDefault(new JwtCache(ApiUri, _dalamudUtil.PlayerNameHashed, SecretKey), string.Empty)); + private string Authorization => _jwtToken.GetValueOrDefault(new JwtCache(ApiUri, _dalamudUtil.PlayerNameHashed, SecretKey), string.Empty); private HubConnection? _mareHub; @@ -49,6 +49,8 @@ public partial class ApiController : IDisposable, IMareHubClient public bool IsAdmin => _connectionDto?.IsAdmin ?? false; + private HttpClient _httpClient; + public ApiController(Configuration pluginConfiguration, DalamudUtil dalamudUtil, FileCacheManager fileDbManager) { Logger.Verbose("Creating " + nameof(ApiController)); @@ -144,6 +146,9 @@ public partial class ApiController : IDisposable, IMareHubClient { Logger.Debug("CreateConnections called"); + _httpClient?.Dispose(); + _httpClient = new(); + if (_pluginConfiguration.FullPause) { Logger.Info("Not recreating Connection, paused"); @@ -181,9 +186,9 @@ public partial class ApiController : IDisposable, IMareHubClient { Logger.Debug("Requesting new JWT"); using HttpClient httpClient = new(); - var postUri = new Uri(new Uri(ApiUri + var postUri = MareAuth.AuthFullPath(new Uri(ApiUri .Replace("wss://", "https://", StringComparison.OrdinalIgnoreCase) - .Replace("ws://", "http://", StringComparison.OrdinalIgnoreCase)), MareAuth.AuthFullPath); + .Replace("ws://", "http://", StringComparison.OrdinalIgnoreCase))); using var sha256 = SHA256.Create(); var auth = BitConverter.ToString(sha256.ComputeHash(Encoding.UTF8.GetBytes(SecretKey))).Replace("-", "", StringComparison.OrdinalIgnoreCase); var result = await httpClient.PostAsync(postUri, new FormUrlEncodedContent(new[] @@ -343,7 +348,7 @@ public partial class ApiController : IDisposable, IMareHubClient return new HubConnectionBuilder() .WithUrl(ApiUri + hubName, options => { - options.Headers.Add(AuthorizationJwtHeader); + options.Headers.Add("Authorization", "Bearer " + Authorization); options.Transports = HttpTransportType.WebSockets | HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; }) .WithAutomaticReconnect(new ForeverRetryPolicy())