add alternative upload, ignore NIN pets, fix progress crashing

MareSynchronos/MareConfiguration/Configurations/MareConfig.cs

@@ -40,6 +40,7 @@ public class MareConfig : IMareConfiguration
     public int TransferBarsHeight { get; set; } = 12;
     public bool TransferBarsShowText { get; set; } = true;
     public int TransferBarsWidth { get; set; } = 250;
+    public bool UseAlternativeFileUpload { get; set; } = false;
     public int Version { get; set; } = 1;
     public NotificationLocation WarningNotification { get; set; } = NotificationLocation.Both;
 }

MareSynchronos/MareSynchronos.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <Authors></Authors>
     <Company></Company>
-    <Version>0.8.16</Version>
+    <Version>0.8.17</Version>
     <Description></Description>
     <Copyright></Copyright>
     <PackageProjectUrl>https://github.com/Penumbra-Sync/client</PackageProjectUrl>

MareSynchronos/PlayerData/Handlers/GameObjectHandler.cs

@@ -268,11 +268,17 @@ public sealed class GameObjectHandler : DisposableMediatorSubscriberBase
 
     private unsafe bool IsBeingDrawn(IntPtr drawObj, IntPtr curPtr)
     {
-        Logger.LogTrace("IsBeingDrawn for ptr {curPtr} : {drawObj}", curPtr.ToString("X"), drawObj.ToString("X"));
+        Logger.LogTrace("IsBeingDrawn for {kind} ptr {curPtr} : {drawObj}", ObjectKind, curPtr.ToString("X"), drawObj.ToString("X"));
+        if (ObjectKind == ObjectKind.Player)
+        {
+            return drawObj == IntPtr.Zero
+                           || (((CharacterBase*)drawObj)->HasModelInSlotLoaded != 0)
+                           || (((CharacterBase*)drawObj)->HasModelFilesInSlotLoaded != 0)
+                           || (((GameObject*)curPtr)->RenderFlags & 0b100000000000) == 0b100000000000;
+        }
+
         return drawObj == IntPtr.Zero
-                       || (((CharacterBase*)drawObj)->HasModelInSlotLoaded != 0)
+            || ((GameObject*)curPtr)->RenderFlags != 0x0;
-                       || (((CharacterBase*)drawObj)->HasModelFilesInSlotLoaded != 0)
-                       || (((GameObject*)curPtr)->RenderFlags & 0b100000000000) == 0b100000000000;
     }
 
     private void ZoneSwitchEnd()

MareSynchronos/Services/DalamudUtilService.cs

@@ -26,6 +26,7 @@ public class DalamudUtilService : IHostedService
     private readonly MareMediator _mediator;
     private readonly ObjectTable _objectTable;
     private readonly PerformanceCollectorService _performanceCollector;
+    private readonly List<uint> ClassJobIdsIgnoredForPets = new() { 30 };
     private uint? _classJobId = 0;
     private DateTime _delayedFrameworkUpdateCheck = DateTime.Now;
     private bool _sentBetweenAreas = false;
@@ -107,6 +108,7 @@ public class DalamudUtilService : IHostedService
 
     public unsafe IntPtr GetPet(IntPtr? playerPointer = null)
     {
+        if (ClassJobIdsIgnoredForPets.Contains(_classJobId ?? 0)) return IntPtr.Zero;
         var mgr = CharacterManager.Instance();
         playerPointer ??= PlayerPointer;
         if (playerPointer == IntPtr.Zero) return IntPtr.Zero;
@@ -320,7 +322,7 @@ public class DalamudUtilService : IHostedService
             if (_classJobId != newclassJobId)
             {
                 _classJobId = newclassJobId;
-                _mediator.Publish(new ClassJobChangedMessage());
+                _mediator.Publish(new ClassJobChangedMessage(_classJobId));
             }
         }
 

MareSynchronos/Services/Mediator/MareMediator.cs

@@ -102,16 +102,12 @@ public sealed class MareMediator : IDisposable
     {
         lock (_addRemoveLock)
         {
-            foreach (KeyValuePair<Type, HashSet<SubscriberAction>> kvp in _subscriberDict)
+            foreach (Type kvp in _subscriberDict.Select(k => k.Key))
             {
-                int unSubbed = _subscriberDict[kvp.Key]?.RemoveWhere(p => p.Subscriber == subscriber) ?? 0;
+                int unSubbed = _subscriberDict[kvp]?.RemoveWhere(p => p.Subscriber == subscriber) ?? 0;
                 if (unSubbed > 0)
                 {
-                    _logger.LogDebug("{sub} unsubscribed from {msg}", subscriber.GetType().Name, kvp.Key.Name);
+                    _logger.LogDebug("{sub} unsubscribed from {msg}", subscriber.GetType().Name, kvp.Name);
-                    if (_subscriberDict[kvp.Key].Any())
-                    {
-                        _logger.LogTrace("Remaining Subscribers: {item}", string.Join(", ", _subscriberDict[kvp.Key].Select(k => k.Subscriber.GetType().Name)));
-                    }
                 }
             }
         }

MareSynchronos/Services/Mediator/Messages.cs

@@ -16,7 +16,7 @@ public record OpenSettingsUiMessage : IMessage;
 public record DalamudLoginMessage : IMessage;
 public record DalamudLogoutMessage : IMessage;
 public record FrameworkUpdateMessage : IMessage;
-public record ClassJobChangedMessage : IMessage;
+public record ClassJobChangedMessage(uint? ClassJob) : IMessage;
 public record DelayedFrameworkUpdateMessage : IMessage;
 public record ZoneSwitchStartMessage : IMessage;
 public record ZoneSwitchEndMessage : IMessage;

MareSynchronos/UI/SettingsUi.cs

@@ -132,12 +132,20 @@ public class SettingsUi : WindowMediatorSubscriberBase
         UiSharedService.FontText("Transfer Settings", _uiShared.UidFont);
 
         int maxParallelDownloads = _configService.Current.ParallelDownloads;
+        bool useAlternativeUpload = _configService.Current.UseAlternativeFileUpload;
         if (ImGui.SliderInt("Maximum Parallel Downloads", ref maxParallelDownloads, 1, 10))
         {
             _configService.Current.ParallelDownloads = maxParallelDownloads;
             _configService.Save();
         }
 
+        if (ImGui.Checkbox("Use Alternative Upload Method", ref useAlternativeUpload))
+        {
+            _configService.Current.UseAlternativeFileUpload = useAlternativeUpload;
+            _configService.Save();
+        }
+        UiSharedService.DrawHelpText("This will attempt to upload files in one go instead of a stream. Typically not necessary to enable. Use if you have upload issues.");
+
         ImGui.Separator();
         UiSharedService.FontText("Transfer UI", _uiShared.UidFont);
 

MareSynchronos/WebAPI/Files/FileDownloadManager.cs

@@ -30,19 +30,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
         _fileDbManager = fileCacheManager;
     }
 
-    public void Initialize()
-    {
-        Mediator.Subscribe<DownloadReadyMessage>(this, (msg) =>
-        {
-            if (_downloadReady.ContainsKey(msg.RequestId))
-            {
-                _downloadReady[msg.RequestId] = true;
-            }
-        });
-    }
-
     public List<DownloadFileTransfer> CurrentDownloads { get; private set; } = new();
+
     public List<FileTransfer> ForbiddenTransfers => _orchestrator.ForbiddenTransfers;
+
     public bool IsDownloading => !CurrentDownloads.Any();
 
     public void CancelDownload()
@@ -69,6 +60,17 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
         }
     }
 
+    public void Initialize()
+    {
+        Mediator.Subscribe<DownloadReadyMessage>(this, (msg) =>
+        {
+            if (_downloadReady.ContainsKey(msg.RequestId))
+            {
+                _downloadReady[msg.RequestId] = true;
+            }
+        });
+    }
+
     protected override void Dispose(bool disposing)
     {
         CancelDownload();
@@ -195,9 +197,16 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                 var tempPath = _fileDbManager.GetCacheFilePath(file.Hash, isTemporaryFile: true);
                 Progress<long> progress = new((bytesDownloaded) =>
                 {
-                    if (!_downloadStatus.ContainsKey(fileGroup.Key)) return;
+                    try
-                    _downloadStatus[fileGroup.Key].TransferredBytes += bytesDownloaded;
+                    {
-                    file.Transferred += bytesDownloaded;
+                        if (!_downloadStatus.ContainsKey(fileGroup.Key)) return;
+                        _downloadStatus[fileGroup.Key].TransferredBytes += bytesDownloaded;
+                        file.Transferred += bytesDownloaded;
+                    }
+                    catch (Exception ex)
+                    {
+                        Logger.LogWarning(ex, "Could not set download progress");
+                    }
                 });
 
                 try

MareSynchronos/WebAPI/Files/FileTransferOrchestrator.cs

@@ -55,7 +55,10 @@ public class FileTransferOrchestrator : DisposableMediatorSubscriberBase
     public async Task<HttpResponseMessage> SendRequestAsync<T>(HttpMethod method, Uri uri, T content, CancellationToken ct) where T : class
     {
         using var requestMessage = new HttpRequestMessage(method, uri);
-        requestMessage.Content = JsonContent.Create(content);
+        if (content is not ByteArrayContent)
+            requestMessage.Content = JsonContent.Create(content);
+        else
+            requestMessage.Content = content as ByteArrayContent;
         return await SendRequestInternalAsync(requestMessage, ct).ConfigureAwait(false);
     }
 
@@ -84,7 +87,7 @@ public class FileTransferOrchestrator : DisposableMediatorSubscriberBase
     {
         requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _serverManager.GetToken());
 
-        if (requestMessage.Content != null && requestMessage.Content is not StreamContent)
+        if (requestMessage.Content != null && requestMessage.Content is not StreamContent && requestMessage.Content is not ByteArrayContent)
         {
             var content = await ((JsonContent)requestMessage.Content).ReadAsStringAsync().ConfigureAwait(false);
             Logger.LogDebug("Sending {method} to {uri} (Content: {content})", requestMessage.Method, requestMessage.RequestUri, content);

MareSynchronos/WebAPI/Files/FileUploadManager.cs

@@ -3,6 +3,7 @@ using MareSynchronos.API.Data;
 using MareSynchronos.API.Dto.Files;
 using MareSynchronos.API.Routes;
 using MareSynchronos.FileCache;
+using MareSynchronos.MareConfiguration;
 using MareSynchronos.Services.Mediator;
 using MareSynchronos.Services.ServerConfiguration;
 using MareSynchronos.UI;
@@ -16,16 +17,19 @@ namespace MareSynchronos.WebAPI.Files;
 public sealed class FileUploadManager : DisposableMediatorSubscriberBase
 {
     private readonly FileCacheManager _fileDbManager;
+    private readonly MareConfigService _mareConfigService;
     private readonly FileTransferOrchestrator _orchestrator;
     private readonly ServerConfigurationManager _serverManager;
     private readonly Dictionary<string, DateTime> _verifiedUploadedHashes = new(StringComparer.Ordinal);
     private CancellationTokenSource? _uploadCancellationTokenSource = new();
 
     public FileUploadManager(ILogger<FileUploadManager> logger, MareMediator mediator,
+        MareConfigService mareConfigService,
         FileTransferOrchestrator orchestrator,
         FileCacheManager fileDbManager,
         ServerConfigurationManager serverManager) : base(logger, mediator)
     {
+        _mareConfigService = mareConfigService;
         _orchestrator = orchestrator;
         _fileDbManager = fileDbManager;
         _serverManager = serverManager;
@@ -142,10 +146,39 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
     {
         if (!_orchestrator.IsInitialized) throw new InvalidOperationException("FileTransferManager is not initialized");
 
-        Logger.LogInformation("Uploading {file}, {size}", fileHash, UiSharedService.ByteToString(compressedFile.Length));
+        Logger.LogInformation("[{hash}] Uploading {size}", fileHash, UiSharedService.ByteToString(compressedFile.Length));
 
         if (uploadToken.IsCancellationRequested) return;
 
+        try
+        {
+            if (!_mareConfigService.Current.UseAlternativeFileUpload)
+                await UploadFileStream(compressedFile, fileHash, uploadToken).ConfigureAwait(false);
+            else
+                await UploadFileFull(compressedFile, fileHash, uploadToken).ConfigureAwait(false);
+        }
+        catch (Exception ex)
+        {
+            if (!_mareConfigService.Current.UseAlternativeFileUpload)
+            {
+                Logger.LogWarning(ex, "[{hash}] Error during file upload, trying alternative file upload", fileHash);
+                await UploadFileFull(compressedFile, fileHash, uploadToken).ConfigureAwait(false);
+            }
+        }
+    }
+
+    private async Task UploadFileFull(byte[] compressedFile, string fileHash, CancellationToken uploadToken)
+    {
+        using var content = new ByteArrayContent(compressedFile);
+        content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
+
+        var response = await _orchestrator.SendRequestAsync(HttpMethod.Post, MareFiles.ServerFilesUploadFullPath(_orchestrator.FilesCdnUri!, fileHash), content, uploadToken).ConfigureAwait(false);
+        Logger.LogDebug("[{hash}] Upload Status: {status}", fileHash, response.StatusCode);
+        CurrentUploads.Single(f => string.Equals(f.Hash, fileHash, StringComparison.Ordinal)).Transferred = compressedFile.Length;
+    }
+
+    private async Task UploadFileStream(byte[] compressedFile, string fileHash, CancellationToken uploadToken)
+    {
         using var ms = new MemoryStream(compressedFile);
 
         Progress<UploadProgress> prog = new((prog) =>
@@ -156,7 +189,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
         streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
 
         var response = await _orchestrator.SendRequestStreamAsync(HttpMethod.Post, MareFiles.ServerFilesUploadFullPath(_orchestrator.FilesCdnUri!, fileHash), streamContent, uploadToken).ConfigureAwait(false);
-        Logger.LogDebug("Upload Status: {status}", response.StatusCode);
+        Logger.LogDebug("[{hash}] Upload Status: {status}", fileHash, response.StatusCode);
     }
 
     private async Task UploadUnverifiedFiles(HashSet<string> unverifiedUploadHashes, List<UserData> visiblePlayers, CancellationToken uploadToken)
@@ -199,9 +232,10 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
         Task uploadTask = Task.CompletedTask;
         foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList())
         {
-            Logger.LogDebug("Compressing {file}", file);
+            Logger.LogDebug("[{hash}] Compressing", file);
             var data = await GetCompressedFileData(file.Hash, uploadToken).ConfigureAwait(false);
             CurrentUploads.Single(e => string.Equals(e.Hash, data.Item1, StringComparison.Ordinal)).Total = data.Item2.Length;
+            Logger.LogDebug("[{hash}] Starting upload for {filePath}", data.Item1, _fileDbManager.GetFileCacheByHash(data.Item1)!.ResolvedFilepath);
             await uploadTask.ConfigureAwait(false);
             uploadTask = UploadFile(data.Item2, file.Hash, uploadToken);
             uploadToken.ThrowIfCancellationRequested();