do not use sendfiles when not necessary

MareSynchronos/MareSynchronos.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <Authors></Authors>
     <Company></Company>
-    <Version>0.3.2</Version>
+    <Version>0.3.3</Version>
     <Description></Description>
     <Copyright></Copyright>
     <PackageProjectUrl>https://github.com/Penumbra-Sync/client</PackageProjectUrl>

MareSynchronos/WebAPI/ApIController.Functions.Files.cs

@@ -19,6 +19,8 @@ namespace MareSynchronos.WebAPI
 {
     public partial class ApiController
     {
+        private readonly HashSet<string> _verifiedUploadedHashes;
+
         private int _downloadId = 0;
         public void CancelUpload()
         {
@@ -149,72 +151,94 @@ namespace MareSynchronos.WebAPI
             var uploadToken = _uploadCancellationTokenSource.Token;
             Logger.Verbose("New Token Created");
 
-            var filesToUpload = await _mareHub!.InvokeAsync<List<UploadFileDto>>(Api.InvokeFileSendFiles, character.FileReplacements.SelectMany(c => c.Value.Select(v => v.Hash)).Distinct(), uploadToken);
+            List<string> unverifiedUploadHashes = new();
-
+            foreach (var item in character.FileReplacements.SelectMany(c => c.Value.Select(v => v.Hash).Distinct()).Distinct().ToList())
-            foreach (var file in filesToUpload.Where(f => !f.IsForbidden))
             {
-                await using var db = new FileCacheContext();
+                if (!_verifiedUploadedHashes.Contains(item))
-                try
                 {
-                    CurrentUploads.Add(new UploadFileTransfer(file)
+                    unverifiedUploadHashes.Add(item);
-                    {
-                        Total = new FileInfo(db.FileCaches.FirstOrDefault(f => f.Hash.ToLower() == file.Hash.ToLower())
-                            ?.Filepath ?? string.Empty).Length
-                    });
-                }
-                catch (Exception ex)
-                {
-                    Logger.Warn("Tried to request file " + file.Hash + " but file was not present");
-                    Logger.Warn(ex.StackTrace!);
                 }
             }
 
-            await using (var db = new FileCacheContext())
+            if (unverifiedUploadHashes.Any())
             {
-                foreach (var file in filesToUpload.Where(c => c.IsForbidden))
+                Logger.Debug("Verifying " + unverifiedUploadHashes.Count + " files");
+                var filesToUpload = await _mareHub!.InvokeAsync<List<UploadFileDto>>(Api.InvokeFileSendFiles, unverifiedUploadHashes, uploadToken);
+
+                foreach (var file in filesToUpload.Where(f => !f.IsForbidden))
                 {
-                    if (ForbiddenTransfers.All(f => f.Hash != file.Hash))
+                    await using var db = new FileCacheContext();
+                    try
                     {
-                        ForbiddenTransfers.Add(new UploadFileTransfer(file)
+                        CurrentUploads.Add(new UploadFileTransfer(file)
                         {
-                            LocalFile = db.FileCaches.FirstOrDefault(f => f.Hash == file.Hash)?.Filepath ?? string.Empty
+                            Total = new FileInfo(db.FileCaches.FirstOrDefault(f => f.Hash.ToLower() == file.Hash.ToLower())
+                                ?.Filepath ?? string.Empty).Length
                         });
                     }
+                    catch (Exception ex)
+                    {
+                        Logger.Warn("Tried to request file " + file.Hash + " but file was not present");
+                        Logger.Warn(ex.StackTrace!);
+                    }
+                }
+
+                await using (var db = new FileCacheContext())
+                {
+                    foreach (var file in filesToUpload.Where(c => c.IsForbidden))
+                    {
+                        if (ForbiddenTransfers.All(f => f.Hash != file.Hash))
+                        {
+                            ForbiddenTransfers.Add(new UploadFileTransfer(file)
+                            {
+                                LocalFile = db.FileCaches.FirstOrDefault(f => f.Hash == file.Hash)?.Filepath ?? string.Empty
+                            });
+                        }
+                    }
+                }
+
+                var totalSize = CurrentUploads.Sum(c => c.Total);
+                Logger.Debug("Compressing and uploading files");
+                foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList())
+                {
+                    Logger.Debug("Compressing and uploading " + file);
+                    var data = await GetCompressedFileData(file.Hash, uploadToken);
+                    CurrentUploads.Single(e => e.Hash == data.Item1).Total = data.Item2.Length;
+                    await UploadFile(data.Item2, file.Hash, uploadToken);
+                    if (!uploadToken.IsCancellationRequested) continue;
+                    Logger.Warn("Cancel in filesToUpload loop detected");
+                    CurrentUploads.Clear();
+                    break;
+                }
+
+                if (CurrentUploads.Any())
+                {
+                    var compressedSize = CurrentUploads.Sum(c => c.Total);
+                    Logger.Debug($"Compressed {totalSize} to {compressedSize} ({(compressedSize / (double)totalSize):P2})");
+                }
+
+                Logger.Debug("Upload tasks complete, waiting for server to confirm");
+                var anyUploadsOpen = await _mareHub!.InvokeAsync<bool>(Api.InvokeFileIsUploadFinished, uploadToken);
+                Logger.Debug("Uploads open: " + anyUploadsOpen);
+                while (anyUploadsOpen && !uploadToken.IsCancellationRequested)
+                {
+                    anyUploadsOpen = await _mareHub!.InvokeAsync<bool>(Api.InvokeFileIsUploadFinished, uploadToken);
+                    await Task.Delay(TimeSpan.FromSeconds(0.5), uploadToken);
+                    Logger.Debug("Waiting for uploads to finish");
+                }
+
+                foreach (var item in unverifiedUploadHashes)
+                {
+                    _verifiedUploadedHashes.Add(item);
                 }
-            }
 
-            var totalSize = CurrentUploads.Sum(c => c.Total);
-            Logger.Debug("Compressing and uploading files");
-            foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList())
-            {
-                Logger.Debug("Compressing and uploading " + file);
-                var data = await GetCompressedFileData(file.Hash, uploadToken);
-                CurrentUploads.Single(e => e.Hash == data.Item1).Total = data.Item2.Length;
-                await UploadFile(data.Item2, file.Hash, uploadToken);
-                if (!uploadToken.IsCancellationRequested) continue;
-                Logger.Warn("Cancel in filesToUpload loop detected");
                 CurrentUploads.Clear();
-                break;
             }
-
+            else
-            if (CurrentUploads.Any())
             {
-                var compressedSize = CurrentUploads.Sum(c => c.Total);
+                Logger.Debug("All files already verified");
-                Logger.Debug($"Compressed {totalSize} to {compressedSize} ({(compressedSize / (double)totalSize):P2})");
             }
 
-            Logger.Debug("Upload tasks complete, waiting for server to confirm");
-            var anyUploadsOpen = await _mareHub!.InvokeAsync<bool>(Api.InvokeFileIsUploadFinished, uploadToken);
-            Logger.Debug("Uploads open: " + anyUploadsOpen);
-            while (anyUploadsOpen && !uploadToken.IsCancellationRequested)
-            {
-                anyUploadsOpen = await _mareHub!.InvokeAsync<bool>(Api.InvokeFileIsUploadFinished, uploadToken);
-                await Task.Delay(TimeSpan.FromSeconds(0.5), uploadToken);
-                Logger.Debug("Waiting for uploads to finish");
-            }
-
-            CurrentUploads.Clear();
-
             if (!uploadToken.IsCancellationRequested)
             {
                 Logger.Info("Pushing character data for " + character.GetHashCode() + " to " + string.Join(", ", visibleCharacterIds));

MareSynchronos/WebAPI/ApiController.Connectivity.cs

@@ -57,6 +57,7 @@ namespace MareSynchronos.WebAPI
             _dalamudUtil.LogIn += DalamudUtilOnLogIn;
             _dalamudUtil.LogOut += DalamudUtilOnLogOut;
             ServerState = ServerState.Offline;
+            _verifiedUploadedHashes = new();
 
             if (_dalamudUtil.IsLoggedIn)
             {
@@ -141,6 +142,7 @@ namespace MareSynchronos.WebAPI
             _connectionCancellationTokenSource.Cancel();
             _connectionCancellationTokenSource = new CancellationTokenSource();
             var token = _connectionCancellationTokenSource.Token;
+            _verifiedUploadedHashes.Clear();
             while (ServerState is not ServerState.Connected && !token.IsCancellationRequested)
             {
                 if (string.IsNullOrEmpty(SecretKey))