group downloads by host, download only one file per host at once

MareSynchronos/WebAPI/ApIController.Functions.Files.cs

@@ -47,9 +47,9 @@ public partial class ApiController
     {
         using var client = new HttpClient();
         client.DefaultRequestHeaders.Add(AuthorizationJwtHeader.Key, AuthorizationJwtHeader.Value);
-        int attempts = 0;
+        int attempts = 1;
         bool failed = true;
-        const int maxAttempts = 10;
+        const int maxAttempts = 16;
 
         HttpResponseMessage response = null!;
         HttpStatusCode? lastError = HttpStatusCode.OK;
@@ -66,13 +66,14 @@ public partial class ApiController
             catch (HttpRequestException ex)
             {
                 Logger.Warn($"Attempt {attempts}: Error during download of {bypassUrl}, HttpStatusCode: {ex.StatusCode}");
+                bypassUrl = new Uri(url, "?nocache=" + DateTime.UtcNow.Ticks);
                 lastError = ex.StatusCode;
                 if (ex.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Unauthorized)
                 {
                     break;
                 }
                 attempts++;
-                await Task.Delay(TimeSpan.FromSeconds(new Random().Next(1, 5)), ct).ConfigureAwait(false);
+                await Task.Delay(TimeSpan.FromSeconds(new Random().NextDouble() * 2), ct).ConfigureAwait(false);
             }
         }
 
@@ -155,12 +156,16 @@ public partial class ApiController
             }
         }
 
-        await Parallel.ForEachAsync(CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred), new ParallelOptions()
+        var downloadGroups = CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred).GroupBy(f => f.DownloadUri.Host, StringComparer.Ordinal);
+
+        await Parallel.ForEachAsync(downloadGroups, new ParallelOptions()
         {
-            MaxDegreeOfParallelism = 2,
+            MaxDegreeOfParallelism = downloadGroups.Count(),
             CancellationToken = ct
         },
-        async (file, token) =>
+        async (fileGroup, token) =>
+        {
+            foreach (var file in fileGroup)
             {
                 Logger.Debug($"Downloading {file.DownloadUri}");
                 var hash = file.Hash;
@@ -205,6 +210,7 @@ public partial class ApiController
                     Logger.Warn(ex.Message);
                     Logger.Warn(ex.StackTrace);
                 }
+            }
         }).ConfigureAwait(false);
 
         Logger.Debug("Download complete, removing " + currentDownloadId);