fixes for file uploads

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs

@@ -206,16 +206,25 @@ namespace MareSynchronosServer.Hubs
                     return;
                 }
 
-                UploadFileRequest req = new();
-                req.FileData = ByteString.CopyFrom(await File.ReadAllBytesAsync(tempFileName).ConfigureAwait(false));
-                File.Delete(tempFileName);
-                req.Hash = computedHashString;
-                req.Uploader = AuthenticatedUserId;
                 Metadata headers = new Metadata()
                 {
                     { "Authorization", Context.User!.Claims.SingleOrDefault(c => c.Type == ClaimTypes.Authentication)?.Value }
                 };
-                _ = await _fileServiceClient.UploadFileAsync(req, headers).ConfigureAwait(false);
+                var streamingCall = _fileServiceClient.UploadFile(headers);
+                using var tempFileStream = new FileStream(tempFileName, FileMode.Open, FileAccess.Read);
+                int size = 1024 * 1024;
+                byte[] data = new byte[size];
+                int readBytes;
+                while ((readBytes = tempFileStream.Read(data, 0, size)) > 0)
+                {
+                    await streamingCall.RequestStream.WriteAsync(new UploadFileRequest()
+                    {
+                        FileData = ByteString.CopyFrom(data, 0, readBytes),
+                        Hash = computedHashString,
+                        Uploader = AuthenticatedUserId
+                    }).ConfigureAwait(false);
+                }
+                await streamingCall.RequestStream.CompleteAsync();
             }
             catch (Exception ex)
             {

MareSynchronosServer/MareSynchronosServices/CleanupService.cs

@@ -85,7 +85,7 @@ namespace MareSynchronosServices
 
                     _logger.LogInformation("Cleaning up users older than {usersOlderThanDays} days", usersOlderThanDays);
 
-                    var allUsers = dbContext.Users.ToList();
+                    var allUsers = dbContext.Users.Where(u => string.IsNullOrEmpty(u.Alias)).ToList();
                     List<User> usersToRemove = new();
                     foreach (var user in allUsers)
                     {

MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto

@@ -11,7 +11,7 @@ service AuthService {
 }
 
 service FileService {
-	rpc UploadFile (UploadFileRequest) returns (Empty);
+	rpc UploadFile (stream UploadFileRequest) returns (Empty);
 	rpc GetFileSizes (FileSizeRequest) returns (FileSizeResponse);
 	rpc DeleteFiles (DeleteFilesRequest) returns (Empty);
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs

@@ -27,21 +27,33 @@ public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBa
         _metricsClient = metricsClient;
     }
 
-    public override async Task<Empty> UploadFile(UploadFileRequest request, ServerCallContext context)
+    public override async Task<Empty> UploadFile(IAsyncStreamReader<UploadFileRequest> requestStream, ServerCallContext context)
     {
-        var filePath = Path.Combine(_basePath, request.Hash);
+        await requestStream.MoveNext();
-        var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == request.Hash && f.UploaderUID == request.Uploader);
+        var uploadMsg = requestStream.Current;
+        var filePath = Path.Combine(_basePath, uploadMsg.Hash);
+        using var fileWriter = File.OpenWrite(filePath);
+        var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == uploadMsg.Hash && f.UploaderUID == uploadMsg.Uploader);
         if (file != null)
         {
-            var byteData = request.FileData.ToArray();
+            await fileWriter.WriteAsync(uploadMsg.FileData.ToArray());
-            await File.WriteAllBytesAsync(filePath, byteData);
+
+            while (await requestStream.MoveNext())
+            {
+                await fileWriter.WriteAsync(requestStream.Current.FileData.ToArray());
+            }
+
+            await fileWriter.FlushAsync();
+            fileWriter.Close();
+
+            var fileSize = new FileInfo(filePath).Length;
             file.Uploaded = true;
 
             _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotal, 1);
-            _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotalSize, byteData.Length);
+            _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotalSize, fileSize);
 
             await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
-            _logger.LogInformation("User {user} uploaded file {hash}", request.Uploader, request.Hash);
+            _logger.LogInformation("User {user} uploaded file {hash}", uploadMsg.Uploader, uploadMsg.Hash);
         }
 
         return new Empty();