From 3a6c9259a0dd7aab8433c4483efc96173ba38da0 Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Sat, 27 Aug 2022 01:13:49 +0200 Subject: [PATCH] fixes for file uploads --- .../Hubs/MareHub.Files.cs | 21 ++++++++++----- .../MareSynchronosServices/CleanupService.cs | 2 +- .../Protos/mareservices.proto | 2 +- .../FileService.cs | 26 ++++++++++++++----- 4 files changed, 36 insertions(+), 15 deletions(-) diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs index fb882d9..9e7158b 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs @@ -206,16 +206,25 @@ namespace MareSynchronosServer.Hubs return; } - UploadFileRequest req = new(); - req.FileData = ByteString.CopyFrom(await File.ReadAllBytesAsync(tempFileName).ConfigureAwait(false)); - File.Delete(tempFileName); - req.Hash = computedHashString; - req.Uploader = AuthenticatedUserId; Metadata headers = new Metadata() { { "Authorization", Context.User!.Claims.SingleOrDefault(c => c.Type == ClaimTypes.Authentication)?.Value } }; - _ = await _fileServiceClient.UploadFileAsync(req, headers).ConfigureAwait(false); + var streamingCall = _fileServiceClient.UploadFile(headers); + using var tempFileStream = new FileStream(tempFileName, FileMode.Open, FileAccess.Read); + int size = 1024 * 1024; + byte[] data = new byte[size]; + int readBytes; + while ((readBytes = tempFileStream.Read(data, 0, size)) > 0) + { + await streamingCall.RequestStream.WriteAsync(new UploadFileRequest() + { + FileData = ByteString.CopyFrom(data, 0, readBytes), + Hash = computedHashString, + Uploader = AuthenticatedUserId + }).ConfigureAwait(false); + } + await streamingCall.RequestStream.CompleteAsync(); } catch (Exception ex) { diff --git a/MareSynchronosServer/MareSynchronosServices/CleanupService.cs b/MareSynchronosServer/MareSynchronosServices/CleanupService.cs index 313b976..4e1db29 100644 --- a/MareSynchronosServer/MareSynchronosServices/CleanupService.cs +++ b/MareSynchronosServer/MareSynchronosServices/CleanupService.cs @@ -85,7 +85,7 @@ namespace MareSynchronosServices _logger.LogInformation("Cleaning up users older than {usersOlderThanDays} days", usersOlderThanDays); - var allUsers = dbContext.Users.ToList(); + var allUsers = dbContext.Users.Where(u => string.IsNullOrEmpty(u.Alias)).ToList(); List usersToRemove = new(); foreach (var user in allUsers) { diff --git a/MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto b/MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto index 5d38bbc..09d2f06 100644 --- a/MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto +++ b/MareSynchronosServer/MareSynchronosShared/Protos/mareservices.proto @@ -11,7 +11,7 @@ service AuthService { } service FileService { - rpc UploadFile (UploadFileRequest) returns (Empty); + rpc UploadFile (stream UploadFileRequest) returns (Empty); rpc GetFileSizes (FileSizeRequest) returns (FileSizeResponse); rpc DeleteFiles (DeleteFilesRequest) returns (Empty); } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs index 08aafb0..1a234c8 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/FileService.cs @@ -27,21 +27,33 @@ public class FileService : MareSynchronosShared.Protos.FileService.FileServiceBa _metricsClient = metricsClient; } - public override async Task UploadFile(UploadFileRequest request, ServerCallContext context) + public override async Task UploadFile(IAsyncStreamReader requestStream, ServerCallContext context) { - var filePath = Path.Combine(_basePath, request.Hash); - var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == request.Hash && f.UploaderUID == request.Uploader); + await requestStream.MoveNext(); + var uploadMsg = requestStream.Current; + var filePath = Path.Combine(_basePath, uploadMsg.Hash); + using var fileWriter = File.OpenWrite(filePath); + var file = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == uploadMsg.Hash && f.UploaderUID == uploadMsg.Uploader); if (file != null) { - var byteData = request.FileData.ToArray(); - await File.WriteAllBytesAsync(filePath, byteData); + await fileWriter.WriteAsync(uploadMsg.FileData.ToArray()); + + while (await requestStream.MoveNext()) + { + await fileWriter.WriteAsync(requestStream.Current.FileData.ToArray()); + } + + await fileWriter.FlushAsync(); + fileWriter.Close(); + + var fileSize = new FileInfo(filePath).Length; file.Uploaded = true; _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotal, 1); - _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotalSize, byteData.Length); + _metricsClient.IncGauge(MetricsAPI.GaugeFilesTotalSize, fileSize); await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); - _logger.LogInformation("User {user} uploaded file {hash}", request.Uploader, request.Hash); + _logger.LogInformation("User {user} uploaded file {hash}", uploadMsg.Uploader, uploadMsg.Hash); } return new Empty();