[Draft] Update 0.8 (#46)
* move stuff out into file transfer manager * obnoxious unsupported version text, adjustments to filetransfermanager * add back file upload transfer progress * restructure code * cleanup some more stuff I guess * downloadids by playername * individual anim/sound bs * fix migration stuff, finalize impl of individual sound/anim pause * fixes with logging stuff * move download manager to transient * rework dl ui first iteration * some refactoring and cleanup * more code cleanup * refactoring * switch to hostbuilder * some more rework I guess * more refactoring * clean up mediator calls and disposal * fun code cleanup * push error message when log level is set to anything but information in non-debug builds * remove notificationservice * move message to after login * add download bars to gameworld * fixes download progress bar * set gpose ui min and max size * remove unnecessary usings * adjustments to reconnection logic * add options to set visible/offline groups visibility * add impl of uploading display, transfer list in settings ui * attempt to fix issues with server selection * add back download status to compact ui * make dl bar fixed size based * some fixes for upload/download handling * adjust text from Syncing back to Uploading --------- Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com> Co-authored-by: Stanley Dimant <stanley.dimant@varian.com>
This commit is contained in:
rootdarkarchon
226
MareSynchronos/WebAPI/Files/FileUploadManager.cs
Normal file
226
MareSynchronos/WebAPI/Files/FileUploadManager.cs
Normal file
@@ -0,0 +1,226 @@
|
||||
using LZ4;
|
||||
using MareSynchronos.API.Data;
|
||||
using MareSynchronos.API.Dto.Files;
|
||||
using MareSynchronos.API.Routes;
|
||||
using MareSynchronos.FileCache;
|
||||
using MareSynchronos.Services.Mediator;
|
||||
using MareSynchronos.Services.ServerConfiguration;
|
||||
using MareSynchronos.UI;
|
||||
using MareSynchronos.WebAPI.Files.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
|
||||
namespace MareSynchronos.WebAPI.Files;
|
||||
|
||||
public sealed class FileUploadManager : DisposableMediatorSubscriberBase
|
||||
{
|
||||
private readonly FileCacheManager _fileDbManager;
|
||||
private readonly FileTransferOrchestrator _orchestrator;
|
||||
private readonly ServerConfigurationManager _serverManager;
|
||||
private readonly Dictionary<string, DateTime> _verifiedUploadedHashes = new(StringComparer.Ordinal);
|
||||
private CancellationTokenSource? _uploadCancellationTokenSource = new();
|
||||
|
||||
public FileUploadManager(ILogger<FileUploadManager> logger, MareMediator mediator,
|
||||
FileTransferOrchestrator orchestrator,
|
||||
FileCacheManager fileDbManager,
|
||||
ServerConfigurationManager serverManager) : base(logger, mediator)
|
||||
{
|
||||
_orchestrator = orchestrator;
|
||||
_fileDbManager = fileDbManager;
|
||||
_serverManager = serverManager;
|
||||
|
||||
Mediator.Subscribe<DisconnectedMessage>(this, (msg) =>
|
||||
{
|
||||
Reset();
|
||||
});
|
||||
}
|
||||
|
||||
public List<FileTransfer> CurrentUploads { get; } = new();
|
||||
public bool IsUploading => CurrentUploads.Count > 0;
|
||||
|
||||
public bool CancelUpload()
|
||||
{
|
||||
if (CurrentUploads.Any())
|
||||
{
|
||||
Logger.LogDebug("Cancelling current upload");
|
||||
_uploadCancellationTokenSource?.Cancel();
|
||||
_uploadCancellationTokenSource?.Dispose();
|
||||
_uploadCancellationTokenSource = null;
|
||||
CurrentUploads.Clear();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public async Task DeleteAllFiles()
|
||||
{
|
||||
if (!_orchestrator.IsInitialized) throw new InvalidOperationException("FileTransferManager is not initialized");
|
||||
|
||||
await _orchestrator.SendRequestAsync(HttpMethod.Post, MareFiles.ServerFilesDeleteAllFullPath(_orchestrator.FilesCdnUri!)).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<CharacterData> UploadFiles(CharacterData data, List<UserData> visiblePlayers)
|
||||
{
|
||||
CancelUpload();
|
||||
|
||||
_uploadCancellationTokenSource = new CancellationTokenSource();
|
||||
var uploadToken = _uploadCancellationTokenSource.Token;
|
||||
Logger.LogDebug("Sending Character data {hash} to service {url}", data.DataHash.Value, _serverManager.CurrentApiUrl);
|
||||
|
||||
HashSet<string> unverifiedUploads = GetUnverifiedFiles(data);
|
||||
if (unverifiedUploads.Any())
|
||||
{
|
||||
await UploadUnverifiedFiles(unverifiedUploads, visiblePlayers, uploadToken).ConfigureAwait(false);
|
||||
Logger.LogInformation("Upload complete for {hash}", data.DataHash.Value);
|
||||
}
|
||||
|
||||
foreach (var kvp in data.FileReplacements)
|
||||
{
|
||||
data.FileReplacements[kvp.Key].RemoveAll(i => _orchestrator.ForbiddenTransfers.Any(f => string.Equals(f.Hash, i.Hash, StringComparison.OrdinalIgnoreCase)));
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
base.Dispose(disposing);
|
||||
Reset();
|
||||
}
|
||||
|
||||
private async Task<List<UploadFileDto>> FilesSend(List<string> hashes, List<string> uids, CancellationToken ct)
|
||||
{
|
||||
if (!_orchestrator.IsInitialized) throw new InvalidOperationException("FileTransferManager is not initialized");
|
||||
FilesSendDto filesSendDto = new()
|
||||
{
|
||||
FileHashes = hashes,
|
||||
UIDs = uids
|
||||
};
|
||||
var response = await _orchestrator.SendRequestAsync(HttpMethod.Post, MareFiles.ServerFilesFilesSendFullPath(_orchestrator.FilesCdnUri!), filesSendDto, ct).ConfigureAwait(false);
|
||||
return await response.Content.ReadFromJsonAsync<List<UploadFileDto>>(cancellationToken: ct).ConfigureAwait(false) ?? new List<UploadFileDto>();
|
||||
}
|
||||
|
||||
private async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken)
|
||||
{
|
||||
var fileCache = _fileDbManager.GetFileCacheByHash(fileHash)!.ResolvedFilepath;
|
||||
return (fileHash, LZ4Codec.WrapHC(await File.ReadAllBytesAsync(fileCache, uploadToken).ConfigureAwait(false), 0,
|
||||
(int)new FileInfo(fileCache).Length));
|
||||
}
|
||||
|
||||
private HashSet<string> GetUnverifiedFiles(CharacterData data)
|
||||
{
|
||||
HashSet<string> unverifiedUploadHashes = new(StringComparer.Ordinal);
|
||||
foreach (var item in data.FileReplacements.SelectMany(c => c.Value.Where(f => string.IsNullOrEmpty(f.FileSwapPath)).Select(v => v.Hash).Distinct(StringComparer.Ordinal)).Distinct(StringComparer.Ordinal).ToList())
|
||||
{
|
||||
if (!_verifiedUploadedHashes.TryGetValue(item, out var verifiedTime))
|
||||
{
|
||||
verifiedTime = DateTime.MinValue;
|
||||
}
|
||||
|
||||
if (verifiedTime < DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10)))
|
||||
{
|
||||
Logger.LogTrace("Verifying {item}, last verified: {date}", item, verifiedTime);
|
||||
unverifiedUploadHashes.Add(item);
|
||||
}
|
||||
}
|
||||
|
||||
return unverifiedUploadHashes;
|
||||
}
|
||||
|
||||
private void Reset()
|
||||
{
|
||||
_uploadCancellationTokenSource?.Cancel();
|
||||
_uploadCancellationTokenSource?.Dispose();
|
||||
_uploadCancellationTokenSource = null;
|
||||
CurrentUploads.Clear();
|
||||
_verifiedUploadedHashes.Clear();
|
||||
}
|
||||
|
||||
private async Task UploadFile(byte[] compressedFile, string fileHash, CancellationToken uploadToken)
|
||||
{
|
||||
if (!_orchestrator.IsInitialized) throw new InvalidOperationException("FileTransferManager is not initialized");
|
||||
|
||||
Logger.LogInformation("Uploading {file}, {size}", fileHash, UiSharedService.ByteToString(compressedFile.Length));
|
||||
|
||||
if (uploadToken.IsCancellationRequested) return;
|
||||
|
||||
using var ms = new MemoryStream(compressedFile);
|
||||
|
||||
Progress<UploadProgress> prog = new((prog) =>
|
||||
{
|
||||
CurrentUploads.Single(f => string.Equals(f.Hash, fileHash, StringComparison.Ordinal)).Transferred = prog.Uploaded;
|
||||
});
|
||||
var streamContent = new ProgressableStreamContent(ms, prog);
|
||||
streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
|
||||
|
||||
var response = await _orchestrator.SendRequestStreamAsync(HttpMethod.Post, MareFiles.ServerFilesUploadFullPath(_orchestrator.FilesCdnUri!, fileHash), streamContent, uploadToken).ConfigureAwait(false);
|
||||
Logger.LogDebug("Upload Status: {status}", response.StatusCode);
|
||||
}
|
||||
|
||||
private async Task UploadUnverifiedFiles(HashSet<string> unverifiedUploadHashes, List<UserData> visiblePlayers, CancellationToken uploadToken)
|
||||
{
|
||||
unverifiedUploadHashes = unverifiedUploadHashes.Where(h => _fileDbManager.GetFileCacheByHash(h) != null).ToHashSet(StringComparer.Ordinal);
|
||||
|
||||
Logger.LogDebug("Verifying {count} files", unverifiedUploadHashes.Count);
|
||||
var filesToUpload = await FilesSend(unverifiedUploadHashes.ToList(), visiblePlayers.Select(p => p.UID).ToList(), uploadToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var file in filesToUpload.Where(f => !f.IsForbidden))
|
||||
{
|
||||
try
|
||||
{
|
||||
CurrentUploads.Add(new UploadFileTransfer(file)
|
||||
{
|
||||
Total = new FileInfo(_fileDbManager.GetFileCacheByHash(file.Hash)!.ResolvedFilepath).Length,
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Tried to request file {hash} but file was not present", file.Hash);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var file in filesToUpload.Where(c => c.IsForbidden))
|
||||
{
|
||||
if (_orchestrator.ForbiddenTransfers.All(f => !string.Equals(f.Hash, file.Hash, StringComparison.Ordinal)))
|
||||
{
|
||||
_orchestrator.ForbiddenTransfers.Add(new UploadFileTransfer(file)
|
||||
{
|
||||
LocalFile = _fileDbManager.GetFileCacheByHash(file.Hash)?.ResolvedFilepath ?? string.Empty,
|
||||
});
|
||||
}
|
||||
|
||||
_verifiedUploadedHashes[file.Hash] = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
var totalSize = CurrentUploads.Sum(c => c.Total);
|
||||
Logger.LogDebug("Compressing and uploading files");
|
||||
Task uploadTask = Task.CompletedTask;
|
||||
foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList())
|
||||
{
|
||||
Logger.LogDebug("Compressing {file}", file);
|
||||
var data = await GetCompressedFileData(file.Hash, uploadToken).ConfigureAwait(false);
|
||||
CurrentUploads.Single(e => string.Equals(e.Hash, data.Item1, StringComparison.Ordinal)).Total = data.Item2.Length;
|
||||
await uploadTask.ConfigureAwait(false);
|
||||
uploadTask = UploadFile(data.Item2, file.Hash, uploadToken);
|
||||
uploadToken.ThrowIfCancellationRequested();
|
||||
_verifiedUploadedHashes[file.Hash] = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
if (CurrentUploads.Any())
|
||||
{
|
||||
await uploadTask.ConfigureAwait(false);
|
||||
|
||||
var compressedSize = CurrentUploads.Sum(c => c.Total);
|
||||
Logger.LogDebug("Upload complete, compressed {size} to {compressed}", UiSharedService.ByteToString(totalSize), UiSharedService.ByteToString(compressedSize));
|
||||
}
|
||||
|
||||
foreach (var file in unverifiedUploadHashes.Where(c => !CurrentUploads.Any(u => string.Equals(u.Hash, c, StringComparison.Ordinal))))
|
||||
{
|
||||
_verifiedUploadedHashes[file] = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
CurrentUploads.Clear();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user