[Draft] Update 0.8 (#46)
* move stuff out into file transfer manager * obnoxious unsupported version text, adjustments to filetransfermanager * add back file upload transfer progress * restructure code * cleanup some more stuff I guess * downloadids by playername * individual anim/sound bs * fix migration stuff, finalize impl of individual sound/anim pause * fixes with logging stuff * move download manager to transient * rework dl ui first iteration * some refactoring and cleanup * more code cleanup * refactoring * switch to hostbuilder * some more rework I guess * more refactoring * clean up mediator calls and disposal * fun code cleanup * push error message when log level is set to anything but information in non-debug builds * remove notificationservice * move message to after login * add download bars to gameworld * fixes download progress bar * set gpose ui min and max size * remove unnecessary usings * adjustments to reconnection logic * add options to set visible/offline groups visibility * add impl of uploading display, transfer list in settings ui * attempt to fix issues with server selection * add back download status to compact ui * make dl bar fixed size based * some fixes for upload/download handling * adjust text from Syncing back to Uploading --------- Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com> Co-authored-by: Stanley Dimant <stanley.dimant@varian.com>
This commit is contained in:
rootdarkarchon
341
MareSynchronos/WebAPI/Files/FileDownloadManager.cs
Normal file
341
MareSynchronos/WebAPI/Files/FileDownloadManager.cs
Normal file
@@ -0,0 +1,341 @@
|
||||
using Dalamud.Utility;
|
||||
using LZ4;
|
||||
using MareSynchronos.API.Data;
|
||||
using MareSynchronos.API.Dto.Files;
|
||||
using MareSynchronos.API.Routes;
|
||||
using MareSynchronos.FileCache;
|
||||
using MareSynchronos.PlayerData.Handlers;
|
||||
using MareSynchronos.Services.Mediator;
|
||||
using MareSynchronos.WebAPI.Files.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
|
||||
namespace MareSynchronos.WebAPI.Files;
|
||||
|
||||
public partial class FileDownloadManager : DisposableMediatorSubscriberBase
|
||||
{
|
||||
private readonly ConcurrentDictionary<Guid, bool> _downloadReady = new();
|
||||
private readonly Dictionary<string, FileDownloadStatus> _downloadStatus;
|
||||
private readonly FileCacheManager _fileDbManager;
|
||||
private readonly FileTransferOrchestrator _orchestrator;
|
||||
|
||||
public FileDownloadManager(ILogger<FileDownloadManager> logger, MareMediator mediator,
|
||||
FileTransferOrchestrator orchestrator,
|
||||
FileCacheManager fileCacheManager) : base(logger, mediator)
|
||||
{
|
||||
_downloadStatus = new Dictionary<string, FileDownloadStatus>(StringComparer.Ordinal);
|
||||
_orchestrator = orchestrator;
|
||||
_fileDbManager = fileCacheManager;
|
||||
|
||||
Mediator.Subscribe<DownloadReadyMessage>(this, (msg) =>
|
||||
{
|
||||
if (_downloadReady.ContainsKey(msg.RequestId))
|
||||
{
|
||||
_downloadReady[msg.RequestId] = true;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public List<DownloadFileTransfer> CurrentDownloads { get; private set; } = new();
|
||||
public List<FileTransfer> ForbiddenTransfers => _orchestrator.ForbiddenTransfers;
|
||||
public bool IsDownloading => !CurrentDownloads.Any();
|
||||
|
||||
public void CancelDownload()
|
||||
{
|
||||
CurrentDownloads.Clear();
|
||||
_downloadStatus.Clear();
|
||||
}
|
||||
|
||||
public async Task DownloadFiles(GameObjectHandler gameObject, List<FileReplacementData> fileReplacementDto, CancellationToken ct)
|
||||
{
|
||||
Mediator.Publish(new HaltScanMessage("Download"));
|
||||
try
|
||||
{
|
||||
await DownloadFilesInternal(gameObject, fileReplacementDto, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
CancelDownload();
|
||||
}
|
||||
finally
|
||||
{
|
||||
Mediator.Publish(new DownloadFinishedMessage(gameObject));
|
||||
Mediator.Publish(new ResumeScanMessage("Download"));
|
||||
}
|
||||
}
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
CancelDownload();
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
|
||||
private async Task DownloadFileHttpClient(string downloadGroup, DownloadFileTransfer fileTransfer, string tempPath, IProgress<long> progress, CancellationToken ct)
|
||||
{
|
||||
var requestId = await GetQueueRequest(fileTransfer, ct).ConfigureAwait(false);
|
||||
|
||||
Logger.LogDebug("GUID {requestId} for file {hash} on server {uri}", requestId, fileTransfer.Hash, fileTransfer.DownloadUri);
|
||||
|
||||
await WaitForDownloadReady(fileTransfer, requestId, ct).ConfigureAwait(false);
|
||||
|
||||
_downloadStatus[downloadGroup].DownloadStatus = DownloadStatus.Downloading;
|
||||
|
||||
HttpResponseMessage response = null!;
|
||||
var requestUrl = MareFiles.CacheGetFullPath(fileTransfer.DownloadUri, requestId);
|
||||
|
||||
Logger.LogDebug("Downloading {requestUrl} for file {hash}", requestUrl, fileTransfer.Hash);
|
||||
try
|
||||
{
|
||||
response = await _orchestrator.SendRequestAsync(HttpMethod.Get, requestUrl, ct).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Error during download of {requestUrl}, HttpStatusCode: {code}", requestUrl, ex.StatusCode);
|
||||
if (ex.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Unauthorized)
|
||||
{
|
||||
throw new InvalidDataException($"Http error {ex.StatusCode} (cancelled: {ct.IsCancellationRequested}): {requestUrl}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var fileStream = File.Create(tempPath);
|
||||
await using (fileStream.ConfigureAwait(false))
|
||||
{
|
||||
var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024;
|
||||
var buffer = new byte[bufferSize];
|
||||
|
||||
var bytesRead = 0;
|
||||
while ((bytesRead = await (await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false)).ReadAsync(buffer, ct).ConfigureAwait(false)) > 0)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead), ct).ConfigureAwait(false);
|
||||
|
||||
progress.Report(bytesRead);
|
||||
}
|
||||
|
||||
Logger.LogDebug("{requestUrl} downloaded to {tempPath}", requestUrl, tempPath);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Error during file download of {requestUrl}", requestUrl);
|
||||
try
|
||||
{
|
||||
if (!tempPath.IsNullOrEmpty())
|
||||
File.Delete(tempPath);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// ignore if file deletion fails
|
||||
}
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DownloadFilesInternal(GameObjectHandler gameObjectHandler, List<FileReplacementData> fileReplacement, CancellationToken ct)
|
||||
{
|
||||
Logger.LogDebug("Downloading files for {id}", gameObjectHandler.Name);
|
||||
|
||||
// force create lazy
|
||||
_ = gameObjectHandler.GameObjectLazy.Value;
|
||||
|
||||
List<DownloadFileDto> downloadFileInfoFromService = new();
|
||||
downloadFileInfoFromService.AddRange(await FilesGetSizes(fileReplacement.Select(f => f.Hash).ToList(), ct).ConfigureAwait(false));
|
||||
|
||||
Logger.LogDebug("Files with size 0 or less: {files}", string.Join(", ", downloadFileInfoFromService.Where(f => f.Size <= 0).Select(f => f.Hash)));
|
||||
|
||||
CurrentDownloads = downloadFileInfoFromService.Distinct().Select(d => new DownloadFileTransfer(d))
|
||||
.Where(d => d.CanBeTransferred).ToList();
|
||||
|
||||
foreach (var dto in downloadFileInfoFromService.Where(c => c.IsForbidden))
|
||||
{
|
||||
if (!_orchestrator.ForbiddenTransfers.Any(f => string.Equals(f.Hash, dto.Hash, StringComparison.Ordinal)))
|
||||
{
|
||||
_orchestrator.ForbiddenTransfers.Add(new DownloadFileTransfer(dto));
|
||||
}
|
||||
}
|
||||
|
||||
var downloadGroups = CurrentDownloads.Where(f => f.CanBeTransferred).GroupBy(f => f.DownloadUri.Host + ":" + f.DownloadUri.Port, StringComparer.Ordinal);
|
||||
|
||||
foreach (var downloadGroup in downloadGroups)
|
||||
{
|
||||
_downloadStatus[downloadGroup.Key] = new FileDownloadStatus()
|
||||
{
|
||||
DownloadStatus = DownloadStatus.Initializing,
|
||||
TotalBytes = downloadGroup.Sum(c => c.Total),
|
||||
TotalFiles = downloadGroup.Count(),
|
||||
TransferredBytes = 0,
|
||||
TransferredFiles = 0
|
||||
};
|
||||
}
|
||||
|
||||
Mediator.Publish(new DownloadStartedMessage(gameObjectHandler, _downloadStatus));
|
||||
|
||||
await Parallel.ForEachAsync(downloadGroups, new ParallelOptions()
|
||||
{
|
||||
MaxDegreeOfParallelism = downloadGroups.Count(),
|
||||
CancellationToken = ct,
|
||||
},
|
||||
async (fileGroup, token) =>
|
||||
{
|
||||
// let server predownload files
|
||||
await _orchestrator.SendRequestAsync(HttpMethod.Post, MareFiles.RequestEnqueueFullPath(fileGroup.First().DownloadUri),
|
||||
fileGroup.Select(c => c.Hash), token).ConfigureAwait(false);
|
||||
|
||||
foreach (var file in fileGroup)
|
||||
{
|
||||
var tempPath = _fileDbManager.GetCacheFilePath(file.Hash, isTemporaryFile: true);
|
||||
Progress<long> progress = new((bytesDownloaded) =>
|
||||
{
|
||||
if (!_downloadStatus.ContainsKey(fileGroup.Key)) return;
|
||||
_downloadStatus[fileGroup.Key].TransferredBytes += bytesDownloaded;
|
||||
file.Transferred += bytesDownloaded;
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
_downloadStatus[fileGroup.Key].DownloadStatus = DownloadStatus.WaitingForSlot;
|
||||
await _orchestrator.WaitForDownloadSlotAsync(token).ConfigureAwait(false);
|
||||
_downloadStatus[fileGroup.Key].DownloadStatus = DownloadStatus.WaitingForQueue;
|
||||
await DownloadFileHttpClient(fileGroup.Key, file, tempPath, progress, token).ConfigureAwait(false);
|
||||
_downloadStatus[fileGroup.Key].TransferredFiles += 1;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
File.Delete(tempPath);
|
||||
Logger.LogDebug("Detected cancellation, removing {id}", gameObjectHandler);
|
||||
CancelDownload();
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "Error during download of {hash}", file.Hash);
|
||||
continue;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_orchestrator.ReleaseDownloadSlot();
|
||||
}
|
||||
|
||||
_downloadStatus[fileGroup.Key].DownloadStatus = DownloadStatus.Decompressing;
|
||||
var tempFileData = await File.ReadAllBytesAsync(tempPath, token).ConfigureAwait(false);
|
||||
var extractedFile = LZ4Codec.Unwrap(tempFileData);
|
||||
File.Delete(tempPath);
|
||||
var filePath = _fileDbManager.GetCacheFilePath(file.Hash, isTemporaryFile: false);
|
||||
await File.WriteAllBytesAsync(filePath, extractedFile, token).ConfigureAwait(false);
|
||||
var fi = new FileInfo(filePath);
|
||||
Func<DateTime> RandomDayInThePast()
|
||||
{
|
||||
DateTime start = new(1995, 1, 1);
|
||||
Random gen = new();
|
||||
int range = (DateTime.Today - start).Days;
|
||||
return () => start.AddDays(gen.Next(range));
|
||||
}
|
||||
|
||||
fi.CreationTime = RandomDayInThePast().Invoke();
|
||||
fi.LastAccessTime = DateTime.Today;
|
||||
fi.LastWriteTime = RandomDayInThePast().Invoke();
|
||||
try
|
||||
{
|
||||
_ = _fileDbManager.CreateCacheEntry(filePath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Issue creating cache entry");
|
||||
}
|
||||
}
|
||||
}).ConfigureAwait(false);
|
||||
|
||||
Logger.LogDebug("Download for {id} complete", gameObjectHandler);
|
||||
CancelDownload();
|
||||
}
|
||||
|
||||
private async Task<List<DownloadFileDto>> FilesGetSizes(List<string> hashes, CancellationToken ct)
|
||||
{
|
||||
if (!_orchestrator.IsInitialized) throw new InvalidOperationException("FileTransferManager is not initialized");
|
||||
var response = await _orchestrator.SendRequestAsync(HttpMethod.Get, MareFiles.ServerFilesGetSizesFullPath(_orchestrator.FilesCdnUri!), hashes, ct).ConfigureAwait(false);
|
||||
return await response.Content.ReadFromJsonAsync<List<DownloadFileDto>>(cancellationToken: ct).ConfigureAwait(false) ?? new List<DownloadFileDto>();
|
||||
}
|
||||
|
||||
private async Task<Guid> GetQueueRequest(DownloadFileTransfer downloadFileTransfer, CancellationToken ct)
|
||||
{
|
||||
var response = await _orchestrator.SendRequestAsync(HttpMethod.Get, MareFiles.RequestRequestFileFullPath(downloadFileTransfer.DownloadUri, downloadFileTransfer.Hash), ct).ConfigureAwait(false);
|
||||
var responseString = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false);
|
||||
var requestId = Guid.Parse(responseString.Trim('"'));
|
||||
if (!_downloadReady.ContainsKey(requestId))
|
||||
{
|
||||
_downloadReady[requestId] = false;
|
||||
}
|
||||
return requestId;
|
||||
}
|
||||
|
||||
private async Task WaitForDownloadReady(DownloadFileTransfer downloadFileTransfer, Guid requestId, CancellationToken downloadCt)
|
||||
{
|
||||
bool alreadyCancelled = false;
|
||||
try
|
||||
{
|
||||
CancellationTokenSource localTimeoutCts = new();
|
||||
localTimeoutCts.CancelAfter(TimeSpan.FromSeconds(5));
|
||||
CancellationTokenSource composite = CancellationTokenSource.CreateLinkedTokenSource(downloadCt, localTimeoutCts.Token);
|
||||
|
||||
while (_downloadReady.TryGetValue(requestId, out bool isReady) && !isReady)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(250, composite.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
if (downloadCt.IsCancellationRequested) throw;
|
||||
|
||||
var req = await _orchestrator.SendRequestAsync(HttpMethod.Get, MareFiles.RequestCheckQueueFullPath(downloadFileTransfer.DownloadUri, requestId, downloadFileTransfer.Hash), downloadCt).ConfigureAwait(false);
|
||||
req.EnsureSuccessStatusCode();
|
||||
localTimeoutCts.Dispose();
|
||||
composite.Dispose();
|
||||
localTimeoutCts = new();
|
||||
localTimeoutCts.CancelAfter(TimeSpan.FromSeconds(5));
|
||||
composite = CancellationTokenSource.CreateLinkedTokenSource(downloadCt, localTimeoutCts.Token);
|
||||
}
|
||||
}
|
||||
|
||||
localTimeoutCts.Dispose();
|
||||
composite.Dispose();
|
||||
|
||||
Logger.LogDebug("Download {requestId} ready", requestId);
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _orchestrator.SendRequestAsync(HttpMethod.Get, MareFiles.RequestCancelFullPath(downloadFileTransfer.DownloadUri, requestId)).ConfigureAwait(false);
|
||||
alreadyCancelled = true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// ignore whatever happens here
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (downloadCt.IsCancellationRequested && !alreadyCancelled)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _orchestrator.SendRequestAsync(HttpMethod.Get, MareFiles.RequestCancelFullPath(downloadFileTransfer.DownloadUri, requestId)).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// ignore whatever happens here
|
||||
}
|
||||
}
|
||||
_downloadReady.Remove(requestId, out _);
|
||||
}
|
||||
}
|
||||
}
|
||||
109
MareSynchronos/WebAPI/Files/FileTransferOrchestrator.cs
Normal file
109
MareSynchronos/WebAPI/Files/FileTransferOrchestrator.cs
Normal file
@@ -0,0 +1,109 @@
|
||||
using MareSynchronos.MareConfiguration;
|
||||
using MareSynchronos.Services.Mediator;
|
||||
using MareSynchronos.Services.ServerConfiguration;
|
||||
using MareSynchronos.WebAPI.Files.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
|
||||
namespace MareSynchronos.WebAPI.Files;
|
||||
|
||||
public class FileTransferOrchestrator : DisposableMediatorSubscriberBase
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly MareConfigService _mareConfig;
|
||||
private readonly object _semaphoreModificationLock = new();
|
||||
private readonly ServerConfigurationManager _serverManager;
|
||||
private int _availableDownloadSlots;
|
||||
private SemaphoreSlim _downloadSemaphore;
|
||||
|
||||
public FileTransferOrchestrator(ILogger<FileTransferOrchestrator> logger, MareConfigService mareConfig, ServerConfigurationManager serverManager, MareMediator mediator) : base(logger, mediator)
|
||||
{
|
||||
_mareConfig = mareConfig;
|
||||
_serverManager = serverManager;
|
||||
_httpClient = new();
|
||||
|
||||
_availableDownloadSlots = mareConfig.Current.ParallelDownloads;
|
||||
_downloadSemaphore = new(_availableDownloadSlots);
|
||||
|
||||
Mediator.Subscribe<ConnectedMessage>(this, (msg) =>
|
||||
{
|
||||
FilesCdnUri = msg.Connection.ServerInfo.FileServerAddress;
|
||||
});
|
||||
|
||||
Mediator.Subscribe<DisconnectedMessage>(this, (msg) =>
|
||||
{
|
||||
FilesCdnUri = null;
|
||||
});
|
||||
}
|
||||
|
||||
public Uri? FilesCdnUri { private set; get; }
|
||||
public List<FileTransfer> ForbiddenTransfers { get; } = new();
|
||||
public bool IsInitialized => FilesCdnUri != null;
|
||||
|
||||
public void ReleaseDownloadSlot()
|
||||
{
|
||||
_downloadSemaphore.Release();
|
||||
}
|
||||
|
||||
public async Task<HttpResponseMessage> SendRequestAsync(HttpMethod method, Uri uri, CancellationToken? ct = null)
|
||||
{
|
||||
using var requestMessage = new HttpRequestMessage(method, uri);
|
||||
return await SendRequestInternalAsync(requestMessage, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<HttpResponseMessage> SendRequestAsync<T>(HttpMethod method, Uri uri, T content, CancellationToken ct) where T : class
|
||||
{
|
||||
using var requestMessage = new HttpRequestMessage(method, uri);
|
||||
requestMessage.Content = JsonContent.Create(content);
|
||||
return await SendRequestInternalAsync(requestMessage, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<HttpResponseMessage> SendRequestStreamAsync(HttpMethod method, Uri uri, ProgressableStreamContent content, CancellationToken ct)
|
||||
{
|
||||
using var requestMessage = new HttpRequestMessage(method, uri);
|
||||
requestMessage.Content = content;
|
||||
return await SendRequestInternalAsync(requestMessage, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task WaitForDownloadSlotAsync(CancellationToken token)
|
||||
{
|
||||
lock (_semaphoreModificationLock)
|
||||
{
|
||||
if (_availableDownloadSlots != _mareConfig.Current.ParallelDownloads && _availableDownloadSlots == _downloadSemaphore.CurrentCount)
|
||||
{
|
||||
_availableDownloadSlots = _mareConfig.Current.ParallelDownloads;
|
||||
_downloadSemaphore = new(_availableDownloadSlots);
|
||||
}
|
||||
}
|
||||
|
||||
await _downloadSemaphore.WaitAsync(token).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<HttpResponseMessage> SendRequestInternalAsync(HttpRequestMessage requestMessage, CancellationToken? ct = null)
|
||||
{
|
||||
requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _serverManager.GetToken());
|
||||
|
||||
if (requestMessage.Content != null && requestMessage.Content is not StreamContent)
|
||||
{
|
||||
var content = await ((JsonContent)requestMessage.Content).ReadAsStringAsync().ConfigureAwait(false);
|
||||
Logger.LogDebug("Sending {method} to {uri} (Content: {content})", requestMessage.Method, requestMessage.RequestUri, content);
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.LogDebug("Sending {method} to {uri}", requestMessage.Method, requestMessage.RequestUri);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (ct != null)
|
||||
return await _httpClient.SendAsync(requestMessage, ct.Value).ConfigureAwait(false);
|
||||
return await _httpClient.SendAsync(requestMessage).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogCritical(ex, "Error during SendRequestInternal for {uri}", requestMessage.RequestUri);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
226
MareSynchronos/WebAPI/Files/FileUploadManager.cs
Normal file
226
MareSynchronos/WebAPI/Files/FileUploadManager.cs
Normal file
@@ -0,0 +1,226 @@
|
||||
using LZ4;
|
||||
using MareSynchronos.API.Data;
|
||||
using MareSynchronos.API.Dto.Files;
|
||||
using MareSynchronos.API.Routes;
|
||||
using MareSynchronos.FileCache;
|
||||
using MareSynchronos.Services.Mediator;
|
||||
using MareSynchronos.Services.ServerConfiguration;
|
||||
using MareSynchronos.UI;
|
||||
using MareSynchronos.WebAPI.Files.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
|
||||
namespace MareSynchronos.WebAPI.Files;
|
||||
|
||||
public sealed class FileUploadManager : DisposableMediatorSubscriberBase
|
||||
{
|
||||
private readonly FileCacheManager _fileDbManager;
|
||||
private readonly FileTransferOrchestrator _orchestrator;
|
||||
private readonly ServerConfigurationManager _serverManager;
|
||||
private readonly Dictionary<string, DateTime> _verifiedUploadedHashes = new(StringComparer.Ordinal);
|
||||
private CancellationTokenSource? _uploadCancellationTokenSource = new();
|
||||
|
||||
public FileUploadManager(ILogger<FileUploadManager> logger, MareMediator mediator,
|
||||
FileTransferOrchestrator orchestrator,
|
||||
FileCacheManager fileDbManager,
|
||||
ServerConfigurationManager serverManager) : base(logger, mediator)
|
||||
{
|
||||
_orchestrator = orchestrator;
|
||||
_fileDbManager = fileDbManager;
|
||||
_serverManager = serverManager;
|
||||
|
||||
Mediator.Subscribe<DisconnectedMessage>(this, (msg) =>
|
||||
{
|
||||
Reset();
|
||||
});
|
||||
}
|
||||
|
||||
public List<FileTransfer> CurrentUploads { get; } = new();
|
||||
public bool IsUploading => CurrentUploads.Count > 0;
|
||||
|
||||
public bool CancelUpload()
|
||||
{
|
||||
if (CurrentUploads.Any())
|
||||
{
|
||||
Logger.LogDebug("Cancelling current upload");
|
||||
_uploadCancellationTokenSource?.Cancel();
|
||||
_uploadCancellationTokenSource?.Dispose();
|
||||
_uploadCancellationTokenSource = null;
|
||||
CurrentUploads.Clear();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public async Task DeleteAllFiles()
|
||||
{
|
||||
if (!_orchestrator.IsInitialized) throw new InvalidOperationException("FileTransferManager is not initialized");
|
||||
|
||||
await _orchestrator.SendRequestAsync(HttpMethod.Post, MareFiles.ServerFilesDeleteAllFullPath(_orchestrator.FilesCdnUri!)).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<CharacterData> UploadFiles(CharacterData data, List<UserData> visiblePlayers)
|
||||
{
|
||||
CancelUpload();
|
||||
|
||||
_uploadCancellationTokenSource = new CancellationTokenSource();
|
||||
var uploadToken = _uploadCancellationTokenSource.Token;
|
||||
Logger.LogDebug("Sending Character data {hash} to service {url}", data.DataHash.Value, _serverManager.CurrentApiUrl);
|
||||
|
||||
HashSet<string> unverifiedUploads = GetUnverifiedFiles(data);
|
||||
if (unverifiedUploads.Any())
|
||||
{
|
||||
await UploadUnverifiedFiles(unverifiedUploads, visiblePlayers, uploadToken).ConfigureAwait(false);
|
||||
Logger.LogInformation("Upload complete for {hash}", data.DataHash.Value);
|
||||
}
|
||||
|
||||
foreach (var kvp in data.FileReplacements)
|
||||
{
|
||||
data.FileReplacements[kvp.Key].RemoveAll(i => _orchestrator.ForbiddenTransfers.Any(f => string.Equals(f.Hash, i.Hash, StringComparison.OrdinalIgnoreCase)));
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
base.Dispose(disposing);
|
||||
Reset();
|
||||
}
|
||||
|
||||
private async Task<List<UploadFileDto>> FilesSend(List<string> hashes, List<string> uids, CancellationToken ct)
|
||||
{
|
||||
if (!_orchestrator.IsInitialized) throw new InvalidOperationException("FileTransferManager is not initialized");
|
||||
FilesSendDto filesSendDto = new()
|
||||
{
|
||||
FileHashes = hashes,
|
||||
UIDs = uids
|
||||
};
|
||||
var response = await _orchestrator.SendRequestAsync(HttpMethod.Post, MareFiles.ServerFilesFilesSendFullPath(_orchestrator.FilesCdnUri!), filesSendDto, ct).ConfigureAwait(false);
|
||||
return await response.Content.ReadFromJsonAsync<List<UploadFileDto>>(cancellationToken: ct).ConfigureAwait(false) ?? new List<UploadFileDto>();
|
||||
}
|
||||
|
||||
private async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken)
|
||||
{
|
||||
var fileCache = _fileDbManager.GetFileCacheByHash(fileHash)!.ResolvedFilepath;
|
||||
return (fileHash, LZ4Codec.WrapHC(await File.ReadAllBytesAsync(fileCache, uploadToken).ConfigureAwait(false), 0,
|
||||
(int)new FileInfo(fileCache).Length));
|
||||
}
|
||||
|
||||
private HashSet<string> GetUnverifiedFiles(CharacterData data)
|
||||
{
|
||||
HashSet<string> unverifiedUploadHashes = new(StringComparer.Ordinal);
|
||||
foreach (var item in data.FileReplacements.SelectMany(c => c.Value.Where(f => string.IsNullOrEmpty(f.FileSwapPath)).Select(v => v.Hash).Distinct(StringComparer.Ordinal)).Distinct(StringComparer.Ordinal).ToList())
|
||||
{
|
||||
if (!_verifiedUploadedHashes.TryGetValue(item, out var verifiedTime))
|
||||
{
|
||||
verifiedTime = DateTime.MinValue;
|
||||
}
|
||||
|
||||
if (verifiedTime < DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10)))
|
||||
{
|
||||
Logger.LogTrace("Verifying {item}, last verified: {date}", item, verifiedTime);
|
||||
unverifiedUploadHashes.Add(item);
|
||||
}
|
||||
}
|
||||
|
||||
return unverifiedUploadHashes;
|
||||
}
|
||||
|
||||
private void Reset()
|
||||
{
|
||||
_uploadCancellationTokenSource?.Cancel();
|
||||
_uploadCancellationTokenSource?.Dispose();
|
||||
_uploadCancellationTokenSource = null;
|
||||
CurrentUploads.Clear();
|
||||
_verifiedUploadedHashes.Clear();
|
||||
}
|
||||
|
||||
private async Task UploadFile(byte[] compressedFile, string fileHash, CancellationToken uploadToken)
|
||||
{
|
||||
if (!_orchestrator.IsInitialized) throw new InvalidOperationException("FileTransferManager is not initialized");
|
||||
|
||||
Logger.LogInformation("Uploading {file}, {size}", fileHash, UiSharedService.ByteToString(compressedFile.Length));
|
||||
|
||||
if (uploadToken.IsCancellationRequested) return;
|
||||
|
||||
using var ms = new MemoryStream(compressedFile);
|
||||
|
||||
Progress<UploadProgress> prog = new((prog) =>
|
||||
{
|
||||
CurrentUploads.Single(f => string.Equals(f.Hash, fileHash, StringComparison.Ordinal)).Transferred = prog.Uploaded;
|
||||
});
|
||||
var streamContent = new ProgressableStreamContent(ms, prog);
|
||||
streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
|
||||
|
||||
var response = await _orchestrator.SendRequestStreamAsync(HttpMethod.Post, MareFiles.ServerFilesUploadFullPath(_orchestrator.FilesCdnUri!, fileHash), streamContent, uploadToken).ConfigureAwait(false);
|
||||
Logger.LogDebug("Upload Status: {status}", response.StatusCode);
|
||||
}
|
||||
|
||||
private async Task UploadUnverifiedFiles(HashSet<string> unverifiedUploadHashes, List<UserData> visiblePlayers, CancellationToken uploadToken)
|
||||
{
|
||||
unverifiedUploadHashes = unverifiedUploadHashes.Where(h => _fileDbManager.GetFileCacheByHash(h) != null).ToHashSet(StringComparer.Ordinal);
|
||||
|
||||
Logger.LogDebug("Verifying {count} files", unverifiedUploadHashes.Count);
|
||||
var filesToUpload = await FilesSend(unverifiedUploadHashes.ToList(), visiblePlayers.Select(p => p.UID).ToList(), uploadToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var file in filesToUpload.Where(f => !f.IsForbidden))
|
||||
{
|
||||
try
|
||||
{
|
||||
CurrentUploads.Add(new UploadFileTransfer(file)
|
||||
{
|
||||
Total = new FileInfo(_fileDbManager.GetFileCacheByHash(file.Hash)!.ResolvedFilepath).Length,
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Tried to request file {hash} but file was not present", file.Hash);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var file in filesToUpload.Where(c => c.IsForbidden))
|
||||
{
|
||||
if (_orchestrator.ForbiddenTransfers.All(f => !string.Equals(f.Hash, file.Hash, StringComparison.Ordinal)))
|
||||
{
|
||||
_orchestrator.ForbiddenTransfers.Add(new UploadFileTransfer(file)
|
||||
{
|
||||
LocalFile = _fileDbManager.GetFileCacheByHash(file.Hash)?.ResolvedFilepath ?? string.Empty,
|
||||
});
|
||||
}
|
||||
|
||||
_verifiedUploadedHashes[file.Hash] = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
var totalSize = CurrentUploads.Sum(c => c.Total);
|
||||
Logger.LogDebug("Compressing and uploading files");
|
||||
Task uploadTask = Task.CompletedTask;
|
||||
foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList())
|
||||
{
|
||||
Logger.LogDebug("Compressing {file}", file);
|
||||
var data = await GetCompressedFileData(file.Hash, uploadToken).ConfigureAwait(false);
|
||||
CurrentUploads.Single(e => string.Equals(e.Hash, data.Item1, StringComparison.Ordinal)).Total = data.Item2.Length;
|
||||
await uploadTask.ConfigureAwait(false);
|
||||
uploadTask = UploadFile(data.Item2, file.Hash, uploadToken);
|
||||
uploadToken.ThrowIfCancellationRequested();
|
||||
_verifiedUploadedHashes[file.Hash] = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
if (CurrentUploads.Any())
|
||||
{
|
||||
await uploadTask.ConfigureAwait(false);
|
||||
|
||||
var compressedSize = CurrentUploads.Sum(c => c.Total);
|
||||
Logger.LogDebug("Upload complete, compressed {size} to {compressed}", UiSharedService.ByteToString(totalSize), UiSharedService.ByteToString(compressedSize));
|
||||
}
|
||||
|
||||
foreach (var file in unverifiedUploadHashes.Where(c => !CurrentUploads.Any(u => string.Equals(u.Hash, c, StringComparison.Ordinal))))
|
||||
{
|
||||
_verifiedUploadedHashes[file] = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
CurrentUploads.Clear();
|
||||
}
|
||||
}
|
||||
20
MareSynchronos/WebAPI/Files/Models/DownloadFileTransfer.cs
Normal file
20
MareSynchronos/WebAPI/Files/Models/DownloadFileTransfer.cs
Normal file
@@ -0,0 +1,20 @@
|
||||
using MareSynchronos.API.Dto.Files;
|
||||
|
||||
namespace MareSynchronos.WebAPI.Files.Models;
|
||||
|
||||
public class DownloadFileTransfer : FileTransfer
|
||||
{
|
||||
private DownloadFileDto Dto => (DownloadFileDto)TransferDto;
|
||||
public DownloadFileTransfer(DownloadFileDto dto) : base(dto) { }
|
||||
public Uri DownloadUri => new(Dto.Url);
|
||||
public override long Total
|
||||
{
|
||||
set
|
||||
{
|
||||
// nothing to set
|
||||
}
|
||||
get => Dto.Size;
|
||||
}
|
||||
|
||||
public override bool CanBeTransferred => Dto.FileExists && !Dto.IsForbidden && Dto.Size > 0;
|
||||
}
|
||||
10
MareSynchronos/WebAPI/Files/Models/DownloadStatus.cs
Normal file
10
MareSynchronos/WebAPI/Files/Models/DownloadStatus.cs
Normal file
@@ -0,0 +1,10 @@
|
||||
namespace MareSynchronos.WebAPI.Files.Models;
|
||||
|
||||
public enum DownloadStatus
|
||||
{
|
||||
Initializing,
|
||||
WaitingForSlot,
|
||||
WaitingForQueue,
|
||||
Downloading,
|
||||
Decompressing
|
||||
}
|
||||
10
MareSynchronos/WebAPI/Files/Models/FileDownloadStatus.cs
Normal file
10
MareSynchronos/WebAPI/Files/Models/FileDownloadStatus.cs
Normal file
@@ -0,0 +1,10 @@
|
||||
namespace MareSynchronos.WebAPI.Files.Models;
|
||||
|
||||
public class FileDownloadStatus
|
||||
{
|
||||
public DownloadStatus DownloadStatus { get; set; }
|
||||
public int TotalFiles { get; set; }
|
||||
public int TransferredFiles { get; set; }
|
||||
public long TotalBytes { get; set; }
|
||||
public long TransferredBytes { get; set; }
|
||||
}
|
||||
27
MareSynchronos/WebAPI/Files/Models/FileTransfer.cs
Normal file
27
MareSynchronos/WebAPI/Files/Models/FileTransfer.cs
Normal file
@@ -0,0 +1,27 @@
|
||||
using MareSynchronos.API.Dto.Files;
|
||||
|
||||
namespace MareSynchronos.WebAPI.Files.Models;
|
||||
|
||||
public abstract class FileTransfer
|
||||
{
|
||||
protected readonly ITransferFileDto TransferDto;
|
||||
|
||||
protected FileTransfer(ITransferFileDto transferDto)
|
||||
{
|
||||
TransferDto = transferDto;
|
||||
}
|
||||
|
||||
public string ForbiddenBy => TransferDto.ForbiddenBy;
|
||||
public long Transferred { get; set; } = 0;
|
||||
public abstract long Total { get; set; }
|
||||
public string Hash => TransferDto.Hash;
|
||||
public bool IsInTransfer => Transferred != Total && Transferred > 0;
|
||||
public bool IsTransferred => Transferred == Total;
|
||||
public virtual bool CanBeTransferred => !TransferDto.IsForbidden && (TransferDto is not DownloadFileDto dto || dto.FileExists);
|
||||
public bool IsForbidden => TransferDto.IsForbidden;
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return Hash;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
using System.Net;
|
||||
|
||||
namespace MareSynchronos.WebAPI.Files.Models;
|
||||
|
||||
public class ProgressableStreamContent : StreamContent
|
||||
{
|
||||
private const int _defaultBufferSize = 4096;
|
||||
private readonly int _bufferSize;
|
||||
private readonly IProgress<UploadProgress> _progress;
|
||||
private readonly Stream _streamToWrite;
|
||||
private bool _contentConsumed;
|
||||
|
||||
public ProgressableStreamContent(Stream streamToWrite, IProgress<UploadProgress> downloader)
|
||||
: this(streamToWrite, _defaultBufferSize, downloader)
|
||||
{
|
||||
}
|
||||
|
||||
public ProgressableStreamContent(Stream streamToWrite, int bufferSize, IProgress<UploadProgress> progress)
|
||||
: base(streamToWrite, bufferSize)
|
||||
{
|
||||
if (streamToWrite == null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(streamToWrite));
|
||||
}
|
||||
|
||||
if (bufferSize <= 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(bufferSize));
|
||||
}
|
||||
|
||||
_streamToWrite = streamToWrite;
|
||||
_bufferSize = bufferSize;
|
||||
_progress = progress;
|
||||
}
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
if (disposing)
|
||||
{
|
||||
_streamToWrite.Dispose();
|
||||
}
|
||||
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
|
||||
protected override async Task SerializeToStreamAsync(Stream stream, TransportContext? context)
|
||||
{
|
||||
PrepareContent();
|
||||
|
||||
var buffer = new byte[_bufferSize];
|
||||
var size = _streamToWrite.Length;
|
||||
var uploaded = 0;
|
||||
|
||||
using (_streamToWrite)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var length = _streamToWrite.Read(buffer, 0, buffer.Length);
|
||||
if (length <= 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
uploaded += length;
|
||||
_progress.Report(new UploadProgress(uploaded, size));
|
||||
await stream.WriteAsync(buffer.AsMemory(0, length)).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected override bool TryComputeLength(out long length)
|
||||
{
|
||||
length = _streamToWrite.Length;
|
||||
return true;
|
||||
}
|
||||
|
||||
private void PrepareContent()
|
||||
{
|
||||
if (_contentConsumed)
|
||||
{
|
||||
if (_streamToWrite.CanSeek)
|
||||
{
|
||||
_streamToWrite.Position = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new InvalidOperationException("The stream has already been read.");
|
||||
}
|
||||
}
|
||||
|
||||
_contentConsumed = true;
|
||||
}
|
||||
}
|
||||
10
MareSynchronos/WebAPI/Files/Models/UploadFileTransfer.cs
Normal file
10
MareSynchronos/WebAPI/Files/Models/UploadFileTransfer.cs
Normal file
@@ -0,0 +1,10 @@
|
||||
using MareSynchronos.API.Dto.Files;
|
||||
|
||||
namespace MareSynchronos.WebAPI.Files.Models;
|
||||
|
||||
public class UploadFileTransfer : FileTransfer
|
||||
{
|
||||
public UploadFileTransfer(UploadFileDto dto) : base(dto) { }
|
||||
public override long Total { get; set; }
|
||||
public string LocalFile { get; set; } = string.Empty;
|
||||
}
|
||||
3
MareSynchronos/WebAPI/Files/Models/UploadProgress.cs
Normal file
3
MareSynchronos/WebAPI/Files/Models/UploadProgress.cs
Normal file
@@ -0,0 +1,3 @@
|
||||
namespace MareSynchronos.WebAPI.Files.Models;
|
||||
|
||||
public record UploadProgress(long Uploaded, long Size);
|
||||
Reference in New Issue
Block a user