add api to mare, change all to file scoped namespace
This commit is contained in:
Stanley Dimant
@@ -13,306 +13,305 @@ using MareSynchronos.Utils;
|
||||
using MareSynchronos.WebAPI.Utils;
|
||||
using Microsoft.AspNetCore.SignalR.Client;
|
||||
|
||||
namespace MareSynchronos.WebAPI
|
||||
namespace MareSynchronos.WebAPI;
|
||||
|
||||
public partial class ApiController
|
||||
{
|
||||
public partial class ApiController
|
||||
private readonly HashSet<string> _verifiedUploadedHashes;
|
||||
|
||||
private int _downloadId = 0;
|
||||
public void CancelUpload()
|
||||
{
|
||||
private readonly HashSet<string> _verifiedUploadedHashes;
|
||||
|
||||
private int _downloadId = 0;
|
||||
public void CancelUpload()
|
||||
if (_uploadCancellationTokenSource != null)
|
||||
{
|
||||
if (_uploadCancellationTokenSource != null)
|
||||
{
|
||||
Logger.Debug("Cancelling upload");
|
||||
_uploadCancellationTokenSource?.Cancel();
|
||||
_mareHub!.SendAsync(Api.SendFileAbortUpload);
|
||||
CurrentUploads.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DeleteAllMyFiles()
|
||||
{
|
||||
await _mareHub!.SendAsync(Api.SendFileDeleteAllFiles);
|
||||
}
|
||||
|
||||
private async Task<string> DownloadFile(int downloadId, string hash, Uri downloadUri, CancellationToken ct)
|
||||
{
|
||||
using WebClient wc = new();
|
||||
wc.Headers.Add("Authorization", SecretKey);
|
||||
DownloadProgressChangedEventHandler progChanged = (s, e) =>
|
||||
{
|
||||
try
|
||||
{
|
||||
CurrentDownloads[downloadId].Single(f => f.Hash == hash).Transferred = e.BytesReceived;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.Warn("Could not set download progress for " + hash);
|
||||
Logger.Warn(ex.Message);
|
||||
Logger.Warn(ex.StackTrace ?? string.Empty);
|
||||
}
|
||||
};
|
||||
wc.DownloadProgressChanged += progChanged;
|
||||
|
||||
string fileName = Path.GetTempFileName();
|
||||
|
||||
ct.Register(wc.CancelAsync);
|
||||
|
||||
try
|
||||
{
|
||||
await wc.DownloadFileTaskAsync(downloadUri, fileName);
|
||||
}
|
||||
catch { }
|
||||
|
||||
CurrentDownloads[downloadId].Single(f => f.Hash == hash).Transferred = CurrentDownloads[downloadId].Single(f => f.Hash == hash).Total;
|
||||
|
||||
wc.DownloadProgressChanged -= progChanged;
|
||||
return fileName;
|
||||
}
|
||||
|
||||
public int GetDownloadId() => _downloadId++;
|
||||
|
||||
public async Task DownloadFiles(int currentDownloadId, List<FileReplacementDto> fileReplacementDto, CancellationToken ct)
|
||||
{
|
||||
DownloadStarted?.Invoke();
|
||||
try
|
||||
{
|
||||
await DownloadFilesInternal(currentDownloadId, fileReplacementDto, ct);
|
||||
}
|
||||
catch
|
||||
{
|
||||
CancelDownload(currentDownloadId);
|
||||
}
|
||||
finally
|
||||
{
|
||||
DownloadFinished?.Invoke();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DownloadFilesInternal(int currentDownloadId, List<FileReplacementDto> fileReplacementDto, CancellationToken ct)
|
||||
{
|
||||
Logger.Debug("Downloading files (Download ID " + currentDownloadId + ")");
|
||||
|
||||
List<DownloadFileDto> downloadFileInfoFromService = new List<DownloadFileDto>();
|
||||
downloadFileInfoFromService.AddRange(await _mareHub!.InvokeAsync<List<DownloadFileDto>>(Api.InvokeGetFilesSizes, fileReplacementDto.Select(f => f.Hash).ToList(), ct));
|
||||
|
||||
Logger.Debug("Files with size 0 or less: " + string.Join(", ", downloadFileInfoFromService.Where(f => f.Size <= 0).Select(f => f.Hash)));
|
||||
|
||||
CurrentDownloads[currentDownloadId] = downloadFileInfoFromService.Distinct().Select(d => new DownloadFileTransfer(d))
|
||||
.Where(d => d.CanBeTransferred).ToList();
|
||||
|
||||
foreach (var dto in downloadFileInfoFromService.Where(c => c.IsForbidden))
|
||||
{
|
||||
if (ForbiddenTransfers.All(f => f.Hash != dto.Hash))
|
||||
{
|
||||
ForbiddenTransfers.Add(new DownloadFileTransfer(dto));
|
||||
}
|
||||
}
|
||||
|
||||
await Parallel.ForEachAsync(CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred), new ParallelOptions()
|
||||
{
|
||||
MaxDegreeOfParallelism = 5,
|
||||
CancellationToken = ct
|
||||
},
|
||||
async (file, token) =>
|
||||
{
|
||||
var hash = file.Hash;
|
||||
var tempFile = await DownloadFile(currentDownloadId, file.Hash, file.DownloadUri, token);
|
||||
if (token.IsCancellationRequested)
|
||||
{
|
||||
File.Delete(tempFile);
|
||||
Logger.Debug("Detected cancellation, removing " + currentDownloadId);
|
||||
DownloadFinished?.Invoke();
|
||||
CancelDownload(currentDownloadId);
|
||||
return;
|
||||
}
|
||||
|
||||
var tempFileData = await File.ReadAllBytesAsync(tempFile, token);
|
||||
var extractedFile = LZ4Codec.Unwrap(tempFileData);
|
||||
File.Delete(tempFile);
|
||||
var filePath = Path.Combine(_pluginConfiguration.CacheFolder, file.Hash);
|
||||
await File.WriteAllBytesAsync(filePath, extractedFile, token);
|
||||
var fi = new FileInfo(filePath);
|
||||
Func<DateTime> RandomDayFunc()
|
||||
{
|
||||
DateTime start = new DateTime(1995, 1, 1);
|
||||
Random gen = new Random();
|
||||
int range = (DateTime.Today - start).Days;
|
||||
return () => start.AddDays(gen.Next(range));
|
||||
}
|
||||
|
||||
fi.CreationTime = RandomDayFunc().Invoke();
|
||||
fi.LastAccessTime = RandomDayFunc().Invoke();
|
||||
fi.LastWriteTime = RandomDayFunc().Invoke();
|
||||
try
|
||||
{
|
||||
_ = _fileDbManager.CreateCacheEntry(filePath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.Warn("Issue adding file to the DB");
|
||||
Logger.Warn(ex.Message);
|
||||
Logger.Warn(ex.StackTrace);
|
||||
}
|
||||
});
|
||||
|
||||
Logger.Debug("Download complete, removing " + currentDownloadId);
|
||||
CancelDownload(currentDownloadId);
|
||||
}
|
||||
|
||||
public async Task PushCharacterData(CharacterCacheDto character, List<string> visibleCharacterIds)
|
||||
{
|
||||
if (!IsConnected || SecretKey == "-") return;
|
||||
Logger.Debug("Sending Character data to service " + ApiUri);
|
||||
|
||||
CancelUpload();
|
||||
_uploadCancellationTokenSource = new CancellationTokenSource();
|
||||
var uploadToken = _uploadCancellationTokenSource.Token;
|
||||
Logger.Verbose("New Token Created");
|
||||
|
||||
List<string> unverifiedUploadHashes = new();
|
||||
foreach (var item in character.FileReplacements.SelectMany(c => c.Value.Where(f => string.IsNullOrEmpty(f.FileSwapPath)).Select(v => v.Hash).Distinct()).Distinct().ToList())
|
||||
{
|
||||
if (!_verifiedUploadedHashes.Contains(item))
|
||||
{
|
||||
unverifiedUploadHashes.Add(item);
|
||||
}
|
||||
}
|
||||
|
||||
if (unverifiedUploadHashes.Any())
|
||||
{
|
||||
Logger.Debug("Verifying " + unverifiedUploadHashes.Count + " files");
|
||||
var filesToUpload = await _mareHub!.InvokeAsync<List<UploadFileDto>>(Api.InvokeFileSendFiles, unverifiedUploadHashes, uploadToken);
|
||||
|
||||
foreach (var file in filesToUpload.Where(f => !f.IsForbidden))
|
||||
{
|
||||
try
|
||||
{
|
||||
CurrentUploads.Add(new UploadFileTransfer(file)
|
||||
{
|
||||
Total = new FileInfo(_fileDbManager.GetFileCacheByHash(file.Hash)!.ResolvedFilepath).Length
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.Warn("Tried to request file " + file.Hash + " but file was not present");
|
||||
Logger.Warn(ex.StackTrace!);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var file in filesToUpload.Where(c => c.IsForbidden))
|
||||
{
|
||||
if (ForbiddenTransfers.All(f => f.Hash != file.Hash))
|
||||
{
|
||||
ForbiddenTransfers.Add(new UploadFileTransfer(file)
|
||||
{
|
||||
LocalFile = _fileDbManager.GetFileCacheByHash(file.Hash)?.ResolvedFilepath ?? string.Empty
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var totalSize = CurrentUploads.Sum(c => c.Total);
|
||||
Logger.Debug("Compressing and uploading files");
|
||||
foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList())
|
||||
{
|
||||
Logger.Debug("Compressing and uploading " + file);
|
||||
var data = await GetCompressedFileData(file.Hash, uploadToken);
|
||||
CurrentUploads.Single(e => e.Hash == data.Item1).Total = data.Item2.Length;
|
||||
await UploadFile(data.Item2, file.Hash, uploadToken);
|
||||
if (!uploadToken.IsCancellationRequested) continue;
|
||||
Logger.Warn("Cancel in filesToUpload loop detected");
|
||||
CurrentUploads.Clear();
|
||||
break;
|
||||
}
|
||||
|
||||
if (CurrentUploads.Any())
|
||||
{
|
||||
var compressedSize = CurrentUploads.Sum(c => c.Total);
|
||||
Logger.Debug($"Compressed {totalSize} to {compressedSize} ({(compressedSize / (double)totalSize):P2})");
|
||||
}
|
||||
|
||||
Logger.Debug("Upload tasks complete, waiting for server to confirm");
|
||||
var anyUploadsOpen = await _mareHub!.InvokeAsync<bool>(Api.InvokeFileIsUploadFinished, uploadToken);
|
||||
Logger.Debug("Uploads open: " + anyUploadsOpen);
|
||||
while (anyUploadsOpen && !uploadToken.IsCancellationRequested)
|
||||
{
|
||||
anyUploadsOpen = await _mareHub!.InvokeAsync<bool>(Api.InvokeFileIsUploadFinished, uploadToken);
|
||||
await Task.Delay(TimeSpan.FromSeconds(0.5), uploadToken);
|
||||
Logger.Debug("Waiting for uploads to finish");
|
||||
}
|
||||
|
||||
foreach (var item in unverifiedUploadHashes)
|
||||
{
|
||||
_verifiedUploadedHashes.Add(item);
|
||||
}
|
||||
|
||||
CurrentUploads.Clear();
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.Debug("All files already verified");
|
||||
}
|
||||
|
||||
if (!uploadToken.IsCancellationRequested)
|
||||
{
|
||||
Logger.Info("Pushing character data for " + character.GetHashCode() + " to " + string.Join(", ", visibleCharacterIds));
|
||||
StringBuilder sb = new StringBuilder();
|
||||
foreach (var item in character.FileReplacements)
|
||||
{
|
||||
sb.AppendLine($"FileReplacements for {item.Key}: {item.Value.Count}");
|
||||
}
|
||||
foreach (var item in character.GlamourerData)
|
||||
{
|
||||
sb.AppendLine($"GlamourerData for {item.Key}: {!string.IsNullOrEmpty(item.Value)}");
|
||||
}
|
||||
Logger.Debug("Chara data contained: " + Environment.NewLine + sb.ToString());
|
||||
await _mareHub!.InvokeAsync(Api.InvokeUserPushCharacterDataToVisibleClients, character, visibleCharacterIds, uploadToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.Warn("=== Upload operation was cancelled ===");
|
||||
}
|
||||
|
||||
Logger.Verbose("Upload complete for " + character.GetHashCode());
|
||||
_uploadCancellationTokenSource = null;
|
||||
}
|
||||
|
||||
private async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken)
|
||||
{
|
||||
var fileCache = _fileDbManager.GetFileCacheByHash(fileHash)!.ResolvedFilepath;
|
||||
return (fileHash, LZ4Codec.WrapHC(await File.ReadAllBytesAsync(fileCache, uploadToken), 0,
|
||||
(int)new FileInfo(fileCache).Length));
|
||||
}
|
||||
|
||||
private async Task UploadFile(byte[] compressedFile, string fileHash, CancellationToken uploadToken)
|
||||
{
|
||||
if (uploadToken.IsCancellationRequested) return;
|
||||
|
||||
async IAsyncEnumerable<byte[]> AsyncFileData([EnumeratorCancellation] CancellationToken token)
|
||||
{
|
||||
var chunkSize = 1024 * 512; // 512kb
|
||||
using var ms = new MemoryStream(compressedFile);
|
||||
var buffer = new byte[chunkSize];
|
||||
int bytesRead;
|
||||
while ((bytesRead = await ms.ReadAsync(buffer, 0, chunkSize, token)) > 0 && !token.IsCancellationRequested)
|
||||
{
|
||||
CurrentUploads.Single(f => f.Hash == fileHash).Transferred += bytesRead;
|
||||
token.ThrowIfCancellationRequested();
|
||||
yield return bytesRead == chunkSize ? buffer.ToArray() : buffer.Take(bytesRead).ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
await _mareHub!.SendAsync(Api.SendFileUploadFileStreamAsync, fileHash, AsyncFileData(uploadToken), uploadToken);
|
||||
}
|
||||
|
||||
public void CancelDownload(int downloadId)
|
||||
{
|
||||
while (CurrentDownloads.ContainsKey(downloadId))
|
||||
{
|
||||
CurrentDownloads.TryRemove(downloadId, out _);
|
||||
}
|
||||
Logger.Debug("Cancelling upload");
|
||||
_uploadCancellationTokenSource?.Cancel();
|
||||
_mareHub!.SendAsync(Api.SendFileAbortUpload);
|
||||
CurrentUploads.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DeleteAllMyFiles()
|
||||
{
|
||||
await _mareHub!.SendAsync(Api.SendFileDeleteAllFiles);
|
||||
}
|
||||
|
||||
private async Task<string> DownloadFile(int downloadId, string hash, Uri downloadUri, CancellationToken ct)
|
||||
{
|
||||
using WebClient wc = new();
|
||||
wc.Headers.Add("Authorization", SecretKey);
|
||||
DownloadProgressChangedEventHandler progChanged = (s, e) =>
|
||||
{
|
||||
try
|
||||
{
|
||||
CurrentDownloads[downloadId].Single(f => f.Hash == hash).Transferred = e.BytesReceived;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.Warn("Could not set download progress for " + hash);
|
||||
Logger.Warn(ex.Message);
|
||||
Logger.Warn(ex.StackTrace ?? string.Empty);
|
||||
}
|
||||
};
|
||||
wc.DownloadProgressChanged += progChanged;
|
||||
|
||||
string fileName = Path.GetTempFileName();
|
||||
|
||||
ct.Register(wc.CancelAsync);
|
||||
|
||||
try
|
||||
{
|
||||
await wc.DownloadFileTaskAsync(downloadUri, fileName);
|
||||
}
|
||||
catch { }
|
||||
|
||||
CurrentDownloads[downloadId].Single(f => f.Hash == hash).Transferred = CurrentDownloads[downloadId].Single(f => f.Hash == hash).Total;
|
||||
|
||||
wc.DownloadProgressChanged -= progChanged;
|
||||
return fileName;
|
||||
}
|
||||
|
||||
public int GetDownloadId() => _downloadId++;
|
||||
|
||||
public async Task DownloadFiles(int currentDownloadId, List<FileReplacementDto> fileReplacementDto, CancellationToken ct)
|
||||
{
|
||||
DownloadStarted?.Invoke();
|
||||
try
|
||||
{
|
||||
await DownloadFilesInternal(currentDownloadId, fileReplacementDto, ct);
|
||||
}
|
||||
catch
|
||||
{
|
||||
CancelDownload(currentDownloadId);
|
||||
}
|
||||
finally
|
||||
{
|
||||
DownloadFinished?.Invoke();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DownloadFilesInternal(int currentDownloadId, List<FileReplacementDto> fileReplacementDto, CancellationToken ct)
|
||||
{
|
||||
Logger.Debug("Downloading files (Download ID " + currentDownloadId + ")");
|
||||
|
||||
List<DownloadFileDto> downloadFileInfoFromService = new List<DownloadFileDto>();
|
||||
downloadFileInfoFromService.AddRange(await _mareHub!.InvokeAsync<List<DownloadFileDto>>(Api.InvokeGetFilesSizes, fileReplacementDto.Select(f => f.Hash).ToList(), ct));
|
||||
|
||||
Logger.Debug("Files with size 0 or less: " + string.Join(", ", downloadFileInfoFromService.Where(f => f.Size <= 0).Select(f => f.Hash)));
|
||||
|
||||
CurrentDownloads[currentDownloadId] = downloadFileInfoFromService.Distinct().Select(d => new DownloadFileTransfer(d))
|
||||
.Where(d => d.CanBeTransferred).ToList();
|
||||
|
||||
foreach (var dto in downloadFileInfoFromService.Where(c => c.IsForbidden))
|
||||
{
|
||||
if (ForbiddenTransfers.All(f => f.Hash != dto.Hash))
|
||||
{
|
||||
ForbiddenTransfers.Add(new DownloadFileTransfer(dto));
|
||||
}
|
||||
}
|
||||
|
||||
await Parallel.ForEachAsync(CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred), new ParallelOptions()
|
||||
{
|
||||
MaxDegreeOfParallelism = 5,
|
||||
CancellationToken = ct
|
||||
},
|
||||
async (file, token) =>
|
||||
{
|
||||
var hash = file.Hash;
|
||||
var tempFile = await DownloadFile(currentDownloadId, file.Hash, file.DownloadUri, token);
|
||||
if (token.IsCancellationRequested)
|
||||
{
|
||||
File.Delete(tempFile);
|
||||
Logger.Debug("Detected cancellation, removing " + currentDownloadId);
|
||||
DownloadFinished?.Invoke();
|
||||
CancelDownload(currentDownloadId);
|
||||
return;
|
||||
}
|
||||
|
||||
var tempFileData = await File.ReadAllBytesAsync(tempFile, token);
|
||||
var extractedFile = LZ4Codec.Unwrap(tempFileData);
|
||||
File.Delete(tempFile);
|
||||
var filePath = Path.Combine(_pluginConfiguration.CacheFolder, file.Hash);
|
||||
await File.WriteAllBytesAsync(filePath, extractedFile, token);
|
||||
var fi = new FileInfo(filePath);
|
||||
Func<DateTime> RandomDayFunc()
|
||||
{
|
||||
DateTime start = new DateTime(1995, 1, 1);
|
||||
Random gen = new Random();
|
||||
int range = (DateTime.Today - start).Days;
|
||||
return () => start.AddDays(gen.Next(range));
|
||||
}
|
||||
|
||||
fi.CreationTime = RandomDayFunc().Invoke();
|
||||
fi.LastAccessTime = RandomDayFunc().Invoke();
|
||||
fi.LastWriteTime = RandomDayFunc().Invoke();
|
||||
try
|
||||
{
|
||||
_ = _fileDbManager.CreateCacheEntry(filePath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.Warn("Issue adding file to the DB");
|
||||
Logger.Warn(ex.Message);
|
||||
Logger.Warn(ex.StackTrace);
|
||||
}
|
||||
});
|
||||
|
||||
Logger.Debug("Download complete, removing " + currentDownloadId);
|
||||
CancelDownload(currentDownloadId);
|
||||
}
|
||||
|
||||
public async Task PushCharacterData(CharacterCacheDto character, List<string> visibleCharacterIds)
|
||||
{
|
||||
if (!IsConnected || SecretKey == "-") return;
|
||||
Logger.Debug("Sending Character data to service " + ApiUri);
|
||||
|
||||
CancelUpload();
|
||||
_uploadCancellationTokenSource = new CancellationTokenSource();
|
||||
var uploadToken = _uploadCancellationTokenSource.Token;
|
||||
Logger.Verbose("New Token Created");
|
||||
|
||||
List<string> unverifiedUploadHashes = new();
|
||||
foreach (var item in character.FileReplacements.SelectMany(c => c.Value.Where(f => string.IsNullOrEmpty(f.FileSwapPath)).Select(v => v.Hash).Distinct()).Distinct().ToList())
|
||||
{
|
||||
if (!_verifiedUploadedHashes.Contains(item))
|
||||
{
|
||||
unverifiedUploadHashes.Add(item);
|
||||
}
|
||||
}
|
||||
|
||||
if (unverifiedUploadHashes.Any())
|
||||
{
|
||||
Logger.Debug("Verifying " + unverifiedUploadHashes.Count + " files");
|
||||
var filesToUpload = await _mareHub!.InvokeAsync<List<UploadFileDto>>(Api.InvokeFileSendFiles, unverifiedUploadHashes, uploadToken);
|
||||
|
||||
foreach (var file in filesToUpload.Where(f => !f.IsForbidden))
|
||||
{
|
||||
try
|
||||
{
|
||||
CurrentUploads.Add(new UploadFileTransfer(file)
|
||||
{
|
||||
Total = new FileInfo(_fileDbManager.GetFileCacheByHash(file.Hash)!.ResolvedFilepath).Length
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.Warn("Tried to request file " + file.Hash + " but file was not present");
|
||||
Logger.Warn(ex.StackTrace!);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var file in filesToUpload.Where(c => c.IsForbidden))
|
||||
{
|
||||
if (ForbiddenTransfers.All(f => f.Hash != file.Hash))
|
||||
{
|
||||
ForbiddenTransfers.Add(new UploadFileTransfer(file)
|
||||
{
|
||||
LocalFile = _fileDbManager.GetFileCacheByHash(file.Hash)?.ResolvedFilepath ?? string.Empty
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var totalSize = CurrentUploads.Sum(c => c.Total);
|
||||
Logger.Debug("Compressing and uploading files");
|
||||
foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList())
|
||||
{
|
||||
Logger.Debug("Compressing and uploading " + file);
|
||||
var data = await GetCompressedFileData(file.Hash, uploadToken);
|
||||
CurrentUploads.Single(e => e.Hash == data.Item1).Total = data.Item2.Length;
|
||||
await UploadFile(data.Item2, file.Hash, uploadToken);
|
||||
if (!uploadToken.IsCancellationRequested) continue;
|
||||
Logger.Warn("Cancel in filesToUpload loop detected");
|
||||
CurrentUploads.Clear();
|
||||
break;
|
||||
}
|
||||
|
||||
if (CurrentUploads.Any())
|
||||
{
|
||||
var compressedSize = CurrentUploads.Sum(c => c.Total);
|
||||
Logger.Debug($"Compressed {totalSize} to {compressedSize} ({(compressedSize / (double)totalSize):P2})");
|
||||
}
|
||||
|
||||
Logger.Debug("Upload tasks complete, waiting for server to confirm");
|
||||
var anyUploadsOpen = await _mareHub!.InvokeAsync<bool>(Api.InvokeFileIsUploadFinished, uploadToken);
|
||||
Logger.Debug("Uploads open: " + anyUploadsOpen);
|
||||
while (anyUploadsOpen && !uploadToken.IsCancellationRequested)
|
||||
{
|
||||
anyUploadsOpen = await _mareHub!.InvokeAsync<bool>(Api.InvokeFileIsUploadFinished, uploadToken);
|
||||
await Task.Delay(TimeSpan.FromSeconds(0.5), uploadToken);
|
||||
Logger.Debug("Waiting for uploads to finish");
|
||||
}
|
||||
|
||||
foreach (var item in unverifiedUploadHashes)
|
||||
{
|
||||
_verifiedUploadedHashes.Add(item);
|
||||
}
|
||||
|
||||
CurrentUploads.Clear();
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.Debug("All files already verified");
|
||||
}
|
||||
|
||||
if (!uploadToken.IsCancellationRequested)
|
||||
{
|
||||
Logger.Info("Pushing character data for " + character.GetHashCode() + " to " + string.Join(", ", visibleCharacterIds));
|
||||
StringBuilder sb = new StringBuilder();
|
||||
foreach (var item in character.FileReplacements)
|
||||
{
|
||||
sb.AppendLine($"FileReplacements for {item.Key}: {item.Value.Count}");
|
||||
}
|
||||
foreach (var item in character.GlamourerData)
|
||||
{
|
||||
sb.AppendLine($"GlamourerData for {item.Key}: {!string.IsNullOrEmpty(item.Value)}");
|
||||
}
|
||||
Logger.Debug("Chara data contained: " + Environment.NewLine + sb.ToString());
|
||||
await _mareHub!.InvokeAsync(Api.InvokeUserPushCharacterDataToVisibleClients, character, visibleCharacterIds, uploadToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.Warn("=== Upload operation was cancelled ===");
|
||||
}
|
||||
|
||||
Logger.Verbose("Upload complete for " + character.GetHashCode());
|
||||
_uploadCancellationTokenSource = null;
|
||||
}
|
||||
|
||||
private async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken)
|
||||
{
|
||||
var fileCache = _fileDbManager.GetFileCacheByHash(fileHash)!.ResolvedFilepath;
|
||||
return (fileHash, LZ4Codec.WrapHC(await File.ReadAllBytesAsync(fileCache, uploadToken), 0,
|
||||
(int)new FileInfo(fileCache).Length));
|
||||
}
|
||||
|
||||
private async Task UploadFile(byte[] compressedFile, string fileHash, CancellationToken uploadToken)
|
||||
{
|
||||
if (uploadToken.IsCancellationRequested) return;
|
||||
|
||||
async IAsyncEnumerable<byte[]> AsyncFileData([EnumeratorCancellation] CancellationToken token)
|
||||
{
|
||||
var chunkSize = 1024 * 512; // 512kb
|
||||
using var ms = new MemoryStream(compressedFile);
|
||||
var buffer = new byte[chunkSize];
|
||||
int bytesRead;
|
||||
while ((bytesRead = await ms.ReadAsync(buffer, 0, chunkSize, token)) > 0 && !token.IsCancellationRequested)
|
||||
{
|
||||
CurrentUploads.Single(f => f.Hash == fileHash).Transferred += bytesRead;
|
||||
token.ThrowIfCancellationRequested();
|
||||
yield return bytesRead == chunkSize ? buffer.ToArray() : buffer.Take(bytesRead).ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
await _mareHub!.SendAsync(Api.SendFileUploadFileStreamAsync, fileHash, AsyncFileData(uploadToken), uploadToken);
|
||||
}
|
||||
|
||||
public void CancelDownload(int downloadId)
|
||||
{
|
||||
while (CurrentDownloads.ContainsKey(downloadId))
|
||||
{
|
||||
CurrentDownloads.TryRemove(downloadId, out _);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user