t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

do not use sendfiles when not necessary

Browse Source
This commit is contained in:
Stanley Dimant
2022-08-13 14:50:10 +02:00
parent 310dd4db5e
commit 110dbd7bdf
3 changed files with 75 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
MareSynchronos/MareSynchronos.csproj
View File

@@ -3,7 +3,7 @@
<PropertyGroup>
<Authors></Authors>
<Company></Company>
<Version>0.3.2</Version>
<Version>0.3.3</Version>
<Description></Description>
<Copyright></Copyright>
<PackageProjectUrl>https://github.com/Penumbra-Sync/client</PackageProjectUrl>

120
MareSynchronos/WebAPI/ApIController.Functions.Files.cs
View File

@@ -19,6 +19,8 @@ namespace MareSynchronos.WebAPI
{
public partial class ApiController
{
private readonly HashSet<string> _verifiedUploadedHashes;
private int _downloadId = 0;
public void CancelUpload()
{
@@ -149,72 +151,94 @@ namespace MareSynchronos.WebAPI
var uploadToken = _uploadCancellationTokenSource.Token;
Logger.Verbose("New Token Created");
var filesToUpload = await _mareHub!.InvokeAsync<List<UploadFileDto>>(Api.InvokeFileSendFiles, character.FileReplacements.SelectMany(c => c.Value.Select(v => v.Hash)).Distinct(), uploadToken);
foreach (var file in filesToUpload.Where(f => !f.IsForbidden))
List<string> unverifiedUploadHashes = new();
foreach (var item in character.FileReplacements.SelectMany(c => c.Value.Select(v => v.Hash).Distinct()).Distinct().ToList())
{
await using var db = new FileCacheContext();
try
if (!_verifiedUploadedHashes.Contains(item))
{
CurrentUploads.Add(new UploadFileTransfer(file)
{
Total = new FileInfo(db.FileCaches.FirstOrDefault(f => f.Hash.ToLower() == file.Hash.ToLower())
?.Filepath ?? string.Empty).Length
});
}
catch (Exception ex)
{
Logger.Warn("Tried to request file " + file.Hash + " but file was not present");
Logger.Warn(ex.StackTrace!);
unverifiedUploadHashes.Add(item);
}
}
await using (var db = new FileCacheContext())
if (unverifiedUploadHashes.Any())
{
foreach (var file in filesToUpload.Where(c => c.IsForbidden))
Logger.Debug("Verifying " + unverifiedUploadHashes.Count + " files");
var filesToUpload = await _mareHub!.InvokeAsync<List<UploadFileDto>>(Api.InvokeFileSendFiles, unverifiedUploadHashes, uploadToken);
foreach (var file in filesToUpload.Where(f => !f.IsForbidden))
{
if (ForbiddenTransfers.All(f => f.Hash != file.Hash))
await using var db = new FileCacheContext();
try
{
ForbiddenTransfers.Add(new UploadFileTransfer(file)
CurrentUploads.Add(new UploadFileTransfer(file)
{
LocalFile = db.FileCaches.FirstOrDefault(f => f.Hash == file.Hash)?.Filepath ?? string.Empty
Total = new FileInfo(db.FileCaches.FirstOrDefault(f => f.Hash.ToLower() == file.Hash.ToLower())
?.Filepath ?? string.Empty).Length
});
}
catch (Exception ex)
{
Logger.Warn("Tried to request file " + file.Hash + " but file was not present");
Logger.Warn(ex.StackTrace!);
}
}
await using (var db = new FileCacheContext())
{
foreach (var file in filesToUpload.Where(c => c.IsForbidden))
{
if (ForbiddenTransfers.All(f => f.Hash != file.Hash))
{
ForbiddenTransfers.Add(new UploadFileTransfer(file)
{
LocalFile = db.FileCaches.FirstOrDefault(f => f.Hash == file.Hash)?.Filepath ?? string.Empty
});
}
}
}
var totalSize = CurrentUploads.Sum(c => c.Total);
Logger.Debug("Compressing and uploading files");
foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList())
{
Logger.Debug("Compressing and uploading " + file);
var data = await GetCompressedFileData(file.Hash, uploadToken);
CurrentUploads.Single(e => e.Hash == data.Item1).Total = data.Item2.Length;
await UploadFile(data.Item2, file.Hash, uploadToken);
if (!uploadToken.IsCancellationRequested) continue;
Logger.Warn("Cancel in filesToUpload loop detected");
CurrentUploads.Clear();
break;
}
if (CurrentUploads.Any())
{
var compressedSize = CurrentUploads.Sum(c => c.Total);
Logger.Debug($"Compressed {totalSize} to {compressedSize} ({(compressedSize / (double)totalSize):P2})");
}
Logger.Debug("Upload tasks complete, waiting for server to confirm");
var anyUploadsOpen = await _mareHub!.InvokeAsync<bool>(Api.InvokeFileIsUploadFinished, uploadToken);
Logger.Debug("Uploads open: " + anyUploadsOpen);
while (anyUploadsOpen && !uploadToken.IsCancellationRequested)
{
anyUploadsOpen = await _mareHub!.InvokeAsync<bool>(Api.InvokeFileIsUploadFinished, uploadToken);
await Task.Delay(TimeSpan.FromSeconds(0.5), uploadToken);
Logger.Debug("Waiting for uploads to finish");
}
foreach (var item in unverifiedUploadHashes)
{
_verifiedUploadedHashes.Add(item);
}
}
var totalSize = CurrentUploads.Sum(c => c.Total);
Logger.Debug("Compressing and uploading files");
foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList())
{
Logger.Debug("Compressing and uploading " + file);
var data = await GetCompressedFileData(file.Hash, uploadToken);
CurrentUploads.Single(e => e.Hash == data.Item1).Total = data.Item2.Length;
await UploadFile(data.Item2, file.Hash, uploadToken);
if (!uploadToken.IsCancellationRequested) continue;
Logger.Warn("Cancel in filesToUpload loop detected");
CurrentUploads.Clear();
break;
}
if (CurrentUploads.Any())
else
{
var compressedSize = CurrentUploads.Sum(c => c.Total);
Logger.Debug($"Compressed {totalSize} to {compressedSize} ({(compressedSize / (double)totalSize):P2})");
Logger.Debug("All files already verified");
}
Logger.Debug("Upload tasks complete, waiting for server to confirm");
var anyUploadsOpen = await _mareHub!.InvokeAsync<bool>(Api.InvokeFileIsUploadFinished, uploadToken);
Logger.Debug("Uploads open: " + anyUploadsOpen);
while (anyUploadsOpen && !uploadToken.IsCancellationRequested)
{
anyUploadsOpen = await _mareHub!.InvokeAsync<bool>(Api.InvokeFileIsUploadFinished, uploadToken);
await Task.Delay(TimeSpan.FromSeconds(0.5), uploadToken);
Logger.Debug("Waiting for uploads to finish");
}
CurrentUploads.Clear();
if (!uploadToken.IsCancellationRequested)
{
Logger.Info("Pushing character data for " + character.GetHashCode() + " to " + string.Join(", ", visibleCharacterIds));

2
MareSynchronos/WebAPI/ApiController.Connectivity.cs
View File

@@ -57,6 +57,7 @@ namespace MareSynchronos.WebAPI
_dalamudUtil.LogIn += DalamudUtilOnLogIn;
_dalamudUtil.LogOut += DalamudUtilOnLogOut;
ServerState = ServerState.Offline;
_verifiedUploadedHashes = new();
if (_dalamudUtil.IsLoggedIn)
{
@@ -141,6 +142,7 @@ namespace MareSynchronos.WebAPI
_connectionCancellationTokenSource.Cancel();
_connectionCancellationTokenSource = new CancellationTokenSource();
var token = _connectionCancellationTokenSource.Token;
_verifiedUploadedHashes.Clear();
while (ServerState is not ServerState.Connected && !token.IsCancellationRequested)
{
if (string.IsNullOrEmpty(SecretKey))