t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Munge block files on disk (#57)

Browse Source 
* Munge block files on disk

* Simplify
This commit is contained in:
N. Lo
2023-08-06 02:36:32 +02:00
committed by GitHub
parent 725e98fb47
commit dcf072e00b
2 changed files with 29 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
MareSynchronos/FileCache/FileCacheManager.cs
View File

@@ -120,9 +120,9 @@ public sealed class FileCacheManager : IDisposable
public List<FileCacheEntity> GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v).ToList();
public string GetCacheFilePath(string hash, string extension, bool isTemporaryFile)
public string GetCacheFilePath(string hash, string extension)
{
return Path.Combine(_configService.Current.CacheFolder, hash + "." + extension + (isTemporaryFile ? ".tmp" : string.Empty));
return Path.Combine(_configService.Current.CacheFolder, hash + "." + extension);
}
public FileCacheEntity? GetFileCacheByHash(string hash)

33
MareSynchronos/WebAPI/Files/FileDownloadManager.cs
View File

@@ -77,17 +77,35 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
base.Dispose(disposing);
}
private static byte MungeByte(int byteOrEof)
{
if (byteOrEof == -1)
{
throw new EndOfStreamException();
}
return (byte)(byteOrEof ^ 42);
}
private static void MungeBuffer(Span<byte> buffer)
{
for (int i = 0; i < buffer.Length; ++i)
{
buffer[i] ^= 42;
}
}
private static (string fileHash, long fileLengthBytes) ReadBlockFileHeader(FileStream fileBlockStream)
{
List<char> hashName = new();
List<char> fileLength = new();
var separator = (char)fileBlockStream.ReadByte();
var separator = (char)MungeByte(fileBlockStream.ReadByte());
if (separator != '#') throw new InvalidDataException("Data is invalid, first char is not #");
bool readHash = false;
while (true)
{
var readChar = (char)fileBlockStream.ReadByte();
var readChar = (char)MungeByte(fileBlockStream.ReadByte());
if (readChar == ':')
{
readHash = true;
@@ -100,7 +118,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
return (string.Join("", hashName), long.Parse(string.Join("", fileLength)));
}
private async Task DownloadFileHttpClient(string downloadGroup, Guid requestId, List<DownloadFileTransfer> fileTransfer, string tempPath, IProgress<long> progress, CancellationToken ct)
private async Task DownloadAndMungeFileHttpClient(string downloadGroup, Guid requestId, List<DownloadFileTransfer> fileTransfer, string tempPath, IProgress<long> progress, CancellationToken ct)
{
Logger.LogDebug("GUID {requestId} on server {uri} for files {files}", requestId, fileTransfer[0].DownloadUri, string.Join(", ", fileTransfer.Select(c => c.Hash).ToList()));
@@ -139,6 +157,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
{
ct.ThrowIfCancellationRequested();
MungeBuffer(buffer.AsSpan(0, bytesRead));
await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead), ct).ConfigureAwait(false);
progress.Report(bytesRead);
@@ -216,7 +236,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
Logger.LogDebug("GUID {requestId} for {n} files on server {uri}", requestId, fileGroup.Count(), fileGroup.First().DownloadUri);
var blockFile = _fileDbManager.GetCacheFilePath(requestId.ToString("N"), "blk", true);
var blockFile = _fileDbManager.GetCacheFilePath(requestId.ToString("N"), "blk");
try
{
_downloadStatus[fileGroup.Key].DownloadStatus = DownloadStatus.WaitingForSlot;
@@ -234,7 +254,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
Logger.LogWarning(ex, "Could not set download progress");
}
});
await DownloadFileHttpClient(fileGroup.Key, requestId, fileGroup.ToList(), blockFile, progress, token).ConfigureAwait(false);
await DownloadAndMungeFileHttpClient(fileGroup.Key, requestId, fileGroup.ToList(), blockFile, progress, token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
@@ -270,9 +290,10 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
byte[] compressedFileContent = new byte[fileLengthBytes];
_ = await fileBlockStream.ReadAsync(compressedFileContent, token).ConfigureAwait(false);
MungeBuffer(compressedFileContent);
var decompressedFile = LZ4Codec.Unwrap(compressedFileContent);
var filePath = _fileDbManager.GetCacheFilePath(fileHash, fileExtension, false);
var filePath = _fileDbManager.GetCacheFilePath(fileHash, fileExtension);
await File.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false);
PersistFileToStorage(fileHash, filePath);