From add2d2eda9dcd5b17aef0052746e33b8a8fe1110 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Sun, 17 Dec 2023 18:53:22 +0100 Subject: [PATCH] potential mcdf fixes --- MareSynchronos/FileCache/FileCacheManager.cs | 2 +- .../PlayerData/Export/MareCharaFileData.cs | 17 +++++--- .../PlayerData/Export/MareCharaFileManager.cs | 41 +++++++++++-------- MareSynchronos/Utils/VariousExtensions.cs | 26 ++++++++++++ 4 files changed, 62 insertions(+), 24 deletions(-) diff --git a/MareSynchronos/FileCache/FileCacheManager.cs b/MareSynchronos/FileCache/FileCacheManager.cs index 7648b0e..306493b 100644 --- a/MareSynchronos/FileCache/FileCacheManager.cs +++ b/MareSynchronos/FileCache/FileCacheManager.cs @@ -247,7 +247,7 @@ public sealed class FileCacheManager : IDisposable { if (_fileCaches.TryGetValue(hash, out var hashes)) { - var item = hashes.FirstOrDefault(); + var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix) ? 0 : 1).FirstOrDefault(); if (item != null) return GetValidatedFileCache(item); } return null; diff --git a/MareSynchronos/PlayerData/Export/MareCharaFileData.cs b/MareSynchronos/PlayerData/Export/MareCharaFileData.cs index 0bb8ea4..f0e7eaa 100644 --- a/MareSynchronos/PlayerData/Export/MareCharaFileData.cs +++ b/MareSynchronos/PlayerData/Export/MareCharaFileData.cs @@ -33,18 +33,23 @@ public record MareCharaFileData if (dto.FileReplacements.TryGetValue(ObjectKind.Player, out var fileReplacements)) { - foreach (var file in fileReplacements) + var grouped = fileReplacements.GroupBy(f => f.Hash, StringComparer.OrdinalIgnoreCase); + + foreach (var file in grouped) { - if (!string.IsNullOrEmpty(file.FileSwapPath)) + if (string.IsNullOrEmpty(file.Key)) { - FileSwaps.Add(new FileSwap(file.GamePaths, file.FileSwapPath)); + foreach (var item in file) + { + FileSwaps.Add(new FileSwap(item.GamePaths, item.FileSwapPath)); + } } else { - var filePath = manager.GetFileCacheByHash(file.Hash)?.ResolvedFilepath; + var filePath = manager.GetFileCacheByHash(file.First().Hash)?.ResolvedFilepath; if (filePath != null) { - Files.Add(new FileData(file.GamePaths, new FileInfo(filePath).Length)); + Files.Add(new FileData(file.SelectMany(f => f.GamePaths), (int)new FileInfo(filePath).Length, file.First().Hash)); } } } @@ -63,5 +68,5 @@ public record MareCharaFileData public record FileSwap(IEnumerable GamePaths, string FileSwapPath); - public record FileData(IEnumerable GamePaths, long Length); + public record FileData(IEnumerable GamePaths, int Length, string Hash); } \ No newline at end of file diff --git a/MareSynchronos/PlayerData/Export/MareCharaFileManager.cs b/MareSynchronos/PlayerData/Export/MareCharaFileManager.cs index 7b05a40..c6a63c2 100644 --- a/MareSynchronos/PlayerData/Export/MareCharaFileManager.cs +++ b/MareSynchronos/PlayerData/Export/MareCharaFileManager.cs @@ -10,6 +10,7 @@ using MareSynchronos.Services; using MareSynchronos.Services.Mediator; using MareSynchronos.Utils; using Microsoft.Extensions.Logging; +using System.Text.Json; using CharacterData = MareSynchronos.API.Data.CharacterData; namespace MareSynchronos.PlayerData.Export; @@ -49,6 +50,7 @@ public class MareCharaFileManager : DisposableMediatorSubscriberBase if ((await dalamudUtil.RunOnFrameworkThread(() => item.Value.CurrentAddress()).ConfigureAwait(false)) != nint.Zero) { await _ipcManager.GlamourerRevert(logger, item.Value.Name, item.Value, Guid.NewGuid(), cts.Token).ConfigureAwait(false); + await _ipcManager.PalettePlusRemovePaletteAsync(item.Value.Address).ConfigureAwait(false); } else { @@ -56,6 +58,7 @@ public class MareCharaFileManager : DisposableMediatorSubscriberBase _ipcManager.GlamourerRevertByName(logger, item.Key, Guid.NewGuid()); } + item.Value.Dispose(); } _gposeGameObjects.Clear(); @@ -80,7 +83,7 @@ public class MareCharaFileManager : DisposableMediatorSubscriberBase using var lz4Stream = new LZ4Stream(unwrapped, LZ4StreamMode.Decompress, LZ4StreamFlags.HighCompression); using var reader = new BinaryReader(lz4Stream); MareCharaFileHeader.AdvanceReaderToData(reader); - _logger.LogDebug("Applying to {chara}, expected length of contents: {exp}", charaTarget.Name.TextValue, expectedLength); + _logger.LogDebug("Applying to {chara}, expected length of contents: {exp}, stream length: {len}", charaTarget.Name.TextValue, expectedLength, reader.BaseStream.Length); extractedFiles = ExtractFilesFromCharaFile(LoadedCharaFile, reader, expectedLength); Dictionary fileSwaps = new(StringComparer.Ordinal); foreach (var fileSwap in LoadedCharaFile.CharaFileData.FileSwaps) @@ -178,11 +181,11 @@ public class MareCharaFileManager : DisposableMediatorSubscriberBase expectedLength += item.Length; foreach (var gamePath in item.GamePaths) { - _logger.LogTrace("File {itemNr}: {gamePath} = {len}", itemNr, gamePath, item.Length); + _logger.LogTrace("File {itemNr}: {gamePath} = {len}", itemNr, gamePath, item.Length.ToByteString()); } } - _logger.LogInformation("Expected length: {expected}", expectedLength); + _logger.LogInformation("Expected length: {expected}", expectedLength.ToByteString()); } return expectedLength; } @@ -203,22 +206,26 @@ public class MareCharaFileManager : DisposableMediatorSubscriberBase using var lz4 = new LZ4Stream(fs, LZ4StreamMode.Compress, LZ4StreamFlags.HighCompression); using var writer = new BinaryWriter(lz4); output.WriteToStream(writer); - var bufferSize = 4 * 1024 * 1024; - byte[] buffer = new byte[bufferSize]; - var playerReplacements = dto.FileReplacements[ObjectKind.Player]; foreach (var item in output.CharaFileData.Files) { - var itemFromData = playerReplacements.First(f => f.GamePaths.Any(p => item.GamePaths.Contains(p, StringComparer.OrdinalIgnoreCase))); - var file = _manager.GetFileCacheByHash(itemFromData.Hash)!; + var file = _manager.GetFileCacheByHash(item.Hash)!; + _logger.LogDebug("Saving to MCDF: {hash}:{file}", item.Hash, file.ResolvedFilepath); + _logger.LogDebug("\tAssociated GamePaths:"); + foreach (var path in item.GamePaths) + { + _logger.LogDebug("\t{path}", path); + } using var fsRead = File.OpenRead(file.ResolvedFilepath); using var br = new BinaryReader(fsRead); - int readBytes = 0; - while ((readBytes = br.Read(buffer, 0, bufferSize)) > 0) - { - writer.Write(readBytes == bufferSize ? buffer : buffer.Take(readBytes).ToArray()); - } + byte[] buffer = new byte[item.Length]; + br.Read(buffer, 0, item.Length); + writer.Write(buffer); } + writer.Flush(); + lz4.Flush(); + fs.Flush(); + fs.Close(); } catch (Exception ex) { @@ -235,11 +242,11 @@ public class MareCharaFileManager : DisposableMediatorSubscriberBase foreach (var fileData in charaFileHeader.CharaFileData.Files) { var fileName = Path.Combine(_configService.Current.CacheFolder, "mare_" + _globalFileCounter++ + ".tmp"); - var length = (int)fileData.Length; + var length = fileData.Length; var bufferSize = length; using var fs = File.OpenWrite(fileName); using var wr = new BinaryWriter(fs); - _logger.LogTrace("Reading {length} of {fileName}", length, fileName); + _logger.LogTrace("Reading {length} of {fileName}", length.ToByteString(), fileName); var buffer = reader.ReadBytes(bufferSize); wr.Write(buffer); wr.Flush(); @@ -248,10 +255,10 @@ public class MareCharaFileManager : DisposableMediatorSubscriberBase foreach (var path in fileData.GamePaths) { gamePathToFilePath[path] = fileName; - _logger.LogTrace("{path} => {fileName}", path, fileName); + _logger.LogTrace("{path} => {fileName} [{hash}]", path, fileName, fileData.Hash); } totalRead += length; - _logger.LogTrace("Read {read}/{expected} bytes", totalRead, expectedLength); + _logger.LogTrace("Read {read}/{expected} bytes", totalRead.ToByteString(), expectedLength.ToByteString()); } return gamePathToFilePath; diff --git a/MareSynchronos/Utils/VariousExtensions.cs b/MareSynchronos/Utils/VariousExtensions.cs index aeff6b4..325a084 100644 --- a/MareSynchronos/Utils/VariousExtensions.cs +++ b/MareSynchronos/Utils/VariousExtensions.cs @@ -10,6 +10,32 @@ namespace MareSynchronos.Utils; public static class VariousExtensions { + public static string ToByteString(this int bytes, bool addSuffix = true) + { + string[] suffix = ["B", "KiB", "MiB", "GiB", "TiB"]; + int i; + double dblSByte = bytes; + for (i = 0; i < suffix.Length && bytes >= 1024; i++, bytes /= 1024) + { + dblSByte = bytes / 1024.0; + } + + return addSuffix ? $"{dblSByte:0.00} {suffix[i]}" : $"{dblSByte:0.00}"; + } + + public static string ToByteString(this long bytes, bool addSuffix = true) + { + string[] suffix = ["B", "KiB", "MiB", "GiB", "TiB"]; + int i; + double dblSByte = bytes; + for (i = 0; i < suffix.Length && bytes >= 1024; i++, bytes /= 1024) + { + dblSByte = bytes / 1024.0; + } + + return addSuffix ? $"{dblSByte:0.00} {suffix[i]}" : $"{dblSByte:0.00}"; + } + public static void CancelDispose(this CancellationTokenSource? cts) { try