From 417c08f9b2169b314ad2485cf8c00c02cad53a68 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Mon, 10 Jul 2023 13:23:44 +0200 Subject: [PATCH] add CharacterAnalzyer, fix DtrEntry (I think) --- MareSynchronos/FileCache/FileCacheManager.cs | 10 +- MareSynchronos/Plugin.cs | 6 +- MareSynchronos/Services/CharacterAnalyzer.cs | 97 +++++++++++++++++++ .../Services/CommandManagerService.cs | 16 ++- MareSynchronos/UI/DtrEntry.cs | 39 +++++--- MareSynchronos/UI/SettingsUi.cs | 22 ++++- .../WebAPI/Files/FileUploadManager.cs | 9 +- 7 files changed, 170 insertions(+), 29 deletions(-) create mode 100644 MareSynchronos/Services/CharacterAnalyzer.cs diff --git a/MareSynchronos/FileCache/FileCacheManager.cs b/MareSynchronos/FileCache/FileCacheManager.cs index 4a8feac..9beb83d 100644 --- a/MareSynchronos/FileCache/FileCacheManager.cs +++ b/MareSynchronos/FileCache/FileCacheManager.cs @@ -1,4 +1,5 @@ -using MareSynchronos.Interop; +using LZ4; +using MareSynchronos.Interop; using MareSynchronos.MareConfiguration; using MareSynchronos.Utils; using Microsoft.Extensions.Logging; @@ -137,6 +138,13 @@ public sealed class FileCacheManager : IDisposable return validatedCacheEntry; } + public async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken) + { + var fileCache = GetFileCacheByHash(fileHash)!.ResolvedFilepath; + return (fileHash, LZ4Codec.WrapHC(await File.ReadAllBytesAsync(fileCache, uploadToken).ConfigureAwait(false), 0, + (int)new FileInfo(fileCache).Length)); + } + public void RemoveHashedFile(FileCacheEntity? fileCache) { if (fileCache == null) return; diff --git a/MareSynchronos/Plugin.cs b/MareSynchronos/Plugin.cs index 44aed3a..c762e79 100644 --- a/MareSynchronos/Plugin.cs +++ b/MareSynchronos/Plugin.cs @@ -72,11 +72,12 @@ public sealed class Plugin : IDalamudPlugin collection.AddSingleton(); collection.AddSingleton(); collection.AddSingleton(); + collection.AddSingleton(); collection.AddSingleton(); collection.AddSingleton((s) => new DalamudUtilService(s.GetRequiredService>(), clientState, objectTable, framework, gameGui, condition, gameData, s.GetRequiredService(), s.GetRequiredService())); - collection.AddSingleton((s) => new DtrEntry(dtrBar, s.GetRequiredService(), + collection.AddSingleton((s) => new DtrEntry(s.GetRequiredService>(), dtrBar, s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService())); collection.AddSingleton((s) => new IpcManager(s.GetRequiredService>(), pluginInterface, s.GetRequiredService(), s.GetRequiredService())); @@ -116,7 +117,8 @@ public sealed class Plugin : IDalamudPlugin s.GetRequiredService(), s.GetServices(), s.GetRequiredService>(), s.GetRequiredService(), s.GetRequiredService())); collection.AddScoped((s) => new CommandManagerService(commandManager, s.GetRequiredService(), s.GetRequiredService(), - s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService())); + s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), + s.GetRequiredService(), s.GetRequiredService())); collection.AddScoped((s) => new NotificationService(s.GetRequiredService>(), s.GetRequiredService(), pluginInterface.UiBuilder, chatGui, s.GetRequiredService())); collection.AddScoped((s) => new UiSharedService(s.GetRequiredService>(), s.GetRequiredService(), s.GetRequiredService(), diff --git a/MareSynchronos/Services/CharacterAnalyzer.cs b/MareSynchronos/Services/CharacterAnalyzer.cs new file mode 100644 index 0000000..16319ba --- /dev/null +++ b/MareSynchronos/Services/CharacterAnalyzer.cs @@ -0,0 +1,97 @@ +using MareSynchronos.API.Data; +using MareSynchronos.FileCache; +using MareSynchronos.Services.Mediator; +using MareSynchronos.UI; +using MareSynchronos.Utils; +using Microsoft.Extensions.Logging; + +namespace MareSynchronos.Services; + +public sealed class CharacterAnalyzer : MediatorSubscriberBase, IDisposable +{ + private readonly FileCacheManager _fileCacheManager; + private CharacterData? _lastCreatedData; + private CancellationTokenSource? _analysisCts; + + public CharacterAnalyzer(ILogger logger, MareMediator mediator, FileCacheManager fileCacheManager) : base(logger, mediator) + { + Mediator.Subscribe(this, (msg) => + { + _lastCreatedData = msg.CharacterData.DeepClone(); + }); + _fileCacheManager = fileCacheManager; + } + + public bool IsAnalysisRunning => _analysisCts != null; + + public void CancelAnalyze() + { + _analysisCts?.CancelDispose(); + _analysisCts = null; + } + + public async Task Analyze() + { + _analysisCts = _analysisCts?.CancelRecreate() ?? new(); + + var cancelToken = _analysisCts.Token; + + if (_lastCreatedData == null) return; + + Logger.LogInformation("=== Calculating Character Analysis, this may take a while ==="); + foreach (var obj in _lastCreatedData.FileReplacements) + { + Logger.LogInformation("=== File Calculation for {obj} ===", obj.Key); + Dictionary> data = new(StringComparer.OrdinalIgnoreCase); + var totalFiles = obj.Value.Count(c => !string.IsNullOrEmpty(c.Hash)); + var currentFile = 1; + foreach (var hash in obj.Value.Select(c => c.Hash)) + { + var fileCacheEntry = _fileCacheManager.GetFileCacheByHash(hash); + if (fileCacheEntry == null) continue; + + Logger.LogInformation("Computing File {x}/{y}: {hash}", currentFile, totalFiles, hash); + + Logger.LogInformation(" File Path: {path}", fileCacheEntry.ResolvedFilepath); + + var filePath = fileCacheEntry.ResolvedFilepath; + FileInfo fi = new(filePath); + var ext = fi.Extension; + if (!data.ContainsKey(ext)) data[ext] = new List(); + + (_, byte[] fileLength) = await _fileCacheManager.GetCompressedFileData(hash, cancelToken).ConfigureAwait(false); + + Logger.LogInformation(" Original Size: {size}, Compressed Size: {compr}", + UiSharedService.ByteToString(fi.Length), UiSharedService.ByteToString(fileLength.LongLength)); + + data[ext].Add(new DataEntry(fi.FullName, fi.Length, fileLength.LongLength)); + + currentFile++; + + cancelToken.ThrowIfCancellationRequested(); + } + + Logger.LogInformation("=== Summary by file type for {obj} ===", obj.Key); + foreach (var entry in data) + { + Logger.LogInformation("{ext} files: {count}, size extracted: {size}, size compressed: {sizeComp}", entry.Key, entry.Value.Count, + UiSharedService.ByteToString(entry.Value.Sum(v => v.OriginalSize)), UiSharedService.ByteToString(entry.Value.Sum(v => v.CompressedSize))); + } + Logger.LogInformation("=== Total summary for {obj} ===", obj.Key); + Logger.LogInformation("Total files: {count}, size extracted: {size}, size compressed: {sizeComp}", data.Values.Sum(c => c.Count), + UiSharedService.ByteToString(data.Values.Sum(v => v.Sum(c => c.OriginalSize))), UiSharedService.ByteToString(data.Values.Sum(v => v.Sum(c => c.CompressedSize)))); + + Logger.LogInformation("IMPORTANT NOTES:\n\r- For Mare up- and downloads only the compressed size is relevant.\n\r- An unusually high total files count beyond 200 and up will also increase your download time to others significantly."); + } + + _analysisCts.CancelDispose(); + _analysisCts = null; + } + + public void Dispose() + { + _analysisCts.CancelDispose(); + } + + private sealed record DataEntry(string filePath, long OriginalSize, long CompressedSize); +} diff --git a/MareSynchronos/Services/CommandManagerService.cs b/MareSynchronos/Services/CommandManagerService.cs index 05a43c1..5a4691c 100644 --- a/MareSynchronos/Services/CommandManagerService.cs +++ b/MareSynchronos/Services/CommandManagerService.cs @@ -14,6 +14,7 @@ public sealed class CommandManagerService : IDisposable private readonly ApiController _apiController; private readonly CommandManager _commandManager; private readonly MareMediator _mediator; + private readonly CharacterAnalyzer _characterAnalyzer; private readonly PerformanceCollectorService _performanceCollectorService; private readonly PeriodicFileScanner _periodicFileScanner; private readonly ServerConfigurationManager _serverConfigurationManager; @@ -21,7 +22,7 @@ public sealed class CommandManagerService : IDisposable public CommandManagerService(CommandManager commandManager, PerformanceCollectorService performanceCollectorService, UiService uiService, ServerConfigurationManager serverConfigurationManager, PeriodicFileScanner periodicFileScanner, - ApiController apiController, MareMediator mediator) + ApiController apiController, MareMediator mediator, CharacterAnalyzer characterAnalyzer) { _commandManager = commandManager; _performanceCollectorService = performanceCollectorService; @@ -30,7 +31,7 @@ public sealed class CommandManagerService : IDisposable _periodicFileScanner = periodicFileScanner; _apiController = apiController; _mediator = mediator; - + _characterAnalyzer = characterAnalyzer; _commandManager.AddHandler(_commandName, new CommandInfo(OnCommand) { HelpMessage = "Opens the Mare Synchronos UI" @@ -99,5 +100,16 @@ public sealed class CommandManagerService : IDisposable { _mediator.PrintSubscriberInfo(); } + else if (string.Equals(splitArgs[0], "analyze", StringComparison.OrdinalIgnoreCase)) + { + if (splitArgs.Length > 1 && string.Equals(splitArgs[1], "cancel", StringComparison.OrdinalIgnoreCase)) + { + _characterAnalyzer.CancelAnalyze(); + } + else + { + _ = _characterAnalyzer.Analyze(); + } + } } } \ No newline at end of file diff --git a/MareSynchronos/UI/DtrEntry.cs b/MareSynchronos/UI/DtrEntry.cs index 768738e..57f28d7 100644 --- a/MareSynchronos/UI/DtrEntry.cs +++ b/MareSynchronos/UI/DtrEntry.cs @@ -2,9 +2,9 @@ using MareSynchronos.MareConfiguration; using MareSynchronos.MareConfiguration.Configurations; using MareSynchronos.PlayerData.Pairs; -using MareSynchronos.Utils; using MareSynchronos.WebAPI; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; namespace MareSynchronos.UI; @@ -12,25 +12,25 @@ public sealed class DtrEntry : IDisposable, IHostedService { private readonly ApiController _apiController; private readonly CancellationTokenSource _cancellationTokenSource = new(); + private readonly ILogger _logger; private readonly ConfigurationServiceBase _configService; - private readonly DtrBarEntry _entry; + private readonly Lazy _entry; private readonly PairManager _pairManager; private Task? _runTask; private string? _text; - public DtrEntry(DtrBar dtrBar, ConfigurationServiceBase configService, PairManager pairManager, ApiController apiController) + public DtrEntry(ILogger logger, DtrBar dtrBar, ConfigurationServiceBase configService, PairManager pairManager, ApiController apiController) { - _entry = dtrBar.Get("Mare Synchronos"); + _entry = new(() => dtrBar.Get("Mare Synchronos")); + _logger = logger; _configService = configService; _pairManager = pairManager; _apiController = apiController; - - Clear(); } public void Dispose() { - _entry.Dispose(); + _entry.Value.Dispose(); } public Task StartAsync(CancellationToken cancellationToken) @@ -49,8 +49,16 @@ public sealed class DtrEntry : IDisposable, IHostedService catch (OperationCanceledException) { } finally { + _logger.LogDebug("Disposing DtrEntry"); + if (_entry.IsValueCreated) + { + Clear(); + + _entry.Value.Remove(); + _entry.Value.Dispose(); + } + _cancellationTokenSource.Dispose(); - Clear(); } } @@ -58,16 +66,17 @@ public sealed class DtrEntry : IDisposable, IHostedService { _text = null; - _entry.Shown = false; - _entry.Text = null; + _entry.Value.Shown = false; + _entry.Value.Text = null; } private async Task RunAsync() { while (!_cancellationTokenSource.IsCancellationRequested) { - Update(); await Task.Delay(1000, _cancellationTokenSource.Token).ConfigureAwait(false); + + Update(); } } @@ -75,16 +84,16 @@ public sealed class DtrEntry : IDisposable, IHostedService { if (!_configService.Current.EnableDtrEntry) { - if (_entry.Shown) + if (_entry.Value.Shown) { Clear(); } return; } - if (!_entry.Shown) + if (!_entry.Value.Shown) { - _entry.Shown = true; + _entry.Value.Shown = true; } string text; @@ -99,7 +108,7 @@ public sealed class DtrEntry : IDisposable, IHostedService if (!string.Equals(text, _text, StringComparison.Ordinal)) { _text = text; - _entry.Text = text; + _entry.Value.Text = text; } } } \ No newline at end of file diff --git a/MareSynchronos/UI/SettingsUi.cs b/MareSynchronos/UI/SettingsUi.cs index efd2c24..9309f8a 100644 --- a/MareSynchronos/UI/SettingsUi.cs +++ b/MareSynchronos/UI/SettingsUi.cs @@ -29,6 +29,7 @@ public class SettingsUi : WindowMediatorSubscriberBase private readonly ConcurrentDictionary> _currentDownloads = new(); private readonly FileUploadManager _fileTransferManager; private readonly FileTransferOrchestrator _fileTransferOrchestrator; + private readonly CharacterAnalyzer _characterAnalyzer; private readonly MareCharaFileManager _mareCharaFileManager; private readonly PairManager _pairManager; private readonly PerformanceCollectorService _performanceCollector; @@ -50,7 +51,8 @@ public class SettingsUi : WindowMediatorSubscriberBase ServerConfigurationManager serverConfigurationManager, MareMediator mediator, PerformanceCollectorService performanceCollector, FileUploadManager fileTransferManager, - FileTransferOrchestrator fileTransferOrchestrator) : base(logger, mediator, "Mare Synchronos Settings") + FileTransferOrchestrator fileTransferOrchestrator, + CharacterAnalyzer characterAnalyzer) : base(logger, mediator, "Mare Synchronos Settings") { _configService = configService; _mareCharaFileManager = mareCharaFileManager; @@ -59,6 +61,7 @@ public class SettingsUi : WindowMediatorSubscriberBase _performanceCollector = performanceCollector; _fileTransferManager = fileTransferManager; _fileTransferOrchestrator = fileTransferOrchestrator; + _characterAnalyzer = characterAnalyzer; _uiShared = uiShared; SizeConstraints = new WindowSizeConstraints() @@ -331,6 +334,23 @@ public class SettingsUi : WindowMediatorSubscriberBase } UiSharedService.AttachToolTip("Use this when reporting mods being rejected from the server."); + var isAnalyzing = _characterAnalyzer.IsAnalysisRunning; + if (isAnalyzing) ImGui.BeginDisabled(); + if (UiSharedService.IconTextButton(FontAwesomeIcon.QuestionCircle, "[DEBUG] Analyze current character composition to /xllog")) + { + _ = _characterAnalyzer.Analyze(); + } + UiSharedService.AttachToolTip("This will compute your current \"Mare load\" and print it to the /xllog"); + if (isAnalyzing) ImGui.EndDisabled(); + ImGui.SameLine(); + if (!isAnalyzing) ImGui.BeginDisabled(); + if (UiSharedService.IconTextButton(FontAwesomeIcon.StopCircle, "Cancel analysis")) + { + _characterAnalyzer.CancelAnalyze(); + } + UiSharedService.AttachToolTip("Cancels the current analysis of your character composition"); + if (!isAnalyzing) ImGui.EndDisabled(); + _uiShared.DrawCombo("Log Level", Enum.GetValues(), (l) => l.ToString(), (l) => { _configService.Current.LogLevel = l; diff --git a/MareSynchronos/WebAPI/Files/FileUploadManager.cs b/MareSynchronos/WebAPI/Files/FileUploadManager.cs index d49746b..91e2cc4 100644 --- a/MareSynchronos/WebAPI/Files/FileUploadManager.cs +++ b/MareSynchronos/WebAPI/Files/FileUploadManager.cs @@ -106,13 +106,6 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase return await response.Content.ReadFromJsonAsync>(cancellationToken: ct).ConfigureAwait(false) ?? new List(); } - private async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken) - { - var fileCache = _fileDbManager.GetFileCacheByHash(fileHash)!.ResolvedFilepath; - return (fileHash, LZ4Codec.WrapHC(await File.ReadAllBytesAsync(fileCache, uploadToken).ConfigureAwait(false), 0, - (int)new FileInfo(fileCache).Length)); - } - private HashSet GetUnverifiedFiles(CharacterData data) { HashSet unverifiedUploadHashes = new(StringComparer.Ordinal); @@ -245,7 +238,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase foreach (var file in CurrentUploads.Where(f => f.CanBeTransferred && !f.IsTransferred).ToList()) { Logger.LogDebug("[{hash}] Compressing", file); - var data = await GetCompressedFileData(file.Hash, uploadToken).ConfigureAwait(false); + var data = await _fileDbManager.GetCompressedFileData(file.Hash, uploadToken).ConfigureAwait(false); CurrentUploads.Single(e => string.Equals(e.Hash, data.Item1, StringComparison.Ordinal)).Total = data.Item2.Length; Logger.LogDebug("[{hash}] Starting upload for {filePath}", data.Item1, _fileDbManager.GetFileCacheByHash(data.Item1)!.ResolvedFilepath); await uploadTask.ConfigureAwait(false);