From e1ca5dd6f8ec1ae772655acaf4c0b2fae3ca45f3 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Thu, 15 Feb 2024 02:38:41 +0100 Subject: [PATCH] make storage size calculation asynchronous and running in parallel --- MareSynchronos/FileCache/CacheMonitor.cs | 79 ++++++++++++++----- MareSynchronos/FileCache/FileCacheManager.cs | 12 +++ MareSynchronos/FileCache/FileCompactor.cs | 14 +++- MareSynchronos/Plugin.cs | 6 +- MareSynchronos/Services/DalamudUtilService.cs | 2 + .../Services/Mediator/MareMediator.cs | 4 +- .../Services/PerformanceCollectorService.cs | 2 + MareSynchronos/UI/DtrEntry.cs | 4 +- MareSynchronos/UI/SettingsUi.cs | 17 ++-- 9 files changed, 105 insertions(+), 35 deletions(-) diff --git a/MareSynchronos/FileCache/CacheMonitor.cs b/MareSynchronos/FileCache/CacheMonitor.cs index 8a149c5..3215cae 100644 --- a/MareSynchronos/FileCache/CacheMonitor.cs +++ b/MareSynchronos/FileCache/CacheMonitor.cs @@ -18,6 +18,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase private readonly PerformanceCollectorService _performanceCollector; private long _currentFileProgress = 0; private CancellationTokenSource _scanCancellationTokenSource = new(); + private readonly CancellationTokenSource _periodicCalculationTokenSource = new(); private readonly string[] _allowedExtensions = [".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".pbd", ".scd", ".skp", ".shpk"]; public CacheMonitor(ILogger logger, IpcManager ipcManager, MareConfigService configService, @@ -57,6 +58,25 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase { StartMareWatcher(configService.Current.CacheFolder); } + + var token = _periodicCalculationTokenSource.Token; + _ = Task.Run(async () => + { + Logger.LogInformation("Starting Periodic Storage Directory Calculation Task"); + var token = _periodicCalculationTokenSource.Token; + while (!token.IsCancellationRequested) + { + try + { + RecalculateFileCacheSize(token); + } + catch + { + // ignore + } + await Task.Delay(TimeSpan.FromMinutes(1), token).ConfigureAwait(false); + } + }, token); } public long CurrentFileProgress => _currentFileProgress; @@ -86,17 +106,21 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase PenumbraWatcher = null; } + public bool StorageisNTFS { get; private set; } = false; + public void StartMareWatcher(string? marePath) { MareWatcher?.Dispose(); - if (string.IsNullOrEmpty(marePath)) + if (string.IsNullOrEmpty(marePath) || !Directory.Exists(marePath)) { MareWatcher = null; Logger.LogWarning("Mare file path is not set, cannot start the FSW for Mare."); return; } - RecalculateFileCacheSize(); + DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName); + StorageisNTFS = string.Equals("NTFS", di.DriveFormat, StringComparison.OrdinalIgnoreCase); + Logger.LogInformation("Mare Storage is on NTFS drive: {isNtfs}", StorageisNTFS); Logger.LogDebug("Initializing Mare FSW on {path}", marePath); MareWatcher = new() @@ -248,9 +272,6 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase } } - _ = RecalculateFileCacheSize(); - - if (changes.Any(c => c.Value.ChangeType == WatcherChangeTypes.Deleted)) { lock (_fileDbManager) @@ -356,26 +377,43 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase }, token); } - public bool RecalculateFileCacheSize() + public void RecalculateFileCacheSize(CancellationToken token) { - FileCacheSize = Directory.EnumerateFiles(_configService.Current.CacheFolder).Sum(f => + if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder)) { - try - { - return _fileCompactor.GetFileSizeOnDisk(f); - } - catch - { - return 0; - } - }); + FileCacheSize = 0; + return; + } - DriveInfo di = new DriveInfo(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName); - FileCacheDriveFree = di.AvailableFreeSpace; + FileCacheSize = -1; + DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName); + try + { + FileCacheDriveFree = di.AvailableFreeSpace; + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder); + } + + FileCacheSize = Directory.EnumerateFiles(_configService.Current.CacheFolder) + .AsParallel().Sum(f => + { + token.ThrowIfCancellationRequested(); + + try + { + return _fileCompactor.GetFileSizeOnDisk(f, StorageisNTFS); + } + catch + { + return 0; + } + }); var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d); - if (FileCacheSize < maxCacheInBytes) return false; + if (FileCacheSize < maxCacheInBytes) return; var allFiles = Directory.EnumerateFiles(_configService.Current.CacheFolder) .Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList(); @@ -387,8 +425,6 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase File.Delete(oldestFile.FullName); allFiles.Remove(oldestFile); } - - return true; } public void ResetLocks() @@ -412,6 +448,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase MareWatcher?.Dispose(); _penumbraFswCts?.CancelDispose(); _mareFswCts?.CancelDispose(); + _periodicCalculationTokenSource?.CancelDispose(); } private void FullFileScan(CancellationToken ct) diff --git a/MareSynchronos/FileCache/FileCacheManager.cs b/MareSynchronos/FileCache/FileCacheManager.cs index fe2a687..57ad31f 100644 --- a/MareSynchronos/FileCache/FileCacheManager.cs +++ b/MareSynchronos/FileCache/FileCacheManager.cs @@ -367,12 +367,17 @@ public sealed class FileCacheManager : IHostedService public Task StartAsync(CancellationToken cancellationToken) { + _logger.LogInformation("Starting FileCacheManager"); lock (_fileWriteLock) { try { + _logger.LogInformation("Checking for {bakPath}", CsvBakPath); + if (File.Exists(CsvBakPath)) { + _logger.LogInformation("{bakPath} found, moving to {csvPath}", CsvBakPath, _csvPath); + File.Move(CsvBakPath, _csvPath, overwrite: true); } } @@ -393,6 +398,8 @@ public sealed class FileCacheManager : IHostedService if (File.Exists(_csvPath)) { + _logger.LogInformation("{csvPath} found, parsing", _csvPath); + bool success = false; string[] entries = []; int attempts = 0; @@ -400,6 +407,7 @@ public sealed class FileCacheManager : IHostedService { try { + _logger.LogInformation("Attempting to read {csvPath}", _csvPath); entries = File.ReadAllLines(_csvPath); success = true; } @@ -416,6 +424,8 @@ public sealed class FileCacheManager : IHostedService _logger.LogWarning("Could not load entries from {path}, continuing with empty file cache", _csvPath); } + _logger.LogInformation("Found {amount} files in {path}", entries.Length, _csvPath); + Dictionary processedFiles = new(StringComparer.OrdinalIgnoreCase); foreach (var entry in entries) { @@ -462,6 +472,8 @@ public sealed class FileCacheManager : IHostedService } } + _logger.LogInformation("Started FileCacheManager"); + return Task.CompletedTask; } diff --git a/MareSynchronos/FileCache/FileCompactor.cs b/MareSynchronos/FileCache/FileCompactor.cs index 0c0efdd..a5f3bea 100644 --- a/MareSynchronos/FileCache/FileCompactor.cs +++ b/MareSynchronos/FileCache/FileCompactor.cs @@ -62,9 +62,11 @@ public sealed class FileCompactor MassCompactRunning = false; } - public long GetFileSizeOnDisk(string filePath) + public long GetFileSizeOnDisk(string filePath, bool? isNTFS = null) { - if (Dalamud.Utility.Util.IsWine()) return new FileInfo(filePath).Length; + bool ntfs = isNTFS ?? string.Equals(new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName).DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase); + + if (Dalamud.Utility.Util.IsWine() || !ntfs) return new FileInfo(filePath).Length; var clusterSize = GetClusterSize(filePath); if (clusterSize == -1) return new FileInfo(filePath).Length; @@ -105,6 +107,14 @@ public sealed class FileCompactor private void CompactFile(string filePath) { + var fs = new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName); + bool isNTFS = string.Equals(fs.DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase); + if (!isNTFS) + { + _logger.LogWarning("Drive for file {file} is not NTFS", filePath); + return; + } + var oldSize = new FileInfo(filePath).Length; var clusterSize = GetClusterSize(filePath); diff --git a/MareSynchronos/Plugin.cs b/MareSynchronos/Plugin.cs index 3ad35cd..f848d9e 100644 --- a/MareSynchronos/Plugin.cs +++ b/MareSynchronos/Plugin.cs @@ -146,13 +146,13 @@ public sealed class Plugin : IDalamudPlugin s.GetRequiredService>(), gameInteropProvider, chatGui, s.GetRequiredService(), s.GetRequiredService())); - collection.AddHostedService(p => p.GetRequiredService()); + collection.AddHostedService(p => p.GetRequiredService()); + collection.AddHostedService(p => p.GetRequiredService()); collection.AddHostedService(p => p.GetRequiredService()); collection.AddHostedService(p => p.GetRequiredService()); collection.AddHostedService(p => p.GetRequiredService()); collection.AddHostedService(p => p.GetRequiredService()); - collection.AddHostedService(p => p.GetRequiredService()); - collection.AddHostedService(p => p.GetRequiredService()); + collection.AddHostedService(p => p.GetRequiredService()); }) .Build() .RunAsync(_pluginCts.Token); diff --git a/MareSynchronos/Services/DalamudUtilService.cs b/MareSynchronos/Services/DalamudUtilService.cs index 74620d0..0429644 100644 --- a/MareSynchronos/Services/DalamudUtilService.cs +++ b/MareSynchronos/Services/DalamudUtilService.cs @@ -305,6 +305,7 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber public Task StartAsync(CancellationToken cancellationToken) { + _logger.LogInformation("Starting DalamudUtilService"); #pragma warning disable S2696 // Instance members should not write to "static" fields LoporritSync.Plugin.Self._realOnFrameworkUpdate = this.FrameworkOnUpdate; #pragma warning restore S2696 @@ -314,6 +315,7 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber _classJobId = _clientState.LocalPlayer!.ClassJob.RowId; } + _logger.LogInformation("Started DalamudUtilService"); return Task.CompletedTask; } diff --git a/MareSynchronos/Services/Mediator/MareMediator.cs b/MareSynchronos/Services/Mediator/MareMediator.cs index b5a654b..e8f24c4 100644 --- a/MareSynchronos/Services/Mediator/MareMediator.cs +++ b/MareSynchronos/Services/Mediator/MareMediator.cs @@ -54,7 +54,7 @@ public sealed class MareMediator : IHostedService public Task StartAsync(CancellationToken cancellationToken) { - _logger.LogTrace("Starting MareMediator"); + _logger.LogInformation("Starting MareMediator"); _ = Task.Run(async () => { @@ -73,7 +73,7 @@ public sealed class MareMediator : IHostedService } }); - _logger.LogTrace("Started MareMediator"); + _logger.LogInformation("Started MareMediator"); return Task.CompletedTask; } diff --git a/MareSynchronos/Services/PerformanceCollectorService.cs b/MareSynchronos/Services/PerformanceCollectorService.cs index aab74a4..617c5b8 100644 --- a/MareSynchronos/Services/PerformanceCollectorService.cs +++ b/MareSynchronos/Services/PerformanceCollectorService.cs @@ -78,7 +78,9 @@ public sealed class PerformanceCollectorService : IHostedService public Task StartAsync(CancellationToken cancellationToken) { + _logger.LogInformation("Starting PerformanceCollectorService"); _ = Task.Run(PeriodicLogPrune, _periodicLogPruneTask.Token); + _logger.LogInformation("Started PerformanceCollectorService"); return Task.CompletedTask; } diff --git a/MareSynchronos/UI/DtrEntry.cs b/MareSynchronos/UI/DtrEntry.cs index ac35361..2bd66cb 100644 --- a/MareSynchronos/UI/DtrEntry.cs +++ b/MareSynchronos/UI/DtrEntry.cs @@ -62,7 +62,9 @@ public sealed class DtrEntry : IDisposable, IHostedService public Task StartAsync(CancellationToken cancellationToken) { + _logger.LogInformation("Starting DtrEntry"); _runTask = Task.Run(RunAsync, _cancellationTokenSource.Token); + _logger.LogInformation("Started DtrEntry"); return Task.CompletedTask; } @@ -144,7 +146,7 @@ public sealed class DtrEntry : IDisposable, IHostedService { visiblePairs = _pairManager.GetOnlineUserPairs() .Where(x => x.IsVisible) - .Select(x => string.Format("{0} ({1})", _configService.Current.PreferNoteInDtrTooltip ? x.GetNoteOrName() : x.PlayerName, x.UserData.AliasOrUID )); + .Select(x => string.Format("{0} ({1})", _configService.Current.PreferNoteInDtrTooltip ? x.GetNoteOrName() : x.PlayerName, x.UserData.AliasOrUID)); } else { diff --git a/MareSynchronos/UI/SettingsUi.cs b/MareSynchronos/UI/SettingsUi.cs index 4775f47..43e1981 100644 --- a/MareSynchronos/UI/SettingsUi.cs +++ b/MareSynchronos/UI/SettingsUi.cs @@ -785,7 +785,10 @@ public class SettingsUi : WindowMediatorSubscriberBase _uiShared.DrawCacheDirectorySetting(); ImGui.AlignTextToFramePadding(); - ImGui.TextUnformatted($"Currently utilized local storage: {UiSharedService.ByteToString(_cacheMonitor.FileCacheSize)}"); + if (_cacheMonitor.FileCacheSize >= 0) + ImGui.TextUnformatted($"Currently utilized local storage: {UiSharedService.ByteToString(_cacheMonitor.FileCacheSize)}"); + else + ImGui.TextUnformatted($"Currently utilized local storage: Calculating..."); ImGui.TextUnformatted($"Remaining space free on drive: {UiSharedService.ByteToString(_cacheMonitor.FileCacheDriveFree)}"); bool useFileCompactor = _configService.Current.UseCompactor; bool isLinux = Util.IsWine(); @@ -793,7 +796,7 @@ public class SettingsUi : WindowMediatorSubscriberBase { UiSharedService.ColorTextWrapped("Hint: To free up space when using Mare consider enabling the File Compactor", ImGuiColors.DalamudYellow); } - if (isLinux) ImGui.BeginDisabled(); + if (isLinux || !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); if (ImGui.Checkbox("Use file compactor", ref useFileCompactor)) { _configService.Current.UseCompactor = useFileCompactor; @@ -809,7 +812,8 @@ public class SettingsUi : WindowMediatorSubscriberBase _ = Task.Run(() => { _fileCompactor.CompactStorage(compress: true); - _ = _cacheMonitor.RecalculateFileCacheSize(); + CancellationTokenSource cts = new(); + _cacheMonitor.RecalculateFileCacheSize(cts.Token); }); } UiSharedService.AttachToolTip("This will run compression on all files in your current storage folder." + Environment.NewLine @@ -820,7 +824,8 @@ public class SettingsUi : WindowMediatorSubscriberBase _ = Task.Run(() => { _fileCompactor.CompactStorage(compress: false); - _ = _cacheMonitor.RecalculateFileCacheSize(); + CancellationTokenSource cts = new(); + _cacheMonitor.RecalculateFileCacheSize(cts.Token); }); } UiSharedService.AttachToolTip("This will run decompression on all files in your current storage folder."); @@ -829,10 +834,10 @@ public class SettingsUi : WindowMediatorSubscriberBase { UiSharedService.ColorText($"File compactor currently running ({_fileCompactor.Progress})", ImGuiColors.DalamudYellow); } - if (isLinux) + if (isLinux || !_cacheMonitor.StorageisNTFS) { ImGui.EndDisabled(); - ImGui.TextUnformatted("The file compactor is only available on Windows."); + ImGui.TextUnformatted("The file compactor is only available on Windows and NTFS drives."); } ImGuiHelpers.ScaledDummy(new Vector2(10, 10));