make storage size calculation asynchronous and running in parallel

MareSynchronos/FileCache/CacheMonitor.cs

@@ -18,6 +18,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
     private readonly PerformanceCollectorService _performanceCollector;
     private long _currentFileProgress = 0;
     private CancellationTokenSource _scanCancellationTokenSource = new();
+    private readonly CancellationTokenSource _periodicCalculationTokenSource = new();
     private readonly string[] _allowedExtensions = [".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".pbd", ".scd", ".skp", ".shpk"];
 
     public CacheMonitor(ILogger<CacheMonitor> logger, IpcManager ipcManager, MareConfigService configService,
@@ -57,6 +58,25 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         {
             StartMareWatcher(configService.Current.CacheFolder);
         }
+
+        var token = _periodicCalculationTokenSource.Token;
+        _ = Task.Run(async () =>
+        {
+            Logger.LogInformation("Starting Periodic Storage Directory Calculation Task");
+            var token = _periodicCalculationTokenSource.Token;
+            while (!token.IsCancellationRequested)
+            {
+                try
+                {
+                    RecalculateFileCacheSize(token);
+                }
+                catch
+                {
+                    // ignore
+                }
+                await Task.Delay(TimeSpan.FromMinutes(1), token).ConfigureAwait(false);
+            }
+        }, token);
     }
 
     public long CurrentFileProgress => _currentFileProgress;
@@ -86,17 +106,21 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         PenumbraWatcher = null;
     }
 
+    public bool StorageisNTFS { get; private set; } = false;
+
     public void StartMareWatcher(string? marePath)
     {
         MareWatcher?.Dispose();
-        if (string.IsNullOrEmpty(marePath))
+        if (string.IsNullOrEmpty(marePath) || !Directory.Exists(marePath))
         {
             MareWatcher = null;
             Logger.LogWarning("Mare file path is not set, cannot start the FSW for Mare.");
             return;
         }
 
-        RecalculateFileCacheSize();
+        DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
+        StorageisNTFS = string.Equals("NTFS", di.DriveFormat, StringComparison.OrdinalIgnoreCase);
+        Logger.LogInformation("Mare Storage is on NTFS drive: {isNtfs}", StorageisNTFS);
 
         Logger.LogDebug("Initializing Mare FSW on {path}", marePath);
         MareWatcher = new()
@@ -248,9 +272,6 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
             }
         }
 
-        _ = RecalculateFileCacheSize();
-
-
         if (changes.Any(c => c.Value.ChangeType == WatcherChangeTypes.Deleted))
         {
             lock (_fileDbManager)
@@ -356,13 +377,33 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         }, token);
     }
 
-    public bool RecalculateFileCacheSize()
+    public void RecalculateFileCacheSize(CancellationToken token)
     {
-        FileCacheSize = Directory.EnumerateFiles(_configService.Current.CacheFolder).Sum(f =>
+        if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder))
         {
+            FileCacheSize = 0;
+            return;
+        }
+
+        FileCacheSize = -1;
+        DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
         try
         {
-                return _fileCompactor.GetFileSizeOnDisk(f);
+            FileCacheDriveFree = di.AvailableFreeSpace;
+        }
+        catch (Exception ex)
+        {
+            Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder);
+        }
+
+        FileCacheSize = Directory.EnumerateFiles(_configService.Current.CacheFolder)
+            .AsParallel().Sum(f =>
+            {
+                token.ThrowIfCancellationRequested();
+
+                try
+                {
+                    return _fileCompactor.GetFileSizeOnDisk(f, StorageisNTFS);
                 }
                 catch
                 {
@@ -370,12 +411,9 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
                 }
             });
 
-        DriveInfo di = new DriveInfo(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
-        FileCacheDriveFree = di.AvailableFreeSpace;
-
         var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d);
 
-        if (FileCacheSize < maxCacheInBytes) return false;
+        if (FileCacheSize < maxCacheInBytes) return;
 
         var allFiles = Directory.EnumerateFiles(_configService.Current.CacheFolder)
             .Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList();
@@ -387,8 +425,6 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
             File.Delete(oldestFile.FullName);
             allFiles.Remove(oldestFile);
         }
-
-        return true;
     }
 
     public void ResetLocks()
@@ -412,6 +448,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         MareWatcher?.Dispose();
         _penumbraFswCts?.CancelDispose();
         _mareFswCts?.CancelDispose();
+        _periodicCalculationTokenSource?.CancelDispose();
     }
 
     private void FullFileScan(CancellationToken ct)

MareSynchronos/FileCache/FileCacheManager.cs

@@ -367,12 +367,17 @@ public sealed class FileCacheManager : IHostedService
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
+        _logger.LogInformation("Starting FileCacheManager");
         lock (_fileWriteLock)
         {
             try
             {
+                _logger.LogInformation("Checking for {bakPath}", CsvBakPath);
+
                 if (File.Exists(CsvBakPath))
                 {
+                    _logger.LogInformation("{bakPath} found, moving to {csvPath}", CsvBakPath, _csvPath);
+
                     File.Move(CsvBakPath, _csvPath, overwrite: true);
                 }
             }
@@ -393,6 +398,8 @@ public sealed class FileCacheManager : IHostedService
 
         if (File.Exists(_csvPath))
         {
+            _logger.LogInformation("{csvPath} found, parsing", _csvPath);
+
             bool success = false;
             string[] entries = [];
             int attempts = 0;
@@ -400,6 +407,7 @@ public sealed class FileCacheManager : IHostedService
             {
                 try
                 {
+                    _logger.LogInformation("Attempting to read {csvPath}", _csvPath);
                     entries = File.ReadAllLines(_csvPath);
                     success = true;
                 }
@@ -416,6 +424,8 @@ public sealed class FileCacheManager : IHostedService
                 _logger.LogWarning("Could not load entries from {path}, continuing with empty file cache", _csvPath);
             }
 
+            _logger.LogInformation("Found {amount} files in {path}", entries.Length, _csvPath);
+
             Dictionary<string, bool> processedFiles = new(StringComparer.OrdinalIgnoreCase);
             foreach (var entry in entries)
             {
@@ -462,6 +472,8 @@ public sealed class FileCacheManager : IHostedService
             }
         }
 
+        _logger.LogInformation("Started FileCacheManager");
+
         return Task.CompletedTask;
     }
 

MareSynchronos/FileCache/FileCompactor.cs

@@ -62,9 +62,11 @@ public sealed class FileCompactor
         MassCompactRunning = false;
     }
 
-    public long GetFileSizeOnDisk(string filePath)
+    public long GetFileSizeOnDisk(string filePath, bool? isNTFS = null)
     {
-        if (Dalamud.Utility.Util.IsWine()) return new FileInfo(filePath).Length;
+        bool ntfs = isNTFS ?? string.Equals(new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName).DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase);
+
+        if (Dalamud.Utility.Util.IsWine() || !ntfs) return new FileInfo(filePath).Length;
 
         var clusterSize = GetClusterSize(filePath);
         if (clusterSize == -1) return new FileInfo(filePath).Length;
@@ -105,6 +107,14 @@ public sealed class FileCompactor
 
     private void CompactFile(string filePath)
     {
+        var fs = new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName);
+        bool isNTFS = string.Equals(fs.DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase);
+        if (!isNTFS)
+        {
+            _logger.LogWarning("Drive for file {file} is not NTFS", filePath);
+            return;
+        }
+
         var oldSize = new FileInfo(filePath).Length;
         var clusterSize = GetClusterSize(filePath);
 

MareSynchronos/Plugin.cs

@@ -146,13 +146,13 @@ public sealed class Plugin : IDalamudPlugin
                 s.GetRequiredService<ILogger<GameChatHooks>>(), gameInteropProvider, chatGui,
                 s.GetRequiredService<MareConfigService>(), s.GetRequiredService<ServerConfigurationManager>()));
 
-            collection.AddHostedService(p => p.GetRequiredService<MareMediator>());
+            collection.AddHostedService(p => p.GetRequiredService<MarePlugin>());
+            collection.AddHostedService(p => p.GetRequiredService<FileCacheManager>());
             collection.AddHostedService(p => p.GetRequiredService<ConfigurationMigrator>());
             collection.AddHostedService(p => p.GetRequiredService<DalamudUtilService>());
             collection.AddHostedService(p => p.GetRequiredService<PerformanceCollectorService>());
             collection.AddHostedService(p => p.GetRequiredService<DtrEntry>());
-            collection.AddHostedService(p => p.GetRequiredService<MarePlugin>());
+            collection.AddHostedService(p => p.GetRequiredService<MareMediator>());
-            collection.AddHostedService(p => p.GetRequiredService<FileCacheManager>());
         })
         .Build()
         .RunAsync(_pluginCts.Token);

MareSynchronos/Services/DalamudUtilService.cs

@@ -305,6 +305,7 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
+        _logger.LogInformation("Starting DalamudUtilService");
 #pragma warning disable S2696 // Instance members should not write to "static" fields
         LoporritSync.Plugin.Self._realOnFrameworkUpdate = this.FrameworkOnUpdate;
 #pragma warning restore S2696
@@ -314,6 +315,7 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber
             _classJobId = _clientState.LocalPlayer!.ClassJob.RowId;
         }
 
+        _logger.LogInformation("Started DalamudUtilService");
         return Task.CompletedTask;
     }
 

MareSynchronos/Services/Mediator/MareMediator.cs

@@ -54,7 +54,7 @@ public sealed class MareMediator : IHostedService
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
-        _logger.LogTrace("Starting MareMediator");
+        _logger.LogInformation("Starting MareMediator");
 
         _ = Task.Run(async () =>
         {
@@ -73,7 +73,7 @@ public sealed class MareMediator : IHostedService
             }
         });
 
-        _logger.LogTrace("Started MareMediator");
+        _logger.LogInformation("Started MareMediator");
 
         return Task.CompletedTask;
     }

MareSynchronos/Services/PerformanceCollectorService.cs

@@ -78,7 +78,9 @@ public sealed class PerformanceCollectorService : IHostedService
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
+        _logger.LogInformation("Starting PerformanceCollectorService");
         _ = Task.Run(PeriodicLogPrune, _periodicLogPruneTask.Token);
+        _logger.LogInformation("Started PerformanceCollectorService");
         return Task.CompletedTask;
     }
 

MareSynchronos/UI/DtrEntry.cs

@@ -62,7 +62,9 @@ public sealed class DtrEntry : IDisposable, IHostedService
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
+        _logger.LogInformation("Starting DtrEntry");
         _runTask = Task.Run(RunAsync, _cancellationTokenSource.Token);
+        _logger.LogInformation("Started DtrEntry");
         return Task.CompletedTask;
     }
 
@@ -144,7 +146,7 @@ public sealed class DtrEntry : IDisposable, IHostedService
                 {
                     visiblePairs = _pairManager.GetOnlineUserPairs()
                         .Where(x => x.IsVisible)
-                        .Select(x => string.Format("{0} ({1})", _configService.Current.PreferNoteInDtrTooltip ? x.GetNoteOrName() : x.PlayerName, x.UserData.AliasOrUID ));
+                        .Select(x => string.Format("{0} ({1})", _configService.Current.PreferNoteInDtrTooltip ? x.GetNoteOrName() : x.PlayerName, x.UserData.AliasOrUID));
                 }
                 else
                 {

MareSynchronos/UI/SettingsUi.cs

@@ -785,7 +785,10 @@ public class SettingsUi : WindowMediatorSubscriberBase
 
         _uiShared.DrawCacheDirectorySetting();
         ImGui.AlignTextToFramePadding();
+        if (_cacheMonitor.FileCacheSize >= 0)
             ImGui.TextUnformatted($"Currently utilized local storage: {UiSharedService.ByteToString(_cacheMonitor.FileCacheSize)}");
+        else
+            ImGui.TextUnformatted($"Currently utilized local storage: Calculating...");
         ImGui.TextUnformatted($"Remaining space free on drive: {UiSharedService.ByteToString(_cacheMonitor.FileCacheDriveFree)}");
         bool useFileCompactor = _configService.Current.UseCompactor;
         bool isLinux = Util.IsWine();
@@ -793,7 +796,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
         {
             UiSharedService.ColorTextWrapped("Hint: To free up space when using Mare consider enabling the File Compactor", ImGuiColors.DalamudYellow);
         }
-        if (isLinux) ImGui.BeginDisabled();
+        if (isLinux || !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled();
         if (ImGui.Checkbox("Use file compactor", ref useFileCompactor))
         {
             _configService.Current.UseCompactor = useFileCompactor;
@@ -809,7 +812,8 @@ public class SettingsUi : WindowMediatorSubscriberBase
                 _ = Task.Run(() =>
                 {
                     _fileCompactor.CompactStorage(compress: true);
-                    _ = _cacheMonitor.RecalculateFileCacheSize();
+                    CancellationTokenSource cts = new();
+                    _cacheMonitor.RecalculateFileCacheSize(cts.Token);
                 });
             }
             UiSharedService.AttachToolTip("This will run compression on all files in your current storage folder." + Environment.NewLine
@@ -820,7 +824,8 @@ public class SettingsUi : WindowMediatorSubscriberBase
                 _ = Task.Run(() =>
                 {
                     _fileCompactor.CompactStorage(compress: false);
-                    _ = _cacheMonitor.RecalculateFileCacheSize();
+                    CancellationTokenSource cts = new();
+                    _cacheMonitor.RecalculateFileCacheSize(cts.Token);
                 });
             }
             UiSharedService.AttachToolTip("This will run decompression on all files in your current storage folder.");
@@ -829,10 +834,10 @@ public class SettingsUi : WindowMediatorSubscriberBase
         {
             UiSharedService.ColorText($"File compactor currently running ({_fileCompactor.Progress})", ImGuiColors.DalamudYellow);
         }
-        if (isLinux)
+        if (isLinux || !_cacheMonitor.StorageisNTFS)
         {
             ImGui.EndDisabled();
-            ImGui.TextUnformatted("The file compactor is only available on Windows.");
+            ImGui.TextUnformatted("The file compactor is only available on Windows and NTFS drives.");
         }
         ImGuiHelpers.ScaledDummy(new Vector2(10, 10));