From 739c02cf0b3d463fe4e2ad7a7da19595471683e6 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Tue, 13 Feb 2024 00:56:27 +0100 Subject: [PATCH] rough impl of FSW, goodbye periodic filescan --- ...PeriodicFileScanner.cs => CacheMonitor.cs} | 381 ++++++++++++++---- MareSynchronos/FileCache/FileCacheManager.cs | 222 +++++----- MareSynchronos/Interop/IpcManager.cs | 14 +- MareSynchronos/MarePlugin.cs | 1 - MareSynchronos/Plugin.cs | 7 +- .../Services/CommandManagerService.cs | 8 +- MareSynchronos/Services/Mediator/Messages.cs | 1 + MareSynchronos/UI/IntroUI.cs | 10 +- MareSynchronos/UI/SettingsUi.cs | 62 ++- MareSynchronos/UI/UISharedService.cs | 71 ++-- MareSynchronos/Utils/Crypto.cs | 3 - 11 files changed, 525 insertions(+), 255 deletions(-) rename MareSynchronos/FileCache/{PeriodicFileScanner.cs => CacheMonitor.cs} (53%) diff --git a/MareSynchronos/FileCache/PeriodicFileScanner.cs b/MareSynchronos/FileCache/CacheMonitor.cs similarity index 53% rename from MareSynchronos/FileCache/PeriodicFileScanner.cs rename to MareSynchronos/FileCache/CacheMonitor.cs index 1cb2639..aefe459 100644 --- a/MareSynchronos/FileCache/PeriodicFileScanner.cs +++ b/MareSynchronos/FileCache/CacheMonitor.cs @@ -2,12 +2,13 @@ using MareSynchronos.MareConfiguration; using MareSynchronos.Services; using MareSynchronos.Services.Mediator; +using MareSynchronos.Utils; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; namespace MareSynchronos.FileCache; -public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase +public sealed class CacheMonitor : DisposableMediatorSubscriberBase { private readonly MareConfigService _configService; private readonly DalamudUtilService _dalamudUtil; @@ -16,11 +17,10 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase private readonly IpcManager _ipcManager; private readonly PerformanceCollectorService _performanceCollector; private long _currentFileProgress = 0; - private bool _fileScanWasRunning = false; private CancellationTokenSource _scanCancellationTokenSource = new(); - private TimeSpan _timeUntilNextScan = TimeSpan.Zero; + private readonly string[] _allowedExtensions = [".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".pbd", ".scd", ".skp", ".shpk"]; - public PeriodicFileScanner(ILogger logger, IpcManager ipcManager, MareConfigService configService, + public CacheMonitor(ILogger logger, IpcManager ipcManager, MareConfigService configService, FileCacheManager fileDbManager, MareMediator mediator, PerformanceCollectorService performanceCollector, DalamudUtilService dalamudUtil, FileCompactor fileCompactor) : base(logger, mediator) { @@ -30,38 +30,287 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase _performanceCollector = performanceCollector; _dalamudUtil = dalamudUtil; _fileCompactor = fileCompactor; - Mediator.Subscribe(this, (_) => StartScan()); + Mediator.Subscribe(this, (_) => + { + StartPenumbraWatcher(_ipcManager.PenumbraModDirectory); + StartMareWatcher(configService.Current.CacheFolder); + InvokeScan(); + }); Mediator.Subscribe(this, (msg) => HaltScan(msg.Source)); Mediator.Subscribe(this, (msg) => ResumeScan(msg.Source)); - Mediator.Subscribe(this, (_) => StartScan()); - Mediator.Subscribe(this, (_) => StartScan()); + Mediator.Subscribe(this, (_) => + { + StartMareWatcher(configService.Current.CacheFolder); + StartPenumbraWatcher(_ipcManager.PenumbraModDirectory); + InvokeScan(); + }); + Mediator.Subscribe(this, (msg) => StartPenumbraWatcher(msg.ModDirectory)); + if (_ipcManager.CheckPenumbraApi() && !string.IsNullOrEmpty(_ipcManager.PenumbraModDirectory)) + StartPenumbraWatcher(_ipcManager.PenumbraModDirectory); + if (configService.Current.HasValidSetup()) + { + StartMareWatcher(configService.Current.CacheFolder); + } } public long CurrentFileProgress => _currentFileProgress; public long FileCacheSize { get; set; } public ConcurrentDictionary HaltScanLocks { get; set; } = new(StringComparer.Ordinal); public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0; - public string TimeUntilNextScan => _timeUntilNextScan.ToString(@"mm\:ss"); public long TotalFiles { get; private set; } public long TotalFilesStorage { get; private set; } - private int TimeBetweenScans => _configService.Current.TimeSpanBetweenScansInSeconds; public void HaltScan(string source) { if (!HaltScanLocks.ContainsKey(source)) HaltScanLocks[source] = 0; HaltScanLocks[source]++; + } - if (IsScanRunning && HaltScanLocks.Any(f => f.Value > 0)) + record WatcherChange(WatcherChangeTypes ChangeType, string? OldPath = null); + private readonly Dictionary _watcherChanges = new Dictionary(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _mareChanges = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public void StopMonitoring() + { + Logger.LogInformation("Stopping monitoring of Penumbra and Mare storage folders"); + MareWatcher?.Dispose(); + PenumbraWatcher?.Dispose(); + MareWatcher = null; + PenumbraWatcher = null; + } + + public void StartMareWatcher(string? marePath) + { + MareWatcher?.Dispose(); + if (string.IsNullOrEmpty(marePath)) { - _scanCancellationTokenSource?.Cancel(); - _fileScanWasRunning = true; + MareWatcher = null; + Logger.LogWarning("Mare file path is not set, cannot start the FSW for Mare."); + return; + } + + RecalculateFileCacheSize(); + + Logger.LogDebug("Initializing Mare FSW on {path}", marePath); + MareWatcher = new() + { + Path = marePath, + InternalBufferSize = 8388608, + NotifyFilter = NotifyFilters.CreationTime + | NotifyFilters.LastWrite + | NotifyFilters.FileName + | NotifyFilters.DirectoryName + | NotifyFilters.Size, + Filter = "*.*", + IncludeSubdirectories = false + }; + + MareWatcher.Deleted += MareWatcher_FileChanged; + MareWatcher.Created += MareWatcher_FileChanged; + } + + private void MareWatcher_FileChanged(object sender, FileSystemEventArgs e) + { + if (!_allowedExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return; + + lock (_watcherChanges) + { + _mareChanges[e.FullPath] = new(e.ChangeType); + } + + _ = MareWatcherExecution(); + } + + public void StartPenumbraWatcher(string? penumbraPath) + { + PenumbraWatcher?.Dispose(); + if (string.IsNullOrEmpty(penumbraPath)) + { + PenumbraWatcher = null; + Logger.LogWarning("Penumbra is not connected or the path is not set, cannot start FSW for Penumbra."); + return; + } + + Logger.LogDebug("Initializing Penumbra FSW on {path}", penumbraPath); + PenumbraWatcher = new() + { + Path = penumbraPath, + InternalBufferSize = 8388608, + NotifyFilter = NotifyFilters.CreationTime + | NotifyFilters.LastWrite + | NotifyFilters.FileName + | NotifyFilters.DirectoryName + | NotifyFilters.Size, + Filter = "*.*", + IncludeSubdirectories = true + }; + + PenumbraWatcher.Deleted += Fs_Changed; + PenumbraWatcher.Created += Fs_Changed; + PenumbraWatcher.Changed += Fs_Changed; + PenumbraWatcher.Renamed += Fs_Renamed; + PenumbraWatcher.EnableRaisingEvents = true; + } + + private void Fs_Changed(object sender, FileSystemEventArgs e) + { + if (Directory.Exists(e.FullPath)) return; + if (!_allowedExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return; + + if (e.ChangeType is not (WatcherChangeTypes.Changed or WatcherChangeTypes.Deleted or WatcherChangeTypes.Created)) + return; + + lock (_watcherChanges) + { + _watcherChanges[e.FullPath] = new(e.ChangeType); + } + + Logger.LogTrace("FSW {event}: {path}", e.ChangeType, e.FullPath); + + _ = PenumbraWatcherExecution(); + } + + private void Fs_Renamed(object sender, RenamedEventArgs e) + { + if (Directory.Exists(e.FullPath)) + { + var directoryFiles = Directory.GetFiles(e.FullPath, "*.*", SearchOption.AllDirectories); + lock (_watcherChanges) + { + foreach (var file in directoryFiles) + { + if (!_allowedExtensions.Any(ext => file.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) continue; + var oldPath = file.Replace(e.FullPath, e.OldFullPath, StringComparison.OrdinalIgnoreCase); + + _watcherChanges.Remove(oldPath); + _watcherChanges[file] = new(WatcherChangeTypes.Renamed, oldPath); + Logger.LogTrace("FSW Renamed: {path} -> {new}", oldPath, file); + + } + } + } + else + { + if (!_allowedExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return; + + lock (_watcherChanges) + { + _watcherChanges.Remove(e.OldFullPath); + _watcherChanges[e.FullPath] = new(WatcherChangeTypes.Renamed, e.OldFullPath); + } + + Logger.LogTrace("FSW Renamed: {path} -> {new}", e.OldFullPath, e.FullPath); + } + + _ = PenumbraWatcherExecution(); + } + + private CancellationTokenSource _penumbraFswCts = new(); + private CancellationTokenSource _mareFswCts = new(); + public FileSystemWatcher? PenumbraWatcher { get; private set; } + public FileSystemWatcher? MareWatcher { get; private set; } + + private async Task MareWatcherExecution() + { + _mareFswCts = _mareFswCts.CancelRecreate(); + var token = _mareFswCts.Token; + var delay = TimeSpan.FromSeconds(5); + Dictionary changes; + lock (_mareChanges) + changes = _mareChanges.ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal); + try + { + do + { + await Task.Delay(delay, token).ConfigureAwait(false); + } while (HaltScanLocks.Any(f => f.Value > 0)); + } + catch (TaskCanceledException) + { + return; + } + + lock (_mareChanges) + { + foreach (var key in changes.Keys) + { + _mareChanges.Remove(key); + } + } + + _ = RecalculateFileCacheSize(); + + if (changes.Any(c => c.Value.ChangeType == WatcherChangeTypes.Deleted)) + { + var threadCount = Math.Clamp((int)(Environment.ProcessorCount / 2.0f), 2, 8); + + Parallel.ForEach(changes, new ParallelOptions() + { + MaxDegreeOfParallelism = threadCount, + }, + (change) => + { + Logger.LogDebug("FSW Change: {change} = {val}", change.Key, change.Value); + _ = _fileDbManager.GetFileCacheByPath(change.Key); + }); + + _fileDbManager.WriteOutFullCsv(); } } - public void InvokeScan(bool forced = false) + private async Task PenumbraWatcherExecution() + { + _penumbraFswCts = _penumbraFswCts.CancelRecreate(); + var token = _penumbraFswCts.Token; + Dictionary changes; + lock (_watcherChanges) + changes = _watcherChanges.ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal); + var delay = TimeSpan.FromSeconds(10); + try + { + do + { + await Task.Delay(delay, token).ConfigureAwait(false); + } while (HaltScanLocks.Any(f => f.Value > 0)); + } + catch (TaskCanceledException) + { + return; + } + + lock (_watcherChanges) + { + foreach (var key in changes.Keys) + { + _watcherChanges.Remove(key); + } + } + + var threadCount = Math.Clamp((int)(Environment.ProcessorCount / 2.0f), 2, 8); + + Parallel.ForEach(changes, new ParallelOptions() + { + MaxDegreeOfParallelism = threadCount, + }, + (change) => + { + Logger.LogDebug("FSW Change: {change} = {val}", change.Key, change.Value); + if (change.Value.ChangeType == WatcherChangeTypes.Deleted) + { + _fileDbManager.GetFileCacheByPath(change.Key); + } + else + { + if (change.Value.OldPath != null) _fileDbManager.GetFileCacheByPath(change.Value.OldPath); + _fileDbManager.CreateFileEntry(change.Key); + } + }); + + _fileDbManager.WriteOutFullCsv(); + } + + public void InvokeScan() { - bool isForced = forced; - bool isForcedFromExternal = forced; TotalFiles = 0; _currentFileProgress = 0; _scanCancellationTokenSource?.Cancel(); @@ -69,56 +318,36 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase var token = _scanCancellationTokenSource.Token; _ = Task.Run(async () => { - while (!token.IsCancellationRequested) + TotalFiles = 0; + _currentFileProgress = 0; + while (_dalamudUtil.IsOnFrameworkThread) { - while (HaltScanLocks.Any(f => f.Value > 0) || !_ipcManager.CheckPenumbraApi() || _dalamudUtil.IsOnFrameworkThread) - { - await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); - } - - isForced |= RecalculateFileCacheSize(); - if (!_configService.Current.FileScanPaused || isForced) - { - isForced = false; - TotalFiles = 0; - _currentFileProgress = 0; - while (_dalamudUtil.IsOnFrameworkThread) - { - Logger.LogWarning("Scanner is on framework, waiting for leaving thread before continuing"); - await Task.Delay(250, token).ConfigureAwait(false); - } - - Thread scanThread = new(() => - { - try - { - _performanceCollector.LogPerformance(this, "PeriodicFileScan", () => PeriodicFileScan(isForcedFromExternal, token)); - } - catch (Exception ex) - { - Logger.LogError(ex, "Error during Periodic File Scan"); - } - }) - { - Priority = ThreadPriority.Lowest, - IsBackground = true - }; - scanThread.Start(); - while (scanThread.IsAlive) - { - await Task.Delay(250).ConfigureAwait(false); - } - if (isForcedFromExternal) isForcedFromExternal = false; - TotalFiles = 0; - _currentFileProgress = 0; - } - _timeUntilNextScan = TimeSpan.FromSeconds(TimeBetweenScans); - while (_timeUntilNextScan.TotalSeconds >= 0 || _dalamudUtil.IsOnFrameworkThread) - { - await Task.Delay(TimeSpan.FromSeconds(1), token).ConfigureAwait(false); - _timeUntilNextScan -= TimeSpan.FromSeconds(1); - } + Logger.LogWarning("Scanner is on framework, waiting for leaving thread before continuing"); + await Task.Delay(250, token).ConfigureAwait(false); } + + Thread scanThread = new(() => + { + try + { + _performanceCollector.LogPerformance(this, "FullFileScan", () => FullFileScan(token)); + } + catch (Exception ex) + { + Logger.LogError(ex, "Error during Full File Scan"); + } + }) + { + Priority = ThreadPriority.Lowest, + IsBackground = true + }; + scanThread.Start(); + while (scanThread.IsAlive) + { + await Task.Delay(250).ConfigureAwait(false); + } + TotalFiles = 0; + _currentFileProgress = 0; }, token); } @@ -165,28 +394,19 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase HaltScanLocks[source]--; if (HaltScanLocks[source] < 0) HaltScanLocks[source] = 0; - - if (_fileScanWasRunning && HaltScanLocks.All(f => f.Value == 0)) - { - _fileScanWasRunning = false; - InvokeScan(forced: true); - } - } - - public void StartScan() - { - if (!_ipcManager.Initialized || !_configService.Current.HasValidSetup()) return; - Logger.LogTrace("Penumbra is active, configuration is valid, scan"); - InvokeScan(forced: true); } protected override void Dispose(bool disposing) { base.Dispose(disposing); _scanCancellationTokenSource?.Cancel(); + PenumbraWatcher?.Dispose(); + MareWatcher?.Dispose(); + _penumbraFswCts?.CancelDispose(); + _mareFswCts?.CancelDispose(); } - private void PeriodicFileScan(bool noWaiting, CancellationToken ct) + private void FullFileScan(CancellationToken ct) { TotalFiles = 1; var penumbraDir = _ipcManager.PenumbraModDirectory; @@ -210,7 +430,6 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase var previousThreadPriority = Thread.CurrentThread.Priority; Thread.CurrentThread.Priority = ThreadPriority.Lowest; Logger.LogDebug("Getting files from {penumbra} and {storage}", penumbraDir, _configService.Current.CacheFolder); - string[] ext = [".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".pbd", ".scd", ".skp", ".shpk"]; Dictionary penumbraFiles = new(StringComparer.Ordinal); foreach (var folder in Directory.EnumerateDirectories(penumbraDir!)) @@ -221,7 +440,7 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase [ .. Directory.GetFiles(folder, "*.*", SearchOption.AllDirectories) .AsParallel() - .Where(f => ext.Any(e => f.EndsWith(e, StringComparison.OrdinalIgnoreCase)) + .Where(f => _allowedExtensions.Any(e => f.EndsWith(e, StringComparison.OrdinalIgnoreCase)) && !f.Contains(@"\bg\", StringComparison.OrdinalIgnoreCase) && !f.Contains(@"\bgcommon\", StringComparison.OrdinalIgnoreCase) && !f.Contains(@"\ui\", StringComparison.OrdinalIgnoreCase)), @@ -309,7 +528,6 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase Logger.LogWarning(ex, "Failed validating {path}", workload.ResolvedFilepath); } Interlocked.Increment(ref _currentFileProgress); - if (!noWaiting) Thread.Sleep(5); } Logger.LogTrace("Ending Worker Thread {i}", threadNr); @@ -390,7 +608,6 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase } Interlocked.Increment(ref _currentFileProgress); - if (!noWaiting) Thread.Sleep(5); }); Logger.LogTrace("Scanner added {notScanned} new files to db", allScannedFiles.Count(c => !c.Value)); @@ -406,6 +623,8 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase { _configService.Current.InitialScanComplete = true; _configService.Save(); + StartMareWatcher(_configService.Current.CacheFolder); + StartPenumbraWatcher(penumbraDir); } } } \ No newline at end of file diff --git a/MareSynchronos/FileCache/FileCacheManager.cs b/MareSynchronos/FileCache/FileCacheManager.cs index 306493b..0f0e162 100644 --- a/MareSynchronos/FileCache/FileCacheManager.cs +++ b/MareSynchronos/FileCache/FileCacheManager.cs @@ -3,6 +3,7 @@ using MareSynchronos.Interop; using MareSynchronos.MareConfiguration; using MareSynchronos.Services.Mediator; using MareSynchronos.Utils; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Globalization; @@ -10,7 +11,7 @@ using System.Text; namespace MareSynchronos.FileCache; -public sealed class FileCacheManager : IDisposable +public sealed class FileCacheManager : IHostedService { public const string CachePrefix = "{cache}"; public const string CsvSplit = "|"; @@ -30,101 +31,6 @@ public sealed class FileCacheManager : IDisposable _configService = configService; _mareMediator = mareMediator; _csvPath = Path.Combine(configService.ConfigurationDirectory, "FileCache.csv"); - - lock (_fileWriteLock) - { - try - { - if (File.Exists(CsvBakPath)) - { - File.Move(CsvBakPath, _csvPath, overwrite: true); - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to move BAK to ORG, deleting BAK"); - try - { - if (File.Exists(CsvBakPath)) - File.Delete(CsvBakPath); - } - catch (Exception ex1) - { - _logger.LogWarning(ex1, "Could not delete bak file"); - } - } - } - - if (File.Exists(_csvPath)) - { - bool success = false; - string[] entries = []; - int attempts = 0; - while (!success && attempts < 10) - { - try - { - entries = File.ReadAllLines(_csvPath); - success = true; - } - catch (Exception ex) - { - attempts++; - _logger.LogWarning(ex, "Could not open {file}, trying again", _csvPath); - Thread.Sleep(100); - } - } - - if (!entries.Any()) - { - _logger.LogWarning("Could not load entries from {path}, continuing with empty file cache", _csvPath); - } - - Dictionary processedFiles = new(StringComparer.OrdinalIgnoreCase); - foreach (var entry in entries) - { - var splittedEntry = entry.Split(CsvSplit, StringSplitOptions.None); - try - { - var hash = splittedEntry[0]; - if (hash.Length != 40) throw new InvalidOperationException("Expected Hash length of 40, received " + hash.Length); - var path = splittedEntry[1]; - var time = splittedEntry[2]; - - if (processedFiles.ContainsKey(path)) - { - _logger.LogWarning("Already processed {file}, ignoring", path); - continue; - } - - processedFiles.Add(path, value: true); - - long size = -1; - long compressed = -1; - if (splittedEntry.Length > 3) - { - if (long.TryParse(splittedEntry[3], CultureInfo.InvariantCulture, out long result)) - { - size = result; - } - if (long.TryParse(splittedEntry[4], CultureInfo.InvariantCulture, out long resultCompressed)) - { - compressed = resultCompressed; - } - } - AddHashedFile(ReplacePathPrefixes(new FileCacheEntity(hash, path, time, size, compressed))); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to initialize entry {entry}, ignoring", entry); - } - } - - if (processedFiles.Count != entries.Length) - { - WriteOutFullCsv(); - } - } } private string CsvBakPath => _csvPath + ".bak"; @@ -151,13 +57,6 @@ public sealed class FileCacheManager : IDisposable return CreateFileCacheEntity(fi, prefixedPath); } - public void Dispose() - { - _logger.LogTrace("Disposing {type}", GetType()); - WriteOutFullCsv(); - GC.SuppressFinalize(this); - } - public List GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v).ToList(); public List GetAllFileCachesByHash(string hash) @@ -346,13 +245,14 @@ public sealed class FileCacheManager : IDisposable public void WriteOutFullCsv() { - StringBuilder sb = new(); - foreach (var entry in _fileCaches.SelectMany(k => k.Value).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase)) - { - sb.AppendLine(entry.CsvEntry); - } lock (_fileWriteLock) { + StringBuilder sb = new(); + foreach (var entry in _fileCaches.SelectMany(k => k.Value).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase)) + { + sb.AppendLine(entry.CsvEntry); + } + if (File.Exists(_csvPath)) { File.Copy(_csvPath, CsvBakPath, overwrite: true); @@ -458,4 +358,110 @@ public sealed class FileCacheManager : IDisposable return fileCache; } + + public Task StartAsync(CancellationToken cancellationToken) + { + lock (_fileWriteLock) + { + try + { + if (File.Exists(CsvBakPath)) + { + File.Move(CsvBakPath, _csvPath, overwrite: true); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to move BAK to ORG, deleting BAK"); + try + { + if (File.Exists(CsvBakPath)) + File.Delete(CsvBakPath); + } + catch (Exception ex1) + { + _logger.LogWarning(ex1, "Could not delete bak file"); + } + } + } + + if (File.Exists(_csvPath)) + { + bool success = false; + string[] entries = []; + int attempts = 0; + while (!success && attempts < 10) + { + try + { + entries = File.ReadAllLines(_csvPath); + success = true; + } + catch (Exception ex) + { + attempts++; + _logger.LogWarning(ex, "Could not open {file}, trying again", _csvPath); + Thread.Sleep(100); + } + } + + if (!entries.Any()) + { + _logger.LogWarning("Could not load entries from {path}, continuing with empty file cache", _csvPath); + } + + Dictionary processedFiles = new(StringComparer.OrdinalIgnoreCase); + foreach (var entry in entries) + { + var splittedEntry = entry.Split(CsvSplit, StringSplitOptions.None); + try + { + var hash = splittedEntry[0]; + if (hash.Length != 40) throw new InvalidOperationException("Expected Hash length of 40, received " + hash.Length); + var path = splittedEntry[1]; + var time = splittedEntry[2]; + + if (processedFiles.ContainsKey(path)) + { + _logger.LogWarning("Already processed {file}, ignoring", path); + continue; + } + + processedFiles.Add(path, value: true); + + long size = -1; + long compressed = -1; + if (splittedEntry.Length > 3) + { + if (long.TryParse(splittedEntry[3], CultureInfo.InvariantCulture, out long result)) + { + size = result; + } + if (long.TryParse(splittedEntry[4], CultureInfo.InvariantCulture, out long resultCompressed)) + { + compressed = resultCompressed; + } + } + AddHashedFile(ReplacePathPrefixes(new FileCacheEntity(hash, path, time, size, compressed))); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to initialize entry {entry}, ignoring", entry); + } + } + + if (processedFiles.Count != entries.Length) + { + WriteOutFullCsv(); + } + } + + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + WriteOutFullCsv(); + return Task.CompletedTask; + } } \ No newline at end of file diff --git a/MareSynchronos/Interop/IpcManager.cs b/MareSynchronos/Interop/IpcManager.cs index b284949..c15d4a6 100644 --- a/MareSynchronos/Interop/IpcManager.cs +++ b/MareSynchronos/Interop/IpcManager.cs @@ -170,7 +170,19 @@ public sealed class IpcManager : DisposableMediatorSubscriberBase } public bool Initialized => CheckPenumbraApiInternal() && CheckGlamourerApiInternal(); - public string? PenumbraModDirectory { get; private set; } + private string? _penumbraModDirectory; + public string? PenumbraModDirectory + { + get => _penumbraModDirectory; + private set + { + if (!string.Equals(_penumbraModDirectory, value, StringComparison.Ordinal)) + { + _penumbraModDirectory = value; + Mediator.Publish(new PenumbraDirectoryChangedMessage(_penumbraModDirectory)); + } + } + } public bool CheckCustomizePlusApi() => _customizePlusAvailable; diff --git a/MareSynchronos/MarePlugin.cs b/MareSynchronos/MarePlugin.cs index 8af332a..90e8c0e 100644 --- a/MareSynchronos/MarePlugin.cs +++ b/MareSynchronos/MarePlugin.cs @@ -142,7 +142,6 @@ public class MarePlugin : MediatorSubscriberBase, IHostedService Mediator.Publish(new SwitchToIntroUiMessage()); return; } - _runtimeServiceScope.ServiceProvider.GetRequiredService().StartScan(); _runtimeServiceScope.ServiceProvider.GetRequiredService(); _runtimeServiceScope.ServiceProvider.GetRequiredService(); _runtimeServiceScope.ServiceProvider.GetRequiredService(); diff --git a/MareSynchronos/Plugin.cs b/MareSynchronos/Plugin.cs index efca1b9..3ad35cd 100644 --- a/MareSynchronos/Plugin.cs +++ b/MareSynchronos/Plugin.cs @@ -109,7 +109,7 @@ public sealed class Plugin : IDalamudPlugin collection.AddSingleton(); // add scoped services - collection.AddScoped(); + collection.AddScoped(); collection.AddScoped(); collection.AddScoped(); collection.AddScoped(); @@ -134,12 +134,12 @@ public sealed class Plugin : IDalamudPlugin s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService())); collection.AddScoped((s) => new CommandManagerService(commandManager, s.GetRequiredService(), - s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), + s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService())); collection.AddScoped((s) => new NotificationService(s.GetRequiredService>(), s.GetRequiredService(), notificationManager, chatGui, s.GetRequiredService())); collection.AddScoped((s) => new UiSharedService(s.GetRequiredService>(), s.GetRequiredService(), s.GetRequiredService(), - s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), + s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), pluginInterface, textureProvider, s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService())); collection.AddScoped((s) => new ChatService(s.GetRequiredService>(), s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), @@ -152,6 +152,7 @@ public sealed class Plugin : IDalamudPlugin collection.AddHostedService(p => p.GetRequiredService()); collection.AddHostedService(p => p.GetRequiredService()); collection.AddHostedService(p => p.GetRequiredService()); + collection.AddHostedService(p => p.GetRequiredService()); }) .Build() .RunAsync(_pluginCts.Token); diff --git a/MareSynchronos/Services/CommandManagerService.cs b/MareSynchronos/Services/CommandManagerService.cs index 3004330..7c8dc65 100644 --- a/MareSynchronos/Services/CommandManagerService.cs +++ b/MareSynchronos/Services/CommandManagerService.cs @@ -25,18 +25,18 @@ public sealed class CommandManagerService : IDisposable private readonly MareMediator _mediator; private readonly MareConfigService _mareConfigService; private readonly PerformanceCollectorService _performanceCollectorService; - private readonly PeriodicFileScanner _periodicFileScanner; + private readonly CacheMonitor _cacheMonitor; private readonly ChatService _chatService; private readonly ServerConfigurationManager _serverConfigurationManager; public CommandManagerService(ICommandManager commandManager, PerformanceCollectorService performanceCollectorService, - ServerConfigurationManager serverConfigurationManager, PeriodicFileScanner periodicFileScanner, ChatService chatService, + ServerConfigurationManager serverConfigurationManager, CacheMonitor periodicFileScanner, ChatService chatService, ApiController apiController, MareMediator mediator, MareConfigService mareConfigService) { _commandManager = commandManager; _performanceCollectorService = performanceCollectorService; _serverConfigurationManager = serverConfigurationManager; - _periodicFileScanner = periodicFileScanner; + _cacheMonitor = periodicFileScanner; _chatService = chatService; _apiController = apiController; _mediator = mediator; @@ -112,7 +112,7 @@ public sealed class CommandManagerService : IDisposable } else if (string.Equals(splitArgs[0], "rescan", StringComparison.OrdinalIgnoreCase)) { - _periodicFileScanner.InvokeScan(forced: true); + _cacheMonitor.InvokeScan(); } else if (string.Equals(splitArgs[0], "perf", StringComparison.OrdinalIgnoreCase)) { diff --git a/MareSynchronos/Services/Mediator/Messages.cs b/MareSynchronos/Services/Mediator/Messages.cs index 28c7caa..520995c 100644 --- a/MareSynchronos/Services/Mediator/Messages.cs +++ b/MareSynchronos/Services/Mediator/Messages.cs @@ -78,6 +78,7 @@ public record CensusUpdateMessage(byte Gender, byte RaceId, byte TribeId) : Mess public record TargetPairMessage(Pair Pair) : MessageBase; public record CombatStartMessage : MessageBase; public record CombatEndMessage : MessageBase; +public record PenumbraDirectoryChangedMessage(string? ModDirectory) : MessageBase; public record UserChatMsgMessage(SignedChatMessage ChatMsg) : MessageBase; public record GroupChatMsgMessage(GroupDto GroupInfo, SignedChatMessage ChatMsg) : MessageBase; diff --git a/MareSynchronos/UI/IntroUI.cs b/MareSynchronos/UI/IntroUI.cs index 67021ce..32fb7fe 100644 --- a/MareSynchronos/UI/IntroUI.cs +++ b/MareSynchronos/UI/IntroUI.cs @@ -24,7 +24,7 @@ public class IntroUi : WindowMediatorSubscriberBase { private readonly ApiController _apiController; private readonly MareConfigService _configService; - private readonly PeriodicFileScanner _fileCacheManager; + private readonly CacheMonitor _cacheMonitor; private readonly Dictionary _languages = new(StringComparer.Ordinal) { { "English", "en" }, { "Deutsch", "de" }, { "Français", "fr" } }; private readonly ServerConfigurationManager _serverConfigurationManager; private readonly UiSharedService _uiShared; @@ -41,12 +41,12 @@ public class IntroUi : WindowMediatorSubscriberBase private RegisterReplyDto? _registrationReply; public IntroUi(ILogger logger, UiSharedService uiShared, MareConfigService configService, ApiController apiController, - PeriodicFileScanner fileCacheManager, ServerConfigurationManager serverConfigurationManager, MareMediator mareMediator) : base(logger, mareMediator, "Loporrit Setup") + CacheMonitor fileCacheManager, ServerConfigurationManager serverConfigurationManager, MareMediator mareMediator) : base(logger, mareMediator, "Loporrit Setup") { _uiShared = uiShared; _configService = configService; _apiController = apiController; - _fileCacheManager = fileCacheManager; + _cacheMonitor = fileCacheManager; _serverConfigurationManager = serverConfigurationManager; IsOpen = false; @@ -176,11 +176,11 @@ public class IntroUi : WindowMediatorSubscriberBase _uiShared.DrawCacheDirectorySetting(); } - if (!_fileCacheManager.IsScanRunning && !string.IsNullOrEmpty(_configService.Current.CacheFolder) && _uiShared.HasValidPenumbraModPath && Directory.Exists(_configService.Current.CacheFolder)) + if (!_cacheMonitor.IsScanRunning && !string.IsNullOrEmpty(_configService.Current.CacheFolder) && _uiShared.HasValidPenumbraModPath && Directory.Exists(_configService.Current.CacheFolder)) { if (ImGui.Button("Start Scan##startScan")) { - _fileCacheManager.InvokeScan(forced: true); + _cacheMonitor.InvokeScan(); } } else diff --git a/MareSynchronos/UI/SettingsUi.cs b/MareSynchronos/UI/SettingsUi.cs index 5cf59a1..be6e947 100644 --- a/MareSynchronos/UI/SettingsUi.cs +++ b/MareSynchronos/UI/SettingsUi.cs @@ -10,6 +10,7 @@ using MareSynchronos.API.Data.Comparer; using MareSynchronos.API.Dto.Account; using MareSynchronos.API.Routes; using MareSynchronos.FileCache; +using MareSynchronos.Interop; using MareSynchronos.MareConfiguration; using MareSynchronos.MareConfiguration.Models; using MareSynchronos.PlayerData.Export; @@ -36,6 +37,8 @@ namespace MareSynchronos.UI; public class SettingsUi : WindowMediatorSubscriberBase { private readonly ApiController _apiController; + private readonly IpcManager _ipcManager; + private readonly CacheMonitor _cacheMonitor; private readonly MareConfigService _configService; private readonly ConcurrentDictionary> _currentDownloads = new(); private readonly FileCompactor _fileCompactor; @@ -77,7 +80,8 @@ public class SettingsUi : WindowMediatorSubscriberBase FileUploadManager fileTransferManager, FileTransferOrchestrator fileTransferOrchestrator, FileCacheManager fileCacheManager, - FileCompactor fileCompactor, ApiController apiController) : base(logger, mediator, "Loporrit Settings") + FileCompactor fileCompactor, ApiController apiController, + IpcManager ipcManager, CacheMonitor cacheMonitor) : base(logger, mediator, "Loporrit Settings") { _configService = configService; _mareCharaFileManager = mareCharaFileManager; @@ -90,6 +94,8 @@ public class SettingsUi : WindowMediatorSubscriberBase _fileTransferOrchestrator = fileTransferOrchestrator; _fileCacheManager = fileCacheManager; _apiController = apiController; + _ipcManager = ipcManager; + _cacheMonitor = cacheMonitor; _fileCompactor = fileCompactor; _uiShared = uiShared; AllowClickthrough = false; @@ -727,7 +733,57 @@ public class SettingsUi : WindowMediatorSubscriberBase "The storage governs itself by clearing data beyond the set storage size. Please set the storage size accordingly. It is not necessary to manually clear the storage."); _uiShared.DrawFileScanState(); - _uiShared.DrawTimeSpanBetweenScansSetting(); + ImGui.AlignTextToFramePadding(); + ImGui.TextUnformatted("Monitoring Penumbra Folder: " + (_cacheMonitor.PenumbraWatcher?.Path ?? "Not monitoring")); + if (string.IsNullOrEmpty(_cacheMonitor.PenumbraWatcher?.Path)) + { + ImGui.SameLine(); + using var id = ImRaii.PushId("penumbraMonitor"); + if (UiSharedService.NormalizedIconTextButton(FontAwesomeIcon.ArrowsToCircle, "Try to reinitialize Monitor")) + { + _cacheMonitor.StartPenumbraWatcher(_ipcManager.PenumbraModDirectory); + } + } + + ImGui.AlignTextToFramePadding(); + ImGui.TextUnformatted("Monitoring Mare Storage Folder: " + (_cacheMonitor.MareWatcher?.Path ?? "Not monitoring")); + if (string.IsNullOrEmpty(_cacheMonitor.MareWatcher?.Path)) + { + ImGui.SameLine(); + using var id = ImRaii.PushId("mareMonitor"); + if (UiSharedService.NormalizedIconTextButton(FontAwesomeIcon.ArrowsToCircle, "Try to reinitialize Monitor")) + { + _cacheMonitor.StartMareWatcher(_configService.Current.CacheFolder); + } + } + if (_cacheMonitor.MareWatcher == null || _cacheMonitor.PenumbraWatcher == null) + { + if (UiSharedService.NormalizedIconTextButton(FontAwesomeIcon.Play, "Resume Monitoring")) + { + _cacheMonitor.StartMareWatcher(_configService.Current.CacheFolder); + _cacheMonitor.StartPenumbraWatcher(_ipcManager.PenumbraModDirectory); + _cacheMonitor.InvokeScan(); + } + UiSharedService.AttachToolTip("Attempts to resume monitoring for both Penumbra and Mare Storage. " + + "Resuming the monitoring will also force a full scan to run." + Environment.NewLine + + "If the button remains present after clicking it, consult /xllog for errors"); + } + else + { + using (ImRaii.Disabled(!UiSharedService.CtrlPressed())) + { + if (UiSharedService.NormalizedIconTextButton(FontAwesomeIcon.Stop, "Stop Monitoring")) + { + _cacheMonitor.StopMonitoring(); + } + } + UiSharedService.AttachToolTip("Stops the monitoring for both Penumbra and Mare Storage. " + + "Do not stop the monitoring, unless you plan to move the Penumbra and Mare Storage folders, to ensure correct functionality of Mare." + Environment.NewLine + + "If you stop the monitoring to move folders around, resume it after you are finished moving the files." + + UiSharedService.TooltipSeparator + "Hold CTRL to enable this button"); + } + + _uiShared.DrawCacheDirectorySetting(); ImGui.TextUnformatted($"Currently utilized local storage: {UiSharedService.ByteToString(_uiShared.FileCacheSize)}"); bool isLinux = Util.IsWine(); @@ -827,8 +883,6 @@ public class SettingsUi : WindowMediatorSubscriberBase { File.Delete(file); } - - _uiShared.RecalculateFileCacheSize(); }); } UiSharedService.AttachToolTip("You normally do not need to do this. THIS IS NOT SOMETHING YOU SHOULD BE DOING TO TRY TO FIX SYNC ISSUES." + Environment.NewLine diff --git a/MareSynchronos/UI/UISharedService.cs b/MareSynchronos/UI/UISharedService.cs index 5279e83..636d299 100644 --- a/MareSynchronos/UI/UISharedService.cs +++ b/MareSynchronos/UI/UISharedService.cs @@ -43,7 +43,7 @@ public partial class UiSharedService : DisposableMediatorSubscriberBase private readonly ApiController _apiController; - private readonly PeriodicFileScanner _cacheScanner; + private readonly CacheMonitor _cacheMonitor; private readonly MareConfigService _configService; @@ -79,14 +79,14 @@ public partial class UiSharedService : DisposableMediatorSubscriberBase private int _serverSelectionIndex = -1; public UiSharedService(ILogger logger, IpcManager ipcManager, ApiController apiController, - PeriodicFileScanner cacheScanner, FileDialogManager fileDialogManager, + CacheMonitor cacheMonitor, FileDialogManager fileDialogManager, MareConfigService configService, DalamudUtilService dalamudUtil, IDalamudPluginInterface pluginInterface, ITextureProvider textureProvider, Dalamud.Localization localization, ServerConfigurationManager serverManager, MareMediator mediator) : base(logger, mediator) { _ipcManager = ipcManager; _apiController = apiController; - _cacheScanner = cacheScanner; + _cacheMonitor = cacheMonitor; FileDialogManager = fileDialogManager; _configService = configService; _dalamudUtil = dalamudUtil; @@ -123,7 +123,7 @@ public partial class UiSharedService : DisposableMediatorSubscriberBase public bool EditTrackerPosition { get; set; } - public long FileCacheSize => _cacheScanner.FileCacheSize; + public long FileCacheSize => _cacheMonitor.FileCacheSize; public bool HasValidPenumbraModPath => !(_ipcManager.PenumbraModDirectory ?? string.Empty).IsNullOrEmpty() && Directory.Exists(_ipcManager.PenumbraModDirectory); @@ -618,7 +618,8 @@ public partial class UiSharedService : DisposableMediatorSubscriberBase { _configService.Current.CacheFolder = path; _configService.Save(); - _cacheScanner.StartScan(); + _cacheMonitor.StartMareWatcher(path); + _cacheMonitor.InvokeScan(); } }); } @@ -742,41 +743,45 @@ public partial class UiSharedService : DisposableMediatorSubscriberBase public void DrawFileScanState() { + ImGui.AlignTextToFramePadding(); ImGui.TextUnformatted("File Scanner Status"); ImGui.SameLine(); - if (_cacheScanner.IsScanRunning) + if (_cacheMonitor.IsScanRunning) { + ImGui.AlignTextToFramePadding(); + ImGui.TextUnformatted("Scan is running"); ImGui.TextUnformatted("Current Progress:"); ImGui.SameLine(); - ImGui.TextUnformatted(_cacheScanner.TotalFiles == 1 + ImGui.TextUnformatted(_cacheMonitor.TotalFiles == 1 ? "Collecting files" - : $"Processing {_cacheScanner.CurrentFileProgress}/{_cacheScanner.TotalFilesStorage} from storage ({_cacheScanner.TotalFiles} scanned in)"); + : $"Processing {_cacheMonitor.CurrentFileProgress}/{_cacheMonitor.TotalFilesStorage} from storage ({_cacheMonitor.TotalFiles} scanned in)"); AttachToolTip("Note: it is possible to have more files in storage than scanned in, " + "this is due to the scanner normally ignoring those files but the game loading them in and using them on your character, so they get " + "added to the local storage."); } - else if (_configService.Current.FileScanPaused) + else if (_cacheMonitor.HaltScanLocks.Any(f => f.Value > 0)) { - ImGui.TextUnformatted("File scanner is paused"); - ImGui.SameLine(); - if (ImGui.Button("Force Rescan##forcedrescan")) - { - _cacheScanner.InvokeScan(forced: true); - } - } - else if (_cacheScanner.HaltScanLocks.Any(f => f.Value > 0)) - { - ImGui.TextUnformatted("Halted (" + string.Join(", ", _cacheScanner.HaltScanLocks.Where(f => f.Value > 0).Select(locker => locker.Key + ": " + locker.Value + " halt requests")) + ")"); + ImGui.AlignTextToFramePadding(); + + ImGui.TextUnformatted("Halted (" + string.Join(", ", _cacheMonitor.HaltScanLocks.Where(f => f.Value > 0).Select(locker => locker.Key + ": " + locker.Value + " halt requests")) + ")"); ImGui.SameLine(); if (ImGui.Button("Reset halt requests##clearlocks")) { - _cacheScanner.ResetLocks(); + _cacheMonitor.ResetLocks(); } } else { - ImGui.TextUnformatted("Next scan in " + _cacheScanner.TimeUntilNextScan); + ImGui.TextUnformatted("Idle"); + if (_configService.Current.InitialScanComplete) + { + ImGui.SameLine(); + if (NormalizedIconTextButton(FontAwesomeIcon.Play, "Force rescan")) + { + _cacheMonitor.InvokeScan(); + } + } } } @@ -940,36 +945,12 @@ public partial class UiSharedService : DisposableMediatorSubscriberBase return _serverSelectionIndex; } - public void DrawTimeSpanBetweenScansSetting() - { - var timeSpan = _configService.Current.TimeSpanBetweenScansInSeconds; - ImGui.SetNextItemWidth(250 * ImGuiHelpers.GlobalScale); - if (ImGui.SliderInt("Seconds between scans##timespan", ref timeSpan, 20, 60)) - { - _configService.Current.TimeSpanBetweenScansInSeconds = timeSpan; - _configService.Save(); - } - DrawHelpText("This is the time in seconds between file scans. Increase it to reduce system load. A too high setting can cause issues when manually fumbling about in the cache or Penumbra mods folders."); - var isPaused = _configService.Current.FileScanPaused; - if (ImGui.Checkbox("Pause periodic file scan##filescanpause", ref isPaused)) - { - _configService.Current.FileScanPaused = isPaused; - _configService.Save(); - } - DrawHelpText("This allows you to stop the periodic scans of your Penumbra and Loporrit cache directories. Use this to move the Loporrit cache and Penumbra mod folders around. If you enable this permanently, run a Force rescan after adding mods to Penumbra."); - } - public void LoadLocalization(string languageCode) { _localization.SetupWithLangCode(languageCode); Strings.ToS = new Strings.ToSStrings(); } - public void RecalculateFileCacheSize() - { - _cacheScanner.InvokeScan(forced: true); - } - [LibraryImport("user32")] internal static partial short GetKeyState(int nVirtKey); diff --git a/MareSynchronos/Utils/Crypto.cs b/MareSynchronos/Utils/Crypto.cs index 97389f1..d029a83 100644 --- a/MareSynchronos/Utils/Crypto.cs +++ b/MareSynchronos/Utils/Crypto.cs @@ -24,8 +24,5 @@ public static class Crypto { return BitConverter.ToString(_sha256CryptoProvider.ComputeHash(Encoding.UTF8.GetBytes(stringToCompute))).Replace("-", "", StringComparison.Ordinal); } - - - #pragma warning restore SYSLIB0021 // Type or member is obsolete } \ No newline at end of file