From 287c1b0eff47ec824d9f470227434a7f01176a9b Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Wed, 28 Sep 2022 00:16:14 +0200 Subject: [PATCH] stop scan on character data creation, downloads, gpose, zone switch, update outside of read loop, adjust verify --- .../Factories/CharacterDataFactory.cs | 1 + MareSynchronos/FileCacheDB/FileDbManager.cs | 47 ++++++++---- .../FileCacheDB/PeriodicFileScanner.cs | 75 ++++++++++++++----- MareSynchronos/Managers/PlayerManager.cs | 8 +- MareSynchronos/MareSynchronos.csproj | 2 +- MareSynchronos/Plugin.cs | 14 ++-- MareSynchronos/Utils/DalamudUtil.cs | 4 +- 7 files changed, 107 insertions(+), 44 deletions(-) diff --git a/MareSynchronos/Factories/CharacterDataFactory.cs b/MareSynchronos/Factories/CharacterDataFactory.cs index 357ca4f..c1b66d7 100644 --- a/MareSynchronos/Factories/CharacterDataFactory.cs +++ b/MareSynchronos/Factories/CharacterDataFactory.cs @@ -8,6 +8,7 @@ using FFXIVClientStructs.FFXIV.Client.Game.Character; using FFXIVClientStructs.FFXIV.Client.Graphics.Scene; using FFXIVClientStructs.FFXIV.Client.System.Resource; using MareSynchronos.API; +using MareSynchronos.FileCacheDB; using MareSynchronos.Interop; using MareSynchronos.Managers; using MareSynchronos.Models; diff --git a/MareSynchronos/FileCacheDB/FileDbManager.cs b/MareSynchronos/FileCacheDB/FileDbManager.cs index 426c1ae..a54d516 100644 --- a/MareSynchronos/FileCacheDB/FileDbManager.cs +++ b/MareSynchronos/FileCacheDB/FileDbManager.cs @@ -8,6 +8,13 @@ using System.Linq; namespace MareSynchronos.Managers; +public enum FileState +{ + Valid, + RequireUpdate, + RequireDeletion +} + public class FileDbManager { private const string PenumbraPrefix = "{penumbra}"; @@ -45,9 +52,19 @@ public class FileDbManager return GetValidatedFileCache(new FileCache(matchingEntries.First())); } - public FileCache? ValidateFileCacheEntity(string hash, string path, string lastModifiedDate) + public (FileState, string) ValidateFileCacheEntity(string hash, string path, string lastModifiedDate) { - return GetValidatedFileCache(new FileCache(hash, path, lastModifiedDate), false); + var fileCache = new FileCache(hash, path, lastModifiedDate); + if (!fileCache.OriginalFilepath.StartsWith(PenumbraPrefix + "\\") && !fileCache.OriginalFilepath.StartsWith(CachePrefix)) + return (FileState.RequireUpdate, path); + fileCache = ReplacePathPrefixes(fileCache); + FileInfo fi = new FileInfo(fileCache.Filepath); + if (!fi.Exists) + return (FileState.RequireDeletion, fileCache.Filepath); + if (fi.LastWriteTimeUtc.Ticks != fileCache.LastModifiedDateTicks) + return (FileState.RequireUpdate, fileCache.Filepath); + + return (FileState.Valid, fileCache.Filepath); } public FileCache? GetFileCacheByPath(string path) @@ -61,6 +78,7 @@ public class FileDbManager if (matchingEntries == null) { + Logger.Debug("Found no entries for " + cleanedPath); return CreateFileEntry(path); } @@ -104,7 +122,8 @@ public class FileDbManager } catch (Exception ex) { - Logger.Warn("Could not add " + fileInfo.FullName); + Logger.Warn("Could not add " + fileInfo.FullName ?? String.Empty); + Logger.Warn(ex.Message); } } var result = GetFileCacheByPath(prefixedPath); @@ -112,25 +131,22 @@ public class FileDbManager return result; } - private FileCache? GetValidatedFileCache(FileCache fileCache, bool removeOnNonExistence = true) + private FileCache? GetValidatedFileCache(FileCache fileCache) { var resulingFileCache = MigrateLegacy(fileCache); if (resulingFileCache == null) return null; resulingFileCache = ReplacePathPrefixes(resulingFileCache); - resulingFileCache = Validate(resulingFileCache, removeOnNonExistence); + resulingFileCache = Validate(resulingFileCache); return resulingFileCache; } - private FileCache? Validate(FileCache fileCache, bool removeOnNonExistence = true) + private FileCache? Validate(FileCache fileCache) { var file = new FileInfo(fileCache.Filepath); if (!file.Exists) { - if (removeOnNonExistence) - { - DeleteFromDatabase(new[] { fileCache }); - } + DeleteFromDatabase(new[] { fileCache }); return null; } @@ -143,7 +159,7 @@ public class FileDbManager return fileCache; } - private FileCache? MigrateLegacy(FileCache fileCache, bool removeOnNonExistence = true) + private FileCache? MigrateLegacy(FileCache fileCache) { if (fileCache.OriginalFilepath.StartsWith(PenumbraPrefix + "\\") || fileCache.OriginalFilepath.StartsWith(CachePrefix)) return fileCache; @@ -168,10 +184,7 @@ public class FileDbManager } else { - if (removeOnNonExistence) - { - DeleteFromDatabase(new[] { fileCache }); - } + DeleteFromDatabase(new[] { fileCache }); return null; } @@ -214,7 +227,9 @@ public class FileDbManager } catch (Exception ex) { - Logger.Warn("Error updating file hash (" + ex.Message + "), returning currently existing"); + Logger.Warn("Error updating file hash (" + ex.Message + "), returning currently existing" ?? string.Empty); + Logger.Warn(ex.InnerException?.Message ?? string.Empty); + Logger.Warn(ex.StackTrace ?? string.Empty); using var db = new FileCacheContext(); var cache = db.FileCaches.First(f => f.Filepath == markedForUpdate.OriginalFilepath); markedForUpdate.UpdateFileCache(cache); diff --git a/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs b/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs index f1976a1..d723ecb 100644 --- a/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs +++ b/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Concurrent; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; @@ -18,9 +17,11 @@ public class PeriodicFileScanner : IDisposable private readonly Configuration _pluginConfiguration; private readonly FileDbManager _fileDbManager; private readonly ApiController _apiController; + private readonly DalamudUtil _dalamudUtil; + private int haltScanRequests = 0; private CancellationTokenSource? _scanCancellationTokenSource; private Task? _fileScannerTask = null; - public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileDbManager fileDbManager, ApiController apiController) + public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileDbManager fileDbManager, ApiController apiController, DalamudUtil dalamudUtil) { Logger.Verbose("Creating " + nameof(PeriodicFileScanner)); @@ -28,27 +29,34 @@ public class PeriodicFileScanner : IDisposable _pluginConfiguration = pluginConfiguration; _fileDbManager = fileDbManager; _apiController = apiController; + _dalamudUtil = dalamudUtil; _ipcManager.PenumbraInitialized += StartScan; if (!string.IsNullOrEmpty(_ipcManager.PenumbraModDirectory())) { StartScan(); } - _apiController.DownloadStarted += _apiController_DownloadStarted; - _apiController.DownloadFinished += _apiController_DownloadFinished; + _apiController.DownloadStarted += HaltScan; + _apiController.DownloadFinished += ResumeScan; + _dalamudUtil.ZoneSwitchStart += HaltScan; + _dalamudUtil.ZoneSwitchEnd += ResumeScan; } - private void _apiController_DownloadFinished() + public void ResumeScan() { - if (fileScanWasRunning) + Interlocked.Decrement(ref haltScanRequests); + + if (fileScanWasRunning && haltScanRequests == 0) { fileScanWasRunning = false; InvokeScan(true); } } - private void _apiController_DownloadStarted() + public void HaltScan() { - if (IsScanRunning) + Interlocked.Increment(ref haltScanRequests); + + if (IsScanRunning && haltScanRequests >= 0) { _scanCancellationTokenSource?.Cancel(); fileScanWasRunning = true; @@ -74,8 +82,10 @@ public class PeriodicFileScanner : IDisposable Logger.Verbose("Disposing " + nameof(PeriodicFileScanner)); _ipcManager.PenumbraInitialized -= StartScan; - _apiController.DownloadStarted -= _apiController_DownloadStarted; - _apiController.DownloadFinished -= _apiController_DownloadFinished; + _apiController.DownloadStarted -= HaltScan; + _apiController.DownloadFinished -= ResumeScan; + _dalamudUtil.ZoneSwitchStart -= HaltScan; + _dalamudUtil.ZoneSwitchEnd -= ResumeScan; _scanCancellationTokenSource?.Cancel(); } @@ -91,6 +101,11 @@ public class PeriodicFileScanner : IDisposable { while (!token.IsCancellationRequested) { + while (haltScanRequests > 0) + { + await Task.Delay(TimeSpan.FromSeconds(1)); + } + isForced |= RecalculateFileCacheSize(); if (!_pluginConfiguration.FileScanPaused || isForced) { @@ -98,6 +113,8 @@ public class PeriodicFileScanner : IDisposable TotalFiles = 0; currentFileProgress = 0; PeriodicFileScan(token); + TotalFiles = 0; + currentFileProgress = 0; } _timeUntilNextScan = TimeSpan.FromSeconds(timeBetweenScans); while (_timeUntilNextScan.TotalSeconds >= 0) @@ -182,6 +199,7 @@ public class PeriodicFileScanner : IDisposable Task[] dbTasks = Enumerable.Range(0, cpuCount).Select(c => Task.CompletedTask).ToArray(); ConcurrentBag> entitiesToRemove = new(); + ConcurrentBag entitiesToUpdate = new(); try { using var ctx = new FileCacheContext(); @@ -200,14 +218,19 @@ public class PeriodicFileScanner : IDisposable { try { - var file = _fileDbManager.ValidateFileCacheEntity(hash, filePath, date); - if (file != null) + var fileState = _fileDbManager.ValidateFileCacheEntity(hash, filePath, date); + switch (fileState.Item1) { - scannedFiles[file.Filepath] = true; - } - else - { - entitiesToRemove.Add(new Tuple(hash, filePath)); + case FileState.Valid: + scannedFiles[fileState.Item2] = true; + break; + case FileState.RequireDeletion: + entitiesToRemove.Add(new Tuple(hash, filePath)); + break; + case FileState.RequireUpdate: + scannedFiles[fileState.Item2] = true; + entitiesToUpdate.Add(fileState.Item2); + break; } } catch (Exception ex) @@ -225,6 +248,10 @@ public class PeriodicFileScanner : IDisposable if (ct.IsCancellationRequested) return; } } + catch (OperationCanceledException) + { + return; + } catch (Exception ex) { Logger.Warn("Error during enumerating FileCaches: " + ex.Message); @@ -243,8 +270,22 @@ public class PeriodicFileScanner : IDisposable db.FileCaches.Remove(toRemove); } + } + + if (entitiesToUpdate.Any()) + { + foreach (var entry in entitiesToUpdate) + { + Logger.Debug("Updating " + entry); + _fileDbManager.GetFileCacheByPath(entry); + } + } + + if (entitiesToUpdate.Any() || entitiesToRemove.Any()) + { db.SaveChanges(); } + } catch (Exception ex) { diff --git a/MareSynchronos/Managers/PlayerManager.cs b/MareSynchronos/Managers/PlayerManager.cs index 138a75b..3db33fd 100644 --- a/MareSynchronos/Managers/PlayerManager.cs +++ b/MareSynchronos/Managers/PlayerManager.cs @@ -9,6 +9,7 @@ using FFXIVClientStructs.FFXIV.Client.Game.Character; using System.Collections.Generic; using System.Linq; using MareSynchronos.Models; +using MareSynchronos.FileCacheDB; #if DEBUG using Newtonsoft.Json; #endif @@ -23,6 +24,7 @@ namespace MareSynchronos.Managers private readonly CharacterDataFactory _characterDataFactory; private readonly DalamudUtil _dalamudUtil; private readonly TransientResourceManager _transientResourceManager; + private readonly PeriodicFileScanner _periodicFileScanner; private readonly IpcManager _ipcManager; public event PlayerHasChanged? PlayerHasChanged; public CharacterCacheDto? LastCreatedCharacterData { get; private set; } @@ -35,7 +37,8 @@ namespace MareSynchronos.Managers private List playerRelatedObjects = new List(); public unsafe PlayerManager(ApiController apiController, IpcManager ipcManager, - CharacterDataFactory characterDataFactory, DalamudUtil dalamudUtil, TransientResourceManager transientResourceManager) + CharacterDataFactory characterDataFactory, DalamudUtil dalamudUtil, TransientResourceManager transientResourceManager, + PeriodicFileScanner periodicFileScanner) { Logger.Verbose("Creating " + nameof(PlayerManager)); @@ -44,6 +47,7 @@ namespace MareSynchronos.Managers _characterDataFactory = characterDataFactory; _dalamudUtil = dalamudUtil; _transientResourceManager = transientResourceManager; + _periodicFileScanner = periodicFileScanner; _apiController.Connected += ApiControllerOnConnected; _apiController.Disconnected += ApiController_Disconnected; _transientResourceManager.TransientResourceLoaded += HandleTransientResourceLoad; @@ -231,12 +235,14 @@ namespace MareSynchronos.Managers Task.Run(async () => { + _periodicFileScanner.HaltScan(); foreach (var item in unprocessedObjects) { _dalamudUtil.WaitWhileCharacterIsDrawing("self " + item.ObjectKind.ToString(), item.Address, 10000, token); } CharacterCacheDto? cacheDto = (await CreateFullCharacterCacheDto(token)); + _periodicFileScanner.ResumeScan(); if (cacheDto == null || token.IsCancellationRequested) return; #if DEBUG diff --git a/MareSynchronos/MareSynchronos.csproj b/MareSynchronos/MareSynchronos.csproj index 8d7ce64..747b75a 100644 --- a/MareSynchronos/MareSynchronos.csproj +++ b/MareSynchronos/MareSynchronos.csproj @@ -3,7 +3,7 @@ - 0.4.18 + 0.4.19 https://github.com/Penumbra-Sync/client diff --git a/MareSynchronos/Plugin.cs b/MareSynchronos/Plugin.cs index f157cb5..6760e1e 100644 --- a/MareSynchronos/Plugin.cs +++ b/MareSynchronos/Plugin.cs @@ -24,7 +24,7 @@ namespace MareSynchronos private readonly ApiController _apiController; private readonly CommandManager _commandManager; private readonly Configuration _configuration; - private readonly PeriodicFileScanner _fileCacheManager; + private readonly PeriodicFileScanner _periodicFileScanner; private readonly IntroUi _introUi; private readonly IpcManager _ipcManager; public static DalamudPluginInterface PluginInterface { get; set; } @@ -66,14 +66,14 @@ namespace MareSynchronos _fileDialogManager = new FileDialogManager(); _fileDbManager = new FileDbManager(_ipcManager, _configuration); _apiController = new ApiController(_configuration, _dalamudUtil, _fileDbManager); - _fileCacheManager = new PeriodicFileScanner(_ipcManager, _configuration, _fileDbManager, _apiController); + _periodicFileScanner = new PeriodicFileScanner(_ipcManager, _configuration, _fileDbManager, _apiController, _dalamudUtil); _uiSharedComponent = - new UiShared(_ipcManager, _apiController, _fileCacheManager, _fileDialogManager, _configuration, _dalamudUtil, PluginInterface, _localization); + new UiShared(_ipcManager, _apiController, _periodicFileScanner, _fileDialogManager, _configuration, _dalamudUtil, PluginInterface, _localization); _settingsUi = new SettingsUi(_windowSystem, _uiSharedComponent, _configuration, _apiController); _compactUi = new CompactUi(_windowSystem, _uiSharedComponent, _configuration, _apiController); - _introUi = new IntroUi(_windowSystem, _uiSharedComponent, _configuration, _fileCacheManager); + _introUi = new IntroUi(_windowSystem, _uiSharedComponent, _configuration, _periodicFileScanner); _settingsUi.SwitchToIntroUi += () => { _introUi.IsOpen = true; @@ -84,7 +84,7 @@ namespace MareSynchronos { _introUi.IsOpen = false; _compactUi.IsOpen = true; - _fileCacheManager.StartWatchers(); + _periodicFileScanner.StartWatchers(); ReLaunchCharacterManager(); }; _compactUi.OpenSettingsUi += () => @@ -119,7 +119,7 @@ namespace MareSynchronos _downloadUi?.Dispose(); _compactUi?.Dispose(); - _fileCacheManager?.Dispose(); + _periodicFileScanner?.Dispose(); _playerManager?.Dispose(); _characterCacheManager?.Dispose(); _ipcManager?.Dispose(); @@ -184,7 +184,7 @@ namespace MareSynchronos var characterCacheFactory = new CharacterDataFactory(_dalamudUtil, _ipcManager, _transientResourceManager, _fileDbManager); _playerManager = new PlayerManager(_apiController, _ipcManager, - characterCacheFactory, _dalamudUtil, _transientResourceManager); + characterCacheFactory, _dalamudUtil, _transientResourceManager, _periodicFileScanner); _characterCacheManager = new OnlinePlayerManager(_apiController, _dalamudUtil, _ipcManager, _playerManager, _fileDbManager); } diff --git a/MareSynchronos/Utils/DalamudUtil.cs b/MareSynchronos/Utils/DalamudUtil.cs index 2f3b742..d2eba06 100644 --- a/MareSynchronos/Utils/DalamudUtil.cs +++ b/MareSynchronos/Utils/DalamudUtil.cs @@ -75,7 +75,7 @@ namespace MareSynchronos.Utils { if (!_sentBetweenAreas) { - Logger.Debug("Zone switch start"); + Logger.Debug("Zone switch/Gpose start"); _sentBetweenAreas = true; ZoneSwitchStart?.Invoke(); } @@ -84,7 +84,7 @@ namespace MareSynchronos.Utils } else if (_sentBetweenAreas) { - Logger.Debug("Zone switch end"); + Logger.Debug("Zone switch/Gpose end"); _sentBetweenAreas = false; ZoneSwitchEnd?.Invoke(); }