stop scan on character data creation, downloads, gpose, zone switch, update outside of read loop, adjust verify

MareSynchronos/Factories/CharacterDataFactory.cs

@@ -8,6 +8,7 @@ using FFXIVClientStructs.FFXIV.Client.Game.Character;
 using FFXIVClientStructs.FFXIV.Client.Graphics.Scene;
 using FFXIVClientStructs.FFXIV.Client.System.Resource;
 using MareSynchronos.API;
+using MareSynchronos.FileCacheDB;
 using MareSynchronos.Interop;
 using MareSynchronos.Managers;
 using MareSynchronos.Models;

MareSynchronos/FileCacheDB/FileDbManager.cs

@@ -8,6 +8,13 @@ using System.Linq;
 
 namespace MareSynchronos.Managers;
 
+public enum FileState
+{
+    Valid,
+    RequireUpdate,
+    RequireDeletion
+}
+
 public class FileDbManager
 {
     private const string PenumbraPrefix = "{penumbra}";
@@ -45,9 +52,19 @@ public class FileDbManager
         return GetValidatedFileCache(new FileCache(matchingEntries.First()));
     }
 
-    public FileCache? ValidateFileCacheEntity(string hash, string path, string lastModifiedDate)
+    public (FileState, string) ValidateFileCacheEntity(string hash, string path, string lastModifiedDate)
     {
-        return GetValidatedFileCache(new FileCache(hash, path, lastModifiedDate), false);
+        var fileCache = new FileCache(hash, path, lastModifiedDate);
+        if (!fileCache.OriginalFilepath.StartsWith(PenumbraPrefix + "\\") && !fileCache.OriginalFilepath.StartsWith(CachePrefix)) 
+            return (FileState.RequireUpdate, path);
+        fileCache = ReplacePathPrefixes(fileCache);
+        FileInfo fi = new FileInfo(fileCache.Filepath);
+        if (!fi.Exists) 
+            return (FileState.RequireDeletion, fileCache.Filepath);
+        if (fi.LastWriteTimeUtc.Ticks != fileCache.LastModifiedDateTicks) 
+            return (FileState.RequireUpdate, fileCache.Filepath);
+
+        return (FileState.Valid, fileCache.Filepath);
     }
 
     public FileCache? GetFileCacheByPath(string path)
@@ -61,6 +78,7 @@ public class FileDbManager
 
         if (matchingEntries == null)
         {
+            Logger.Debug("Found no entries for " + cleanedPath);
             return CreateFileEntry(path);
         }
 
@@ -104,7 +122,8 @@ public class FileDbManager
             }
             catch (Exception ex)
             {
-                Logger.Warn("Could not add " + fileInfo.FullName);
+                Logger.Warn("Could not add " + fileInfo.FullName ?? String.Empty);
+                Logger.Warn(ex.Message);
             }
         }
         var result = GetFileCacheByPath(prefixedPath);
@@ -112,25 +131,22 @@ public class FileDbManager
         return result;
     }
 
-    private FileCache? GetValidatedFileCache(FileCache fileCache, bool removeOnNonExistence = true)
+    private FileCache? GetValidatedFileCache(FileCache fileCache)
     {
         var resulingFileCache = MigrateLegacy(fileCache);
         if (resulingFileCache == null) return null;
 
         resulingFileCache = ReplacePathPrefixes(resulingFileCache);
-        resulingFileCache = Validate(resulingFileCache, removeOnNonExistence);
+        resulingFileCache = Validate(resulingFileCache);
         return resulingFileCache;
     }
 
-    private FileCache? Validate(FileCache fileCache, bool removeOnNonExistence = true)
+    private FileCache? Validate(FileCache fileCache)
     {
         var file = new FileInfo(fileCache.Filepath);
         if (!file.Exists)
-        {
-            if (removeOnNonExistence)
         {
             DeleteFromDatabase(new[] { fileCache });
-            }
             return null;
         }
 
@@ -143,7 +159,7 @@ public class FileDbManager
         return fileCache;
     }
 
-    private FileCache? MigrateLegacy(FileCache fileCache, bool removeOnNonExistence = true)
+    private FileCache? MigrateLegacy(FileCache fileCache)
     {
         if (fileCache.OriginalFilepath.StartsWith(PenumbraPrefix + "\\") || fileCache.OriginalFilepath.StartsWith(CachePrefix)) return fileCache;
 
@@ -167,11 +183,8 @@ public class FileDbManager
             MigrateLegacyFilePath(fileCache, newPath);
         }
         else
-        {
-            if (removeOnNonExistence)
         {
             DeleteFromDatabase(new[] { fileCache });
-            }
             return null;
         }
 
@@ -214,7 +227,9 @@ public class FileDbManager
             }
             catch (Exception ex)
             {
-                Logger.Warn("Error updating file hash (" + ex.Message + "), returning currently existing");
+                Logger.Warn("Error updating file hash (" + ex.Message + "), returning currently existing" ?? string.Empty);
+                Logger.Warn(ex.InnerException?.Message ?? string.Empty);
+                Logger.Warn(ex.StackTrace ?? string.Empty);
                 using var db = new FileCacheContext();
                 var cache = db.FileCaches.First(f => f.Filepath == markedForUpdate.OriginalFilepath);
                 markedForUpdate.UpdateFileCache(cache);

MareSynchronos/FileCacheDB/PeriodicFileScanner.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Concurrent;
-using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Threading;
@@ -18,9 +17,11 @@ public class PeriodicFileScanner : IDisposable
     private readonly Configuration _pluginConfiguration;
     private readonly FileDbManager _fileDbManager;
     private readonly ApiController _apiController;
+    private readonly DalamudUtil _dalamudUtil;
+    private int haltScanRequests = 0;
     private CancellationTokenSource? _scanCancellationTokenSource;
     private Task? _fileScannerTask = null;
-    public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileDbManager fileDbManager, ApiController apiController)
+    public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileDbManager fileDbManager, ApiController apiController, DalamudUtil dalamudUtil)
     {
         Logger.Verbose("Creating " + nameof(PeriodicFileScanner));
 
@@ -28,27 +29,34 @@ public class PeriodicFileScanner : IDisposable
         _pluginConfiguration = pluginConfiguration;
         _fileDbManager = fileDbManager;
         _apiController = apiController;
+        _dalamudUtil = dalamudUtil;
         _ipcManager.PenumbraInitialized += StartScan;
         if (!string.IsNullOrEmpty(_ipcManager.PenumbraModDirectory()))
         {
             StartScan();
         }
-        _apiController.DownloadStarted += _apiController_DownloadStarted;
+        _apiController.DownloadStarted += HaltScan;
-        _apiController.DownloadFinished += _apiController_DownloadFinished;
+        _apiController.DownloadFinished += ResumeScan;
+        _dalamudUtil.ZoneSwitchStart += HaltScan;
+        _dalamudUtil.ZoneSwitchEnd += ResumeScan;
     }
 
-    private void _apiController_DownloadFinished()
+    public void ResumeScan()
     {
-        if (fileScanWasRunning)
+        Interlocked.Decrement(ref haltScanRequests);
+
+        if (fileScanWasRunning && haltScanRequests == 0)
         {
             fileScanWasRunning = false;
             InvokeScan(true);
         }
     }
 
-    private void _apiController_DownloadStarted()
+    public void HaltScan()
     {
-        if (IsScanRunning)
+        Interlocked.Increment(ref haltScanRequests);
+
+        if (IsScanRunning && haltScanRequests >= 0)
         {
             _scanCancellationTokenSource?.Cancel();
             fileScanWasRunning = true;
@@ -74,8 +82,10 @@ public class PeriodicFileScanner : IDisposable
         Logger.Verbose("Disposing " + nameof(PeriodicFileScanner));
 
         _ipcManager.PenumbraInitialized -= StartScan;
-        _apiController.DownloadStarted -= _apiController_DownloadStarted;
+        _apiController.DownloadStarted -= HaltScan;
-        _apiController.DownloadFinished -= _apiController_DownloadFinished;
+        _apiController.DownloadFinished -= ResumeScan;
+        _dalamudUtil.ZoneSwitchStart -= HaltScan;
+        _dalamudUtil.ZoneSwitchEnd -= ResumeScan;
         _scanCancellationTokenSource?.Cancel();
     }
 
@@ -91,6 +101,11 @@ public class PeriodicFileScanner : IDisposable
         {
             while (!token.IsCancellationRequested)
             {
+                while (haltScanRequests > 0)
+                {
+                    await Task.Delay(TimeSpan.FromSeconds(1));
+                }
+
                 isForced |= RecalculateFileCacheSize();
                 if (!_pluginConfiguration.FileScanPaused || isForced)
                 {
@@ -98,6 +113,8 @@ public class PeriodicFileScanner : IDisposable
                     TotalFiles = 0;
                     currentFileProgress = 0;
                     PeriodicFileScan(token);
+                    TotalFiles = 0;
+                    currentFileProgress = 0;
                 }
                 _timeUntilNextScan = TimeSpan.FromSeconds(timeBetweenScans);
                 while (_timeUntilNextScan.TotalSeconds >= 0)
@@ -182,6 +199,7 @@ public class PeriodicFileScanner : IDisposable
         Task[] dbTasks = Enumerable.Range(0, cpuCount).Select(c => Task.CompletedTask).ToArray();
 
         ConcurrentBag<Tuple<string, string>> entitiesToRemove = new();
+        ConcurrentBag<string> entitiesToUpdate = new();
         try
         {
             using var ctx = new FileCacheContext();
@@ -200,14 +218,19 @@ public class PeriodicFileScanner : IDisposable
                 {
                     try
                     {
-                        var file = _fileDbManager.ValidateFileCacheEntity(hash, filePath, date);
+                        var fileState = _fileDbManager.ValidateFileCacheEntity(hash, filePath, date);
-                        if (file != null)
+                        switch (fileState.Item1)
-                        {
-                            scannedFiles[file.Filepath] = true;
-                        }
-                        else
                         {
+                            case FileState.Valid:
+                                scannedFiles[fileState.Item2] = true;
+                                break;
+                            case FileState.RequireDeletion:
                                 entitiesToRemove.Add(new Tuple<string, string>(hash, filePath));
+                                break;
+                            case FileState.RequireUpdate:
+                                scannedFiles[fileState.Item2] = true;
+                                entitiesToUpdate.Add(fileState.Item2);
+                                break;
                         }
                     }
                     catch (Exception ex)
@@ -225,6 +248,10 @@ public class PeriodicFileScanner : IDisposable
                 if (ct.IsCancellationRequested) return;
             }
         }
+        catch (OperationCanceledException)
+        {
+            return;
+        }
         catch (Exception ex)
         {
             Logger.Warn("Error during enumerating FileCaches: " + ex.Message);
@@ -243,8 +270,22 @@ public class PeriodicFileScanner : IDisposable
                         db.FileCaches.Remove(toRemove);
                     }
 
+                }
+
+                if (entitiesToUpdate.Any())
+                {
+                    foreach (var entry in entitiesToUpdate)
+                    {
+                        Logger.Debug("Updating " + entry);
+                        _fileDbManager.GetFileCacheByPath(entry);
+                    }
+                }
+
+                if (entitiesToUpdate.Any() || entitiesToRemove.Any())
+                {
                     db.SaveChanges();
                 }
+
             }
             catch (Exception ex)
             {

MareSynchronos/Managers/PlayerManager.cs

@@ -9,6 +9,7 @@ using FFXIVClientStructs.FFXIV.Client.Game.Character;
 using System.Collections.Generic;
 using System.Linq;
 using MareSynchronos.Models;
+using MareSynchronos.FileCacheDB;
 #if DEBUG
 using Newtonsoft.Json;
 #endif
@@ -23,6 +24,7 @@ namespace MareSynchronos.Managers
         private readonly CharacterDataFactory _characterDataFactory;
         private readonly DalamudUtil _dalamudUtil;
         private readonly TransientResourceManager _transientResourceManager;
+        private readonly PeriodicFileScanner _periodicFileScanner;
         private readonly IpcManager _ipcManager;
         public event PlayerHasChanged? PlayerHasChanged;
         public CharacterCacheDto? LastCreatedCharacterData { get; private set; }
@@ -35,7 +37,8 @@ namespace MareSynchronos.Managers
         private List<PlayerRelatedObject> playerRelatedObjects = new List<PlayerRelatedObject>();
 
         public unsafe PlayerManager(ApiController apiController, IpcManager ipcManager,
-            CharacterDataFactory characterDataFactory, DalamudUtil dalamudUtil, TransientResourceManager transientResourceManager)
+            CharacterDataFactory characterDataFactory, DalamudUtil dalamudUtil, TransientResourceManager transientResourceManager,
+            PeriodicFileScanner periodicFileScanner)
         {
             Logger.Verbose("Creating " + nameof(PlayerManager));
 
@@ -44,6 +47,7 @@ namespace MareSynchronos.Managers
             _characterDataFactory = characterDataFactory;
             _dalamudUtil = dalamudUtil;
             _transientResourceManager = transientResourceManager;
+            _periodicFileScanner = periodicFileScanner;
             _apiController.Connected += ApiControllerOnConnected;
             _apiController.Disconnected += ApiController_Disconnected;
             _transientResourceManager.TransientResourceLoaded += HandleTransientResourceLoad;
@@ -231,12 +235,14 @@ namespace MareSynchronos.Managers
 
             Task.Run(async () =>
             {
+                _periodicFileScanner.HaltScan();
                 foreach (var item in unprocessedObjects)
                 {
                     _dalamudUtil.WaitWhileCharacterIsDrawing("self " + item.ObjectKind.ToString(), item.Address, 10000, token);
                 }
 
                 CharacterCacheDto? cacheDto = (await CreateFullCharacterCacheDto(token));
+                _periodicFileScanner.ResumeScan();
                 if (cacheDto == null || token.IsCancellationRequested) return;
 
 #if DEBUG

MareSynchronos/MareSynchronos.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <Authors></Authors>
     <Company></Company>
-    <Version>0.4.18</Version>
+    <Version>0.4.19</Version>
     <Description></Description>
     <Copyright></Copyright>
     <PackageProjectUrl>https://github.com/Penumbra-Sync/client</PackageProjectUrl>

MareSynchronos/Plugin.cs

@@ -24,7 +24,7 @@ namespace MareSynchronos
         private readonly ApiController _apiController;
         private readonly CommandManager _commandManager;
         private readonly Configuration _configuration;
-        private readonly PeriodicFileScanner _fileCacheManager;
+        private readonly PeriodicFileScanner _periodicFileScanner;
         private readonly IntroUi _introUi;
         private readonly IpcManager _ipcManager;
         public static DalamudPluginInterface PluginInterface { get; set; }
@@ -66,14 +66,14 @@ namespace MareSynchronos
             _fileDialogManager = new FileDialogManager();
             _fileDbManager = new FileDbManager(_ipcManager, _configuration);
             _apiController = new ApiController(_configuration, _dalamudUtil, _fileDbManager);
-            _fileCacheManager = new PeriodicFileScanner(_ipcManager, _configuration, _fileDbManager, _apiController);
+            _periodicFileScanner = new PeriodicFileScanner(_ipcManager, _configuration, _fileDbManager, _apiController, _dalamudUtil);
 
             _uiSharedComponent =
-                new UiShared(_ipcManager, _apiController, _fileCacheManager, _fileDialogManager, _configuration, _dalamudUtil, PluginInterface, _localization);
+                new UiShared(_ipcManager, _apiController, _periodicFileScanner, _fileDialogManager, _configuration, _dalamudUtil, PluginInterface, _localization);
             _settingsUi = new SettingsUi(_windowSystem, _uiSharedComponent, _configuration, _apiController);
             _compactUi = new CompactUi(_windowSystem, _uiSharedComponent, _configuration, _apiController);
 
-            _introUi = new IntroUi(_windowSystem, _uiSharedComponent, _configuration, _fileCacheManager);
+            _introUi = new IntroUi(_windowSystem, _uiSharedComponent, _configuration, _periodicFileScanner);
             _settingsUi.SwitchToIntroUi += () =>
             {
                 _introUi.IsOpen = true;
@@ -84,7 +84,7 @@ namespace MareSynchronos
             {
                 _introUi.IsOpen = false;
                 _compactUi.IsOpen = true;
-                _fileCacheManager.StartWatchers();
+                _periodicFileScanner.StartWatchers();
                 ReLaunchCharacterManager();
             };
             _compactUi.OpenSettingsUi += () =>
@@ -119,7 +119,7 @@ namespace MareSynchronos
             _downloadUi?.Dispose();
             _compactUi?.Dispose();
 
-            _fileCacheManager?.Dispose();
+            _periodicFileScanner?.Dispose();
             _playerManager?.Dispose();
             _characterCacheManager?.Dispose();
             _ipcManager?.Dispose();
@@ -184,7 +184,7 @@ namespace MareSynchronos
                 var characterCacheFactory =
                     new CharacterDataFactory(_dalamudUtil, _ipcManager, _transientResourceManager, _fileDbManager);
                 _playerManager = new PlayerManager(_apiController, _ipcManager,
-                    characterCacheFactory, _dalamudUtil, _transientResourceManager);
+                    characterCacheFactory, _dalamudUtil, _transientResourceManager, _periodicFileScanner);
                 _characterCacheManager = new OnlinePlayerManager(_apiController,
                     _dalamudUtil, _ipcManager, _playerManager, _fileDbManager);
             }

MareSynchronos/Utils/DalamudUtil.cs

@@ -75,7 +75,7 @@ namespace MareSynchronos.Utils
             {
                 if (!_sentBetweenAreas)
                 {
-                    Logger.Debug("Zone switch start");
+                    Logger.Debug("Zone switch/Gpose start");
                     _sentBetweenAreas = true;
                     ZoneSwitchStart?.Invoke();
                 }
@@ -84,7 +84,7 @@ namespace MareSynchronos.Utils
             }
             else if (_sentBetweenAreas)
             {
-                Logger.Debug("Zone switch end");
+                Logger.Debug("Zone switch/Gpose end");
                 _sentBetweenAreas = false;
                 ZoneSwitchEnd?.Invoke();
             }