add periodic file scanner, parallelize downloads, fix transient files being readded when not necessary, fix disposal of players on plugin shutdown

MareSynchronos/Configuration.cs

@@ -60,6 +60,9 @@ namespace MareSynchronos
 
         public bool ReverseUserSort { get; set; } = true;
 
+        public int TimeSpanBetweenScansInSeconds { get; set; } = 30;
+        public bool FileScanPaused { get; set; } = false;
+
         public bool InitialScanComplete { get; set; } = false;
         public int MaxParallelScan
         {
@@ -207,6 +210,12 @@ namespace MareSynchronos
                 Version = 5;
                 Save();
             }
+
+            if (FileScanPaused)
+            {
+                FileScanPaused = false;
+                Save();
+            }
         }
     }
 }

MareSynchronos/Factories/CharacterDataFactory.cs

@@ -23,11 +23,12 @@ public class CharacterDataFactory
     private readonly DalamudUtil _dalamudUtil;
     private readonly IpcManager _ipcManager;
     private readonly TransientResourceManager transientResourceManager;
+    private readonly FileDbManager fileDbManager;
 
-    public CharacterDataFactory(DalamudUtil dalamudUtil, IpcManager ipcManager, TransientResourceManager transientResourceManager)
+    public CharacterDataFactory(DalamudUtil dalamudUtil, IpcManager ipcManager, TransientResourceManager transientResourceManager, FileDbManager fileDbManager)
     {
         Logger.Verbose("Creating " + nameof(CharacterDataFactory));
-
+        this.fileDbManager = fileDbManager;
         _dalamudUtil = dalamudUtil;
         _ipcManager = ipcManager;
         this.transientResourceManager = transientResourceManager;
@@ -416,7 +417,7 @@ public class CharacterDataFactory
 
     private FileReplacement CreateFileReplacement(string path, bool doNotReverseResolve = false)
     {
-        var fileReplacement = new FileReplacement(_ipcManager.PenumbraModDirectory()!);
+        var fileReplacement = new FileReplacement(fileDbManager);
         if (!doNotReverseResolve)
         {
             fileReplacement.GamePaths =

MareSynchronos/FileCacheDB/FileCache.cs

@@ -1,12 +1,36 @@
 #nullable disable
 
+
 namespace MareSynchronos.FileCacheDB
 {
-    public partial class FileCache
+
+    public class FileCache
     {
-        public string Hash { get; set; }
+        private FileCacheEntity entity;
-        public string Filepath { get; set; }
+        public string Filepath { get; private set; }
-        public string LastModifiedDate { get; set; }
+        public string Hash { get; private set; }
-        public int Version { get; set; }
+        public string OriginalFilepath => entity.Filepath;
+        public string OriginalHash => entity.Hash;
+        public long LastModifiedDateTicks => long.Parse(entity.LastModifiedDate);
+
+        public FileCache(FileCacheEntity entity)
+        {
+            this.entity = entity;
+        }
+
+        public void SetResolvedFilePath(string filePath)
+        {
+            Filepath = filePath.ToLowerInvariant();
+        }
+
+        public void SetHash(string hash)
+        {
+            Hash = hash;
+        }
+
+        public void UpdateFileCache(FileCacheEntity entity)
+        {
+            this.entity = entity;
+        }
     }
 }

MareSynchronos/FileCacheDB/FileCacheContext.cs

@@ -36,7 +36,7 @@ namespace MareSynchronos.FileCacheDB
         {
         }
 
-        public virtual DbSet<FileCache> FileCaches { get; set; }
+        public virtual DbSet<FileCacheEntity> FileCaches { get; set; }
 
         protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
         {
@@ -48,7 +48,7 @@ namespace MareSynchronos.FileCacheDB
 
         protected override void OnModelCreating(ModelBuilder modelBuilder)
         {
-            modelBuilder.Entity<FileCache>(entity =>
+            modelBuilder.Entity<FileCacheEntity>(entity =>
             {
                 entity.HasKey(e => new { e.Hash, e.Filepath });
 

MareSynchronos/FileCacheDB/FileCacheEntity.cs

@@ -0,0 +1,13 @@
+#nullable disable
+
+
+namespace MareSynchronos.FileCacheDB
+{
+    public partial class FileCacheEntity
+    {
+        public string Hash { get; set; }
+        public string Filepath { get; set; }
+        public string LastModifiedDate { get; set; }
+        public int Version { get; set; }
+    }
+}

MareSynchronos/FileCacheDB/FileDbManager.cs

@@ -0,0 +1,227 @@
+using MareSynchronos.FileCacheDB;
+using MareSynchronos.Utils;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+
+namespace MareSynchronos.Managers;
+
+public class FileDbManager
+{
+    private const string PenumbraPrefix = "{penumbra}";
+    private const string CachePrefix = "{cache}";
+    private readonly IpcManager _ipcManager;
+    private readonly Configuration _configuration;
+    private static object _lock = new();
+
+    public FileDbManager(IpcManager ipcManager, Configuration configuration)
+    {
+        _ipcManager = ipcManager;
+        _configuration = configuration;
+    }
+
+    public FileCache? GetFileCacheByHash(string hash)
+    {
+        List<FileCacheEntity> matchingEntries = new List<FileCacheEntity>();
+        using (var db = new FileCacheContext())
+        {
+            matchingEntries = db.FileCaches.Where(f => f.Hash.ToLower() == hash.ToLower()).ToList();
+        }
+
+        if (!matchingEntries.Any()) return null;
+
+        if (matchingEntries.Any(f => f.Filepath.Contains(PenumbraPrefix) && matchingEntries.Any(f => f.Filepath.Contains(CachePrefix))))
+        {
+            var cachedEntries = matchingEntries.Where(f => f.Filepath.Contains(CachePrefix));
+            DeleteFromDatabase(cachedEntries.Select(f => new FileCache(f)));
+            foreach (var entry in cachedEntries)
+            {
+                matchingEntries.Remove(entry);
+            }
+        }
+
+        return GetValidatedFileCache(matchingEntries.First());
+    }
+
+    public FileCache? ValidateFileCache(FileCacheEntity fileCacheEntity)
+    {
+        return GetValidatedFileCache(fileCacheEntity);
+    }
+
+    public FileCache? GetFileCacheByPath(string path)
+    {
+        FileCacheEntity? matchingEntries = null;
+        var cleanedPath = path.Replace("/", "\\").ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), "");
+        using (var db = new FileCacheContext())
+        {
+            matchingEntries = db.FileCaches.FirstOrDefault(f => f.Filepath.EndsWith(cleanedPath));
+        }
+
+        if (matchingEntries == null)
+        {
+            return CreateFileCacheEntity(path);
+        }
+
+        var validatedCacheEntry = GetValidatedFileCache(matchingEntries);
+
+        return validatedCacheEntry;
+    }
+
+    public FileCache? CreateFileCacheEntity(string path)
+    {
+        Logger.Verbose("Creating entry for " + path);
+        FileInfo fi = new FileInfo(path);
+        if (!fi.Exists) return null;
+        string prefixedPath = fi.FullName.ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), PenumbraPrefix + "\\")
+            .Replace(_configuration.CacheFolder.ToLowerInvariant(), CachePrefix + "\\").Replace("\\\\", "\\");
+        var hash = Crypto.GetFileHash(path);
+        lock (_lock)
+        {
+            var entity = new FileCacheEntity();
+            entity.Hash = hash;
+            entity.Filepath = prefixedPath;
+            entity.LastModifiedDate = fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
+            try
+            {
+                using var db = new FileCacheContext();
+                db.FileCaches.Add(entity);
+                db.SaveChanges();
+            }
+            catch (Exception ex)
+            {
+                Logger.Warn("Could not add " + path);
+            }
+        }
+        return GetFileCacheByPath(prefixedPath)!;
+    }
+
+    private FileCache? GetValidatedFileCache(FileCacheEntity e)
+    {
+        var fileCache = new FileCache(e);
+
+        var resulingFileCache = MigrateLegacy(fileCache);
+
+        if (resulingFileCache == null) return null;
+
+        resulingFileCache = ReplacePathPrefixes(resulingFileCache);
+
+        resulingFileCache = Validate(resulingFileCache);
+        return resulingFileCache;
+    }
+
+    private FileCache? Validate(FileCache fileCache)
+    {
+        var file = new FileInfo(fileCache.Filepath);
+        if (!file.Exists)
+        {
+            DeleteFromDatabase(new[] { fileCache });
+            return null;
+        }
+
+        if (file.LastWriteTimeUtc.Ticks != fileCache.LastModifiedDateTicks)
+        {
+            fileCache.SetHash(Crypto.GetFileHash(fileCache.Filepath));
+            UpdateCacheHash(fileCache, file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture));
+        }
+
+        return fileCache;
+    }
+
+    private FileCache? MigrateLegacy(FileCache fileCache)
+    {
+        if (fileCache.OriginalFilepath.Contains(PenumbraPrefix) || fileCache.OriginalFilepath.Contains(CachePrefix)) return fileCache;
+
+        var fileInfo = new FileInfo(fileCache.OriginalFilepath);
+        var penumbraDir = _ipcManager.PenumbraModDirectory()!;
+        // check if it's a cache file
+        if (fileInfo.Exists && fileInfo.Name.Length == 40)
+        {
+            MigrateLegacyFilePath(fileCache, CachePrefix + "\\" + fileInfo.Name.ToLower());
+        }
+        else if (fileInfo.Exists && fileInfo.FullName.ToLowerInvariant().Contains(penumbraDir))
+        {
+            // attempt to replace penumbra mod folder path with {penumbra}
+            var newPath = PenumbraPrefix + fileCache.OriginalFilepath.ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!, string.Empty);
+            MigrateLegacyFilePath(fileCache, newPath);
+        }
+        else
+        {
+            DeleteFromDatabase(new[] { fileCache });
+            return null;
+        }
+
+        return fileCache;
+    }
+
+    private FileCache ReplacePathPrefixes(FileCache fileCache)
+    {
+        if (fileCache.OriginalFilepath.Contains(PenumbraPrefix))
+        {
+            fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory()));
+        }
+        else if (fileCache.OriginalFilepath.Contains(CachePrefix))
+        {
+            fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(CachePrefix, _configuration.CacheFolder));
+        }
+
+        return fileCache;
+    }
+
+    private void UpdateCacheHash(FileCache markedForUpdate, string lastModifiedDate)
+    {
+        lock (_lock)
+        {
+            Logger.Verbose("Updating Hash for " + markedForUpdate.OriginalFilepath);
+            using var db = new FileCacheContext();
+            var cache = db.FileCaches.First(f => f.Filepath == markedForUpdate.OriginalFilepath && f.Hash == markedForUpdate.OriginalHash);
+            var newcache = new FileCacheEntity()
+            {
+                Filepath = cache.Filepath,
+                Hash = markedForUpdate.Hash,
+                LastModifiedDate = lastModifiedDate
+            };
+            db.Remove(cache);
+            db.FileCaches.Add(newcache);
+            markedForUpdate.UpdateFileCache(newcache);
+            db.SaveChanges();
+        }
+    }
+
+    private void MigrateLegacyFilePath(FileCache fileCacheToMigrate, string newPath)
+    {
+        lock (_lock)
+        {
+            Logger.Verbose("Migrating legacy file path for " + fileCacheToMigrate.OriginalFilepath);
+            using var db = new FileCacheContext();
+            var cache = db.FileCaches.First(f => f.Filepath == fileCacheToMigrate.OriginalFilepath && f.Hash == fileCacheToMigrate.OriginalHash);
+            var newcache = new FileCacheEntity()
+            {
+                Filepath = newPath,
+                Hash = cache.Hash,
+                LastModifiedDate = cache.LastModifiedDate
+            };
+            db.Remove(cache);
+            db.FileCaches.Add(newcache);
+            fileCacheToMigrate.UpdateFileCache(newcache);
+            db.SaveChanges();
+        }
+    }
+
+    private void DeleteFromDatabase(IEnumerable<FileCache> markedForDeletion)
+    {
+        lock (_lock)
+        {
+            using var db = new FileCacheContext();
+            foreach (var item in markedForDeletion)
+            {
+                Logger.Verbose("Removing " + item.OriginalFilepath);
+                var itemToRemove = db.FileCaches.FirstOrDefault(f => f.Hash == item.OriginalHash && f.Filepath == item.OriginalFilepath);
+                if (itemToRemove == null) continue;
+                db.FileCaches.Remove(itemToRemove);
+            }
+            db.SaveChanges();
+        }
+    }
+}

MareSynchronos/FileCacheDB/PeriodicFileScanner.cs

@@ -0,0 +1,233 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using MareSynchronos.Managers;
+using MareSynchronos.Utils;
+using MareSynchronos.WebAPI;
+using Microsoft.EntityFrameworkCore;
+
+namespace MareSynchronos.FileCacheDB;
+
+public class PeriodicFileScanner : IDisposable
+{
+    private readonly IpcManager _ipcManager;
+    private readonly Configuration _pluginConfiguration;
+    private readonly FileDbManager _fileDbManager;
+    private readonly ApiController _apiController;
+    private CancellationTokenSource? _scanCancellationTokenSource;
+    private Task? _fileScannerTask = null;
+    public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileDbManager fileDbManager, ApiController apiController)
+    {
+        Logger.Verbose("Creating " + nameof(PeriodicFileScanner));
+
+        _ipcManager = ipcManager;
+        _pluginConfiguration = pluginConfiguration;
+        _fileDbManager = fileDbManager;
+        _apiController = apiController;
+        _ipcManager.PenumbraInitialized += StartScan;
+        if (!string.IsNullOrEmpty(_ipcManager.PenumbraModDirectory()))
+        {
+            StartScan();
+        }
+        _apiController.DownloadStarted += _apiController_DownloadStarted;
+        _apiController.DownloadFinished += _apiController_DownloadFinished;
+    }
+
+    private void _apiController_DownloadFinished()
+    {
+        InvokeScan();
+    }
+
+    private void _apiController_DownloadStarted()
+    {
+        _scanCancellationTokenSource?.Cancel();
+    }
+
+    private long currentFileProgress = 0;
+    public long CurrentFileProgress => currentFileProgress;
+
+    public long FileCacheSize { get; set; }
+
+    public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
+
+    public long TotalFiles { get; private set; }
+
+    public string TimeUntilNextScan => _timeUntilNextScan.ToString(@"mm\:ss");
+    private TimeSpan _timeUntilNextScan = TimeSpan.Zero;
+    private int timeBetweenScans => _pluginConfiguration.TimeSpanBetweenScansInSeconds;
+
+    public void Dispose()
+    {
+        Logger.Verbose("Disposing " + nameof(PeriodicFileScanner));
+
+        _ipcManager.PenumbraInitialized -= StartScan;
+        _apiController.DownloadStarted -= _apiController_DownloadStarted;
+        _apiController.DownloadFinished -= _apiController_DownloadFinished;
+        _scanCancellationTokenSource?.Cancel();
+    }
+
+    public void InvokeScan()
+    {
+        _scanCancellationTokenSource?.Cancel();
+        _scanCancellationTokenSource = new CancellationTokenSource();
+        var token = _scanCancellationTokenSource.Token;
+        _fileScannerTask = Task.Run(async () =>
+        {
+            while (!token.IsCancellationRequested)
+            {
+                RecalculateFileCacheSize();
+                if (!_pluginConfiguration.FileScanPaused)
+                {
+                    await PeriodicFileScan(token);
+                }
+                _timeUntilNextScan = TimeSpan.FromSeconds(timeBetweenScans);
+                while (_timeUntilNextScan.TotalSeconds >= 0)
+                {
+                    await Task.Delay(TimeSpan.FromSeconds(1), token);
+                    _timeUntilNextScan -= TimeSpan.FromSeconds(1);
+                }
+            }
+        });
+    }
+
+    internal void StartWatchers()
+    {
+        InvokeScan();
+    }
+
+    public void RecalculateFileCacheSize()
+    {
+        FileCacheSize = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder).Sum(f =>
+        {
+            try
+            {
+                return new FileInfo(f).Length;
+            }
+            catch
+            {
+                return 0;
+            }
+        });
+
+        if (FileCacheSize < (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024) return;
+
+        var allFiles = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder)
+            .Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList();
+        while (FileCacheSize > (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024)
+        {
+            var oldestFile = allFiles.First();
+            FileCacheSize -= oldestFile.Length;
+            File.Delete(oldestFile.FullName);
+            allFiles.Remove(oldestFile);
+        }
+    }
+
+    private async Task PeriodicFileScan(CancellationToken ct)
+    {
+        TotalFiles = 1;
+        var penumbraDir = _ipcManager.PenumbraModDirectory();
+        bool penDirExists = true;
+        bool cacheDirExists = true;
+        if (string.IsNullOrEmpty(penumbraDir) || !Directory.Exists(penumbraDir))
+        {
+            penDirExists = false;
+            Logger.Warn("Penumbra directory is not set or does not exist.");
+        }
+        if (string.IsNullOrEmpty(_pluginConfiguration.CacheFolder) || !Directory.Exists(_pluginConfiguration.CacheFolder))
+        {
+            cacheDirExists = false;
+            Logger.Warn("Mare Cache directory is not set or does not exist.");
+        }
+        if (!penDirExists || !cacheDirExists)
+        {
+            return;
+        }
+
+        Logger.Debug("Getting files from " + penumbraDir + " and " + _pluginConfiguration.CacheFolder);
+        string[] ext = { ".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp" };
+        var scannedFiles = new ConcurrentDictionary<string, bool>(
+            Directory.EnumerateFiles(penumbraDir, "*.*", SearchOption.AllDirectories)
+                            .Select(s => new FileInfo(s))
+                            .Where(f => ext.Contains(f.Extension) && !f.FullName.Contains(@"\bg\") && !f.FullName.Contains(@"\bgcommon\") && !f.FullName.Contains(@"\ui\"))
+                            .Select(f => f.FullName.ToLowerInvariant())
+                            .Concat(Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.AllDirectories)
+                                .Where(f => new FileInfo(f).Name.Length == 40)
+                                .Select(s => s.ToLowerInvariant()))
+                            .Select(p => new KeyValuePair<string, bool>(p, false)).ToList());
+        List<FileCacheEntity> fileDbEntries;
+        using (var db = new FileCacheContext())
+        {
+            fileDbEntries = await db.FileCaches.ToListAsync(cancellationToken: ct);
+        }
+
+        TotalFiles = scannedFiles.Count;
+
+        Logger.Debug("Database contains " + fileDbEntries.Count + " files, local system contains " + TotalFiles);
+        // scan files from database
+        Parallel.ForEach(fileDbEntries.ToList(), new ParallelOptions()
+        {
+            MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
+            CancellationToken = ct,
+        },
+        dbEntry =>
+        {
+            if (ct.IsCancellationRequested) return;
+            try
+            {
+                var file = _fileDbManager.ValidateFileCache(dbEntry);
+                if (file != null && scannedFiles.ContainsKey(file.Filepath))
+                {
+                    scannedFiles[file.Filepath] = true;
+                }
+            }
+            catch (Exception ex)
+            {
+                Logger.Warn(ex.Message);
+                Logger.Warn(ex.StackTrace);
+            }
+
+            Interlocked.Increment(ref currentFileProgress);
+        });
+
+        Logger.Debug("Scanner validated existing db files");
+
+        if (ct.IsCancellationRequested) return;
+
+        // scan new files
+        Parallel.ForEach(scannedFiles.Where(c => c.Value == false), new ParallelOptions()
+        {
+            MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
+            CancellationToken = ct
+        },
+        file =>
+        {
+            if (ct.IsCancellationRequested) return;
+
+            _ = _fileDbManager.CreateFileCacheEntity(file.Key);
+            Interlocked.Increment(ref currentFileProgress);
+        });
+
+        Logger.Debug("Scanner added new files to db");
+
+        Logger.Debug("Scan complete");
+        TotalFiles = 0;
+        currentFileProgress = 0;
+
+        if (!_pluginConfiguration.InitialScanComplete)
+        {
+            _pluginConfiguration.InitialScanComplete = true;
+            _pluginConfiguration.Save();
+        }
+    }
+
+    private void StartScan()
+    {
+        if (!_ipcManager.Initialized || !_pluginConfiguration.HasValidSetup()) return;
+        Logger.Verbose("Penumbra is active, configuration is valid, starting watchers and scan");
+        InvokeScan();
+    }
+}

MareSynchronos/Managers/CachedPlayer.cs

@@ -3,32 +3,30 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
-using Dalamud.Game.ClientState.Objects.SubKinds;
 using Dalamud.Logging;
 using FFXIVClientStructs.FFXIV.Client.Game.Character;
 using MareSynchronos.API;
-using MareSynchronos.FileCacheDB;
-using MareSynchronos.Interop;
 using MareSynchronos.Models;
 using MareSynchronos.Utils;
 using MareSynchronos.WebAPI;
-using Newtonsoft.Json;
 
 namespace MareSynchronos.Managers;
 
 public class CachedPlayer
 {
     private readonly DalamudUtil _dalamudUtil;
+    private readonly FileDbManager fileDbManager;
     private readonly IpcManager _ipcManager;
     private readonly ApiController _apiController;
     private bool _isVisible;
 
-    public CachedPlayer(string nameHash, IpcManager ipcManager, ApiController apiController, DalamudUtil dalamudUtil)
+    public CachedPlayer(string nameHash, IpcManager ipcManager, ApiController apiController, DalamudUtil dalamudUtil, FileDbManager fileDbManager)
     {
         PlayerNameHash = nameHash;
         _ipcManager = ipcManager;
         _apiController = apiController;
         _dalamudUtil = dalamudUtil;
+        this.fileDbManager = fileDbManager;
     }
 
     public bool IsVisible
@@ -196,12 +194,11 @@ public class CachedPlayer
         moddedDictionary = new Dictionary<string, string>();
         try
         {
-            using var db = new FileCacheContext();
             foreach (var item in _cachedData.FileReplacements.SelectMany(k => k.Value.Where(v => string.IsNullOrEmpty(v.FileSwapPath))).ToList())
             {
                 foreach (var gamePath in item.GamePaths)
                 {
-                    var fileCache = db.FileCaches.FirstOrDefault(f => f.Hash == item.Hash);
+                    var fileCache = fileDbManager.GetFileCacheByHash(item.Hash);
                     if (fileCache != null)
                     {
                         moddedDictionary[gamePath] = fileCache.Filepath;
@@ -311,7 +308,7 @@ public class CachedPlayer
             if (companion != IntPtr.Zero)
             {
                 Logger.Debug("Request Redraw for Companion");
-                _dalamudUtil.WaitWhileCharacterIsDrawing(PlayerName! + " companion", companion, 10000,ct);
+                _dalamudUtil.WaitWhileCharacterIsDrawing(PlayerName! + " companion", companion, 10000, ct);
                 ct.ThrowIfCancellationRequested();
                 if (_ipcManager.CheckGlamourerApi() && !string.IsNullOrEmpty(glamourerData))
                 {

MareSynchronos/Managers/FileCacheManager.cs

@@ -1,381 +0,0 @@
-using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using Dalamud.Logging;
-using MareSynchronos.FileCacheDB;
-using MareSynchronos.Utils;
-using Microsoft.EntityFrameworkCore;
-
-namespace MareSynchronos.Managers
-{
-    public class FileCacheManager : IDisposable
-    {
-        private readonly IpcManager _ipcManager;
-        private readonly ConcurrentBag<string> _modifiedFiles = new();
-        private readonly Configuration _pluginConfiguration;
-        private FileSystemWatcher? _cacheDirWatcher;
-        private FileSystemWatcher? _penumbraDirWatcher;
-        private Task? _rescanTask;
-        private readonly CancellationTokenSource _rescanTaskCancellationTokenSource = new();
-        private CancellationTokenSource _rescanTaskRunCancellationTokenSource = new();
-        private CancellationTokenSource? _scanCancellationTokenSource;
-        private object modifiedFilesLock = new object();
-        public FileCacheManager(IpcManager ipcManager, Configuration pluginConfiguration)
-        {
-            Logger.Verbose("Creating " + nameof(FileCacheManager));
-
-            _ipcManager = ipcManager;
-            _pluginConfiguration = pluginConfiguration;
-
-            StartWatchersAndScan();
-
-            _ipcManager.PenumbraInitialized += StartWatchersAndScan;
-            _ipcManager.PenumbraDisposed += StopWatchersAndScan;
-        }
-
-        public long CurrentFileProgress { get; private set; }
-
-        public long FileCacheSize { get; set; }
-
-        public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
-
-        public long TotalFiles { get; private set; }
-
-        public string WatchedCacheDirectory => (_cacheDirWatcher?.EnableRaisingEvents ?? false) ? _cacheDirWatcher!.Path : "Not watched";
-
-        public string WatchedPenumbraDirectory => (_penumbraDirWatcher?.EnableRaisingEvents ?? false) ? _penumbraDirWatcher!.Path : "Not watched";
-
-        public FileCache? Create(string file, CancellationToken? token)
-        {
-            FileInfo fileInfo = new(file);
-            int attempt = 0;
-            while (IsFileLocked(fileInfo) && attempt++ <= 10)
-            {
-                Thread.Sleep(1000);
-                Logger.Debug("Waiting for file release " + fileInfo.FullName + " attempt " + attempt);
-                token?.ThrowIfCancellationRequested();
-            }
-
-            if (attempt >= 10) return null;
-
-            var sha1Hash = Crypto.GetFileHash(fileInfo.FullName);
-            return new FileCache()
-            {
-                Filepath = fileInfo.FullName.ToLowerInvariant(),
-                Hash = sha1Hash,
-                LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString(),
-            };
-        }
-
-        public void Dispose()
-        {
-            Logger.Verbose("Disposing " + nameof(FileCacheManager));
-
-            _ipcManager.PenumbraInitialized -= StartWatchersAndScan;
-            _ipcManager.PenumbraDisposed -= StopWatchersAndScan;
-            _rescanTaskCancellationTokenSource?.Cancel();
-            _rescanTaskRunCancellationTokenSource?.Cancel();
-            _scanCancellationTokenSource?.Cancel();
-
-            StopWatchersAndScan();
-        }
-
-        public void StartInitialScan()
-        {
-            _scanCancellationTokenSource = new CancellationTokenSource();
-            Task.Run(() => StartFileScan(_scanCancellationTokenSource.Token));
-        }
-
-        public void StartWatchers()
-        {
-            if (!_ipcManager.Initialized || string.IsNullOrEmpty(_pluginConfiguration.CacheFolder)) return;
-            Logger.Verbose("Starting File System Watchers");
-            _penumbraDirWatcher?.Dispose();
-            _cacheDirWatcher?.Dispose();
-
-            _penumbraDirWatcher = new FileSystemWatcher(_ipcManager.PenumbraModDirectory()!)
-            {
-                IncludeSubdirectories = true,
-            };
-            _penumbraDirWatcher.NotifyFilter = NotifyFilters.FileName | NotifyFilters.Size;
-            _penumbraDirWatcher.Deleted += OnModified;
-            _penumbraDirWatcher.Changed += OnModified;
-            _penumbraDirWatcher.Renamed += OnModified;
-            _penumbraDirWatcher.Filters.Add("*.mtrl");
-            _penumbraDirWatcher.Filters.Add("*.mdl");
-            _penumbraDirWatcher.Filters.Add("*.tex");
-            _penumbraDirWatcher.Error += (sender, args) => PluginLog.Error(args.GetException(), "Error in Penumbra Dir Watcher");
-            _penumbraDirWatcher.EnableRaisingEvents = true;
-
-            _cacheDirWatcher = new FileSystemWatcher(_pluginConfiguration.CacheFolder)
-            {
-                IncludeSubdirectories = false,
-            };
-            _cacheDirWatcher.NotifyFilter = NotifyFilters.FileName | NotifyFilters.Size;
-            _cacheDirWatcher.Deleted += OnModified;
-            _cacheDirWatcher.Changed += OnModified;
-            _cacheDirWatcher.Renamed += OnModified;
-            _cacheDirWatcher.Filters.Add("*");
-            _cacheDirWatcher.Error +=
-                (sender, args) => PluginLog.Error(args.GetException(), "Error in Cache Dir Watcher");
-            _cacheDirWatcher.EnableRaisingEvents = true;
-
-            Task.Run(RecalculateFileCacheSize);
-        }
-
-        private bool IsFileLocked(FileInfo file)
-        {
-            try
-            {
-                using var fs = file.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
-            }
-            catch
-            {
-                return true;
-            }
-
-            return false;
-        }
-
-        private void OnModified(object sender, FileSystemEventArgs e)
-        {
-            lock (modifiedFilesLock)
-            {
-                _modifiedFiles.Add(e.FullPath);
-            }
-            _ = StartRescan();
-        }
-
-        private void RecalculateFileCacheSize()
-        {
-            FileCacheSize = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder).Sum(f =>
-            {
-                try
-                {
-                    return new FileInfo(f).Length;
-                }
-                catch
-                {
-                    return 0;
-                }
-            });
-
-            if (FileCacheSize < (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024) return;
-
-            var allFiles = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder)
-                .Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList();
-            while (FileCacheSize > (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024)
-            {
-                var oldestFile = allFiles.First();
-                FileCacheSize -= oldestFile.Length;
-                File.Delete(oldestFile.FullName);
-                allFiles.Remove(oldestFile);
-            }
-        }
-
-        public async Task StartRescan(bool force = false)
-        {
-            _rescanTaskRunCancellationTokenSource.Cancel();
-            _rescanTaskRunCancellationTokenSource = new CancellationTokenSource();
-            var token = _rescanTaskRunCancellationTokenSource.Token;
-            if (!force)
-                await Task.Delay(TimeSpan.FromSeconds(1), token);
-            while ((!_rescanTask?.IsCompleted ?? false) && !token.IsCancellationRequested)
-            {
-                await Task.Delay(TimeSpan.FromSeconds(1), token);
-            }
-
-            if (token.IsCancellationRequested) return;
-
-            Logger.Debug("File changes detected");
-
-            lock (modifiedFilesLock)
-            {
-                if (!_modifiedFiles.Any()) return;
-            }
-
-            _rescanTask = Task.Run(async () =>
-            {
-                List<string> modifiedFilesCopy = new List<string>();
-                lock (modifiedFilesLock)
-                {
-                    modifiedFilesCopy = _modifiedFiles.ToList();
-                    _modifiedFiles.Clear();
-                }
-                await using var db = new FileCacheContext();
-                foreach (var item in modifiedFilesCopy.Distinct())
-                {
-                    var fi = new FileInfo(item);
-                    if (!fi.Exists)
-                    {
-                        PluginLog.Verbose("Removed: " + item);
-
-                        db.RemoveRange(db.FileCaches.Where(f => f.Filepath.ToLower() == item.ToLowerInvariant()));
-                    }
-                    else
-                    {
-                        PluginLog.Verbose("Changed :" + item);
-                        var fileCache = Create(item, _rescanTaskCancellationTokenSource.Token);
-                        if (fileCache != null)
-                        {
-                            db.RemoveRange(db.FileCaches.Where(f => f.Filepath.ToLower() == fileCache.Filepath.ToLowerInvariant()));
-                            await db.AddAsync(fileCache, _rescanTaskCancellationTokenSource.Token);
-                        }
-                    }
-                }
-
-                await db.SaveChangesAsync(_rescanTaskCancellationTokenSource.Token);
-
-                RecalculateFileCacheSize();
-            }, _rescanTaskCancellationTokenSource.Token);
-        }
-
-        private async Task StartFileScan(CancellationToken ct)
-        {
-            TotalFiles = 1;
-            _scanCancellationTokenSource = new CancellationTokenSource();
-            var penumbraDir = _ipcManager.PenumbraModDirectory()!;
-            Logger.Debug("Getting files from " + penumbraDir + " and " + _pluginConfiguration.CacheFolder);
-            var scannedFiles = new ConcurrentDictionary<string, bool>(
-                Directory.EnumerateFiles(penumbraDir, "*.*", SearchOption.AllDirectories)
-                                .Select(s => s.ToLowerInvariant())
-                                .Where(f => f.Contains(@"\chara\"))
-                                .Where(f =>
-                                    (f.EndsWith(".tex", StringComparison.OrdinalIgnoreCase)
-                                     || f.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase)
-                                     || f.EndsWith(".mtrl", StringComparison.OrdinalIgnoreCase)))
-                                .Concat(Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.AllDirectories)
-                                    .Where(f => new FileInfo(f).Name.Length == 40)
-                                    .Select(s => s.ToLowerInvariant()))
-                                .Select(p => new KeyValuePair<string, bool>(p, false)).ToList());
-            List<FileCache> fileCaches;
-            await using (var db = new FileCacheContext())
-                fileCaches = db.FileCaches.ToList();
-
-            TotalFiles = scannedFiles.Count;
-
-            var fileCachesToDelete = new ConcurrentBag<FileCache>();
-            var fileCachesToAdd = new ConcurrentBag<FileCache>();
-
-            Logger.Debug("Database contains " + fileCaches.Count + " files, local system contains " + TotalFiles);
-            // scan files from database
-            Parallel.ForEach(fileCaches, new ParallelOptions()
-            {
-                MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
-                CancellationToken = ct,
-            },
-            cache =>
-            {
-                if (ct.IsCancellationRequested) return;
-                if (!File.Exists(cache.Filepath))
-                {
-                    fileCachesToDelete.Add(cache);
-                }
-                else
-                {
-                    if (scannedFiles.ContainsKey(cache.Filepath))
-                    {
-                        scannedFiles[cache.Filepath] = true;
-                    }
-                    FileInfo fileInfo = new(cache.Filepath);
-                    if (fileInfo.LastWriteTimeUtc.Ticks == long.Parse(cache.LastModifiedDate)) return;
-                    var newCache = Create(cache.Filepath, ct);
-                    if (newCache != null)
-                    {
-                        fileCachesToAdd.Add(newCache);
-                        fileCachesToDelete.Add(cache);
-                    }
-                }
-
-                var files = CurrentFileProgress;
-                Interlocked.Increment(ref files);
-                CurrentFileProgress = files;
-            });
-
-            if (ct.IsCancellationRequested) return;
-
-            // scan new files
-            Parallel.ForEach(scannedFiles.Where(c => c.Value == false), new ParallelOptions()
-            {
-                MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
-                CancellationToken = ct
-            },
-                file =>
-                {
-                    var newCache = Create(file.Key, ct);
-                    if (newCache != null)
-                    {
-                        fileCachesToAdd.Add(newCache);
-                    }
-
-                    var files = CurrentFileProgress;
-                    Interlocked.Increment(ref files);
-                    CurrentFileProgress = files;
-                });
-
-            if (fileCachesToAdd.Any() || fileCachesToDelete.Any())
-            {
-                await using FileCacheContext db = new();
-
-                Logger.Debug("Found " + fileCachesToAdd.Count + " additions and " + fileCachesToDelete.Count + " deletions");
-                try
-                {
-                    foreach (var deletion in fileCachesToDelete)
-                    {
-                        var entries = db.FileCaches.Where(f =>
-                            f.Hash == deletion.Hash && f.Filepath.ToLower() == deletion.Filepath.ToLower());
-                        if (await entries.AnyAsync(ct))
-                        {
-                            Logger.Verbose("Removing file from DB: " + deletion.Filepath);
-                            db.FileCaches.RemoveRange(entries);
-                        }
-                    }
-                    await db.SaveChangesAsync(ct);
-                    foreach (var entry in fileCachesToAdd)
-                    {
-                        try
-                        {
-                            db.FileCaches.Add(entry);
-                        }
-                        catch
-                        {
-                            // ignored
-                        }
-                    }
-                    await db.SaveChangesAsync(ct);
-                }
-                catch (Exception ex)
-                {
-                    PluginLog.Error(ex, ex.Message);
-                }
-            }
-
-            Logger.Debug("Scan complete");
-            TotalFiles = 0;
-            CurrentFileProgress = 0;
-
-            if (!_pluginConfiguration.InitialScanComplete)
-            {
-                _pluginConfiguration.InitialScanComplete = true;
-                _pluginConfiguration.Save();
-            }
-        }
-
-        private void StartWatchersAndScan()
-        {
-            if (!_ipcManager.Initialized || !_pluginConfiguration.HasValidSetup()) return;
-            Logger.Verbose("Penumbra is active, configuration is valid, starting watchers and scan");
-            StartWatchers();
-            StartInitialScan();
-        }
-
-        private void StopWatchersAndScan()
-        {
-            _cacheDirWatcher?.Dispose();
-        }
-    }
-}

MareSynchronos/Managers/IpcManager.cs

@@ -174,11 +174,16 @@ namespace MareSynchronos.Managers
             while (actionQueue.Count > 0 && totalSleepTime < 2000)
             {
                 Logger.Verbose("Waiting for actionqueue to clear...");
+                HandleActionQueue();
                 System.Threading.Thread.Sleep(16);
                 totalSleepTime += 16;
             }
 
-            Logger.Verbose("Action queue clear or not, disposing");
+            if (totalSleepTime >= 2000)
+            {
+                Logger.Verbose("Action queue clear or not, disposing");
+            }
+
             _dalamudUtil.FrameworkUpdate -= HandleActionQueue;
             _dalamudUtil.ZoneSwitchEnd -= ClearActionQueue;
             actionQueue.Clear();
@@ -304,7 +309,7 @@ namespace MareSynchronos.Managers
         public string? PenumbraModDirectory()
         {
             if (!CheckPenumbraApi()) return null;
-            return _penumbraResolveModDir!.InvokeFunc();
+            return _penumbraResolveModDir!.InvokeFunc().ToLowerInvariant();
         }
 
         public void PenumbraRedraw(IntPtr obj)

MareSynchronos/Managers/OnlinePlayerManager.cs

@@ -17,6 +17,7 @@ public class OnlinePlayerManager : IDisposable
     private readonly DalamudUtil _dalamudUtil;
     private readonly IpcManager _ipcManager;
     private readonly PlayerManager _playerManager;
+    private readonly FileDbManager _fileDbManager;
     private readonly ConcurrentDictionary<string, CachedPlayer> _onlineCachedPlayers = new();
     private readonly ConcurrentDictionary<string, CharacterCacheDto> _temporaryStoredCharacterCache = new();
     private readonly ConcurrentDictionary<CachedPlayer, CancellationTokenSource> _playerTokenDisposal = new();
@@ -24,7 +25,7 @@ public class OnlinePlayerManager : IDisposable
     private List<string> OnlineVisiblePlayerHashes => _onlineCachedPlayers.Select(p => p.Value).Where(p => p.PlayerCharacter != IntPtr.Zero)
         .Select(p => p.PlayerNameHash).ToList();
 
-    public OnlinePlayerManager(ApiController apiController, DalamudUtil dalamudUtil, IpcManager ipcManager, PlayerManager playerManager)
+    public OnlinePlayerManager(ApiController apiController, DalamudUtil dalamudUtil, IpcManager ipcManager, PlayerManager playerManager, FileDbManager fileDbManager)
     {
         Logger.Verbose("Creating " + nameof(OnlinePlayerManager));
 
@@ -32,7 +33,7 @@ public class OnlinePlayerManager : IDisposable
         _dalamudUtil = dalamudUtil;
         _ipcManager = ipcManager;
         _playerManager = playerManager;
-
+        _fileDbManager = fileDbManager;
         _apiController.PairedClientOnline += ApiControllerOnPairedClientOnline;
         _apiController.PairedClientOffline += ApiControllerOnPairedClientOffline;
         _apiController.PairedWithOther += ApiControllerOnPairedWithOther;
@@ -249,6 +250,6 @@ public class OnlinePlayerManager : IDisposable
 
     private CachedPlayer CreateCachedPlayer(string hashedName)
     {
-        return new CachedPlayer(hashedName, _ipcManager, _apiController, _dalamudUtil);
+        return new CachedPlayer(hashedName, _ipcManager, _apiController, _dalamudUtil, _fileDbManager);
     }
 }

MareSynchronos/Managers/TransientResourceManager.cs

@@ -101,7 +101,7 @@ namespace MareSynchronos.Managers
                 filePath = filePath.Split("|")[2];
             }
 
-            filePath = filePath.ToLowerInvariant();
+            filePath = filePath.ToLowerInvariant().Replace("\\", "/");
 
             var replacedGamePath = gamePath.ToLowerInvariant().Replace("\\", "/");
 

MareSynchronos/Models/FileReplacement.cs

@@ -9,16 +9,17 @@ using System.IO;
 using MareSynchronos.API;
 using MareSynchronos.Utils;
 using System.Text.RegularExpressions;
+using MareSynchronos.Managers;
 
 namespace MareSynchronos.Models
 {
     public class FileReplacement
     {
-        private readonly string _penumbraDirectory;
+        private readonly FileDbManager fileDbManager;
 
-        public FileReplacement(string penumbraDirectory)
+        public FileReplacement(FileDbManager fileDbManager)
         {
-            _penumbraDirectory = penumbraDirectory;
+            this.fileDbManager = fileDbManager;
         }
 
         public bool Computed => IsFileSwap || !HasFileReplacement || !string.IsNullOrEmpty(Hash);
@@ -35,38 +36,14 @@ namespace MareSynchronos.Models
 
         public void SetResolvedPath(string path)
         {
-            ResolvedPath = path.ToLowerInvariant().Replace('\\', '/');//.Replace('/', '\\').Replace(_penumbraDirectory, "").Replace('\\', '/');
+            ResolvedPath = path.ToLowerInvariant().Replace('\\', '/');
             if (!HasFileReplacement || IsFileSwap) return;
 
             _ = Task.Run(() =>
             {
-                FileCache? fileCache;
+                var cache = fileDbManager.GetFileCacheByPath(ResolvedPath);
-                using (FileCacheContext db = new())
+                cache ??= fileDbManager.CreateFileCacheEntity(ResolvedPath);
-                {
+                Hash = cache.OriginalHash;
-                    fileCache = db.FileCaches.FirstOrDefault(f => f.Filepath == path.Replace('/', '\\').ToLowerInvariant());
-                }
-
-                if (fileCache != null)
-                {
-                    FileInfo fi = new(fileCache.Filepath);
-                    if (fi.LastWriteTimeUtc.Ticks == long.Parse(fileCache.LastModifiedDate))
-                    {
-                        Hash = fileCache.Hash;
-                    }
-                    else
-                    {
-                        Hash = ComputeHash(fi);
-                        using var db = new FileCacheContext();
-                        var newTempCache = db.FileCaches.Single(f => f.Filepath == path.Replace('/', '\\').ToLowerInvariant());
-                        newTempCache.Hash = Hash;
-                        db.Update(newTempCache);
-                        db.SaveChanges();
-                    }
-                }
-                else
-                {
-                    Hash = ComputeHash(new FileInfo(path.Replace('/', '\\').ToLowerInvariant()));
-                }
             });
         }
 
@@ -85,33 +62,5 @@ namespace MareSynchronos.Models
             builder.AppendLine($"Modded: {HasFileReplacement} - {string.Join(",", GamePaths)} => {ResolvedPath}");
             return builder.ToString();
         }
-
-        private string ComputeHash(FileInfo fi)
-        {
-            // compute hash if hash is not present
-            string hash = Crypto.GetFileHash(fi.FullName);
-
-            using FileCacheContext db = new();
-            var fileAddedDuringCompute = db.FileCaches.FirstOrDefault(f => f.Filepath == fi.FullName.ToLowerInvariant());
-            if (fileAddedDuringCompute != null) return fileAddedDuringCompute.Hash;
-
-            try
-            {
-                Logger.Debug("Adding new file to DB: " + fi.FullName + ", " + hash);
-                db.Add(new FileCache()
-                {
-                    Hash = hash,
-                    Filepath = fi.FullName.ToLowerInvariant(),
-                    LastModifiedDate = fi.LastWriteTimeUtc.Ticks.ToString()
-                });
-                db.SaveChanges();
-            }
-            catch (Exception ex)
-            {
-                PluginLog.Error(ex, "Error adding files to database. Most likely not an issue though.");
-            }
-
-            return hash;
-        }
     }
 }

MareSynchronos/Plugin.cs

@@ -25,7 +25,7 @@ namespace MareSynchronos
         private readonly CommandManager _commandManager;
         private readonly Framework _framework;
         private readonly Configuration _configuration;
-        private readonly FileCacheManager _fileCacheManager;
+        private readonly PeriodicFileScanner _fileCacheManager;
         private readonly IntroUi _introUi;
         private readonly IpcManager _ipcManager;
         public static DalamudPluginInterface PluginInterface { get; set; }
@@ -37,6 +37,7 @@ namespace MareSynchronos
         private OnlinePlayerManager? _characterCacheManager;
         private readonly DownloadUi _downloadUi;
         private readonly FileDialogManager _fileDialogManager;
+        private readonly FileDbManager _fileDbManager;
         private readonly CompactUi _compactUi;
         private readonly UiShared _uiSharedComponent;
         private readonly Dalamud.Localization _localization;
@@ -63,15 +64,16 @@ namespace MareSynchronos
             // those can be initialized outside of game login
             _dalamudUtil = new DalamudUtil(clientState, objectTable, framework, condition);
 
-            _apiController = new ApiController(_configuration, _dalamudUtil);
             _ipcManager = new IpcManager(PluginInterface, _dalamudUtil);
 
             // Compatibility for FileSystemWatchers under OSX
             if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
                 Environment.SetEnvironmentVariable("MONO_MANAGED_WATCHER", "enabled");
 
-            _fileCacheManager = new FileCacheManager(_ipcManager, _configuration);
             _fileDialogManager = new FileDialogManager();
+            _fileDbManager = new FileDbManager(_ipcManager, _configuration);
+            _apiController = new ApiController(_configuration, _dalamudUtil, _fileDbManager);
+            _fileCacheManager = new PeriodicFileScanner(_ipcManager, _configuration, _fileDbManager, _apiController);
 
             _uiSharedComponent =
                 new UiShared(_ipcManager, _apiController, _fileCacheManager, _fileDialogManager, _configuration, _dalamudUtil, PluginInterface, _localization);
@@ -101,7 +103,6 @@ namespace MareSynchronos
 
             _dalamudUtil.LogIn += DalamudUtilOnLogIn;
             _dalamudUtil.LogOut += DalamudUtilOnLogOut;
-            _apiController.RegisterFinalized += ApiControllerOnRegisterFinalized;
 
             if (_dalamudUtil.IsLoggedIn)
             {
@@ -109,17 +110,10 @@ namespace MareSynchronos
             }
         }
 
-        private void ApiControllerOnRegisterFinalized()
-        {
-            _introUi.IsOpen = false;
-            _compactUi.IsOpen = true;
-        }
-
         public string Name => "Mare Synchronos";
         public void Dispose()
         {
             Logger.Verbose("Disposing " + Name);
-            _apiController.RegisterFinalized -= ApiControllerOnRegisterFinalized;
             _apiController?.Dispose();
 
             _commandManager.RemoveHandler(CommandName);
@@ -133,9 +127,9 @@ namespace MareSynchronos
             _compactUi?.Dispose();
 
             _fileCacheManager?.Dispose();
-            _ipcManager?.Dispose();
             _playerManager?.Dispose();
             _characterCacheManager?.Dispose();
+            _ipcManager?.Dispose();
             _transientResourceManager?.Dispose();
             _dalamudUtil.Dispose();
             Logger.Debug("Shut down");
@@ -195,11 +189,11 @@ namespace MareSynchronos
             {
                 _transientResourceManager = new TransientResourceManager(_ipcManager, _dalamudUtil);
                 var characterCacheFactory =
-                    new CharacterDataFactory(_dalamudUtil, _ipcManager, _transientResourceManager);
+                    new CharacterDataFactory(_dalamudUtil, _ipcManager, _transientResourceManager, _fileDbManager);
                 _playerManager = new PlayerManager(_apiController, _ipcManager,
                     characterCacheFactory, _dalamudUtil, _transientResourceManager);
                 _characterCacheManager = new OnlinePlayerManager(_apiController,
-                    _dalamudUtil, _ipcManager, _playerManager);
+                    _dalamudUtil, _ipcManager, _playerManager, _fileDbManager);
             }
             catch (Exception ex)
             {

MareSynchronos/UI/CompactUI.cs

@@ -52,6 +52,7 @@ namespace MareSynchronos.UI
                 Logger.Warn(ex.StackTrace);
             }
             this.WindowName = "Mare Synchronos " + dateTime + "###MareSynchronosMainUI";
+            Toggle();
 #else
             this.WindowName = "Mare Synchronos " + Assembly.GetExecutingAssembly().GetName().Version;
 #endif

MareSynchronos/UI/IntroUI.cs

@@ -7,10 +7,10 @@ using System.Threading.Tasks;
 using Dalamud.Interface.Colors;
 using Dalamud.Interface.Windowing;
 using ImGuiNET;
-using MareSynchronos.Managers;
 using MareSynchronos.Utils;
 using MareSynchronos.Localization;
 using Dalamud.Utility;
+using MareSynchronos.FileCacheDB;
 
 namespace MareSynchronos.UI
 {
@@ -18,7 +18,7 @@ namespace MareSynchronos.UI
     {
         private readonly UiShared _uiShared;
         private readonly Configuration _pluginConfiguration;
-        private readonly FileCacheManager _fileCacheManager;
+        private readonly PeriodicFileScanner _fileCacheManager;
         private readonly WindowSystem _windowSystem;
         private bool _readFirstPage;
 
@@ -53,7 +53,7 @@ namespace MareSynchronos.UI
         }
 
         public IntroUi(WindowSystem windowSystem, UiShared uiShared, Configuration pluginConfiguration,
-            FileCacheManager fileCacheManager) : base("Mare Synchronos Setup")
+            PeriodicFileScanner fileCacheManager) : base("Mare Synchronos Setup")
         {
             Logger.Verbose("Creating " + nameof(IntroUi));
 
@@ -226,7 +226,7 @@ namespace MareSynchronos.UI
 
                     if (ImGui.Button("Start Scan##startScan"))
                     {
-                        _fileCacheManager.StartInitialScan();
+                        _fileCacheManager.InvokeScan();
                     }
                 }
                 else

MareSynchronos/UI/SettingsUi.cs

@@ -586,6 +586,8 @@ namespace MareSynchronos.UI
         private void DrawFileCacheSettings()
         {
             _uiShared.DrawFileScanState();
+            _uiShared.DrawParallelScansSetting();
+            _uiShared.DrawTimeSpanBetweenScansSetting();
             _uiShared.DrawCacheDirectorySetting();
             ImGui.Text($"Local cache size: {UiShared.ByteToString(_uiShared.FileCacheSize)}");
             ImGui.SameLine();
@@ -597,6 +599,8 @@ namespace MareSynchronos.UI
                     {
                         File.Delete(file);
                     }
+
+                    _uiShared.RecalculateFileCacheSize();
                 });
             }
         }

MareSynchronos/UI/UIShared.cs

@@ -11,6 +11,7 @@ using Dalamud.Interface.ImGuiFileDialog;
 using Dalamud.Plugin;
 using Dalamud.Utility;
 using ImGuiNET;
+using MareSynchronos.FileCacheDB;
 using MareSynchronos.Localization;
 using MareSynchronos.Managers;
 using MareSynchronos.Utils;
@@ -25,13 +26,13 @@ namespace MareSynchronos.UI
 
         private readonly IpcManager _ipcManager;
         private readonly ApiController _apiController;
-        private readonly FileCacheManager _fileCacheManager;
+        private readonly PeriodicFileScanner _cacheScanner;
         private readonly FileDialogManager _fileDialogManager;
         private readonly Configuration _pluginConfiguration;
         private readonly DalamudUtil _dalamudUtil;
         private readonly DalamudPluginInterface _pluginInterface;
         private readonly Dalamud.Localization _localization;
-        public long FileCacheSize => _fileCacheManager.FileCacheSize;
+        public long FileCacheSize => _cacheScanner.FileCacheSize;
         public string PlayerName => _dalamudUtil.PlayerName;
         public bool HasValidPenumbraModPath => !(_ipcManager.PenumbraModDirectory() ?? string.Empty).IsNullOrEmpty() && Directory.Exists(_ipcManager.PenumbraModDirectory());
         public bool EditTrackerPosition { get; set; }
@@ -42,11 +43,12 @@ namespace MareSynchronos.UI
 
         public ApiController ApiController => _apiController;
 
-        public UiShared(IpcManager ipcManager, ApiController apiController, FileCacheManager fileCacheManager, FileDialogManager fileDialogManager, Configuration pluginConfiguration, DalamudUtil dalamudUtil, DalamudPluginInterface pluginInterface, Dalamud.Localization localization)
+        public UiShared(IpcManager ipcManager, ApiController apiController, PeriodicFileScanner cacheScanner, FileDialogManager fileDialogManager,
+            Configuration pluginConfiguration, DalamudUtil dalamudUtil, DalamudPluginInterface pluginInterface, Dalamud.Localization localization)
         {
             _ipcManager = ipcManager;
             _apiController = apiController;
-            _fileCacheManager = fileCacheManager;
+            _cacheScanner = cacheScanner;
             _fileDialogManager = fileDialogManager;
             _pluginConfiguration = pluginConfiguration;
             _dalamudUtil = dalamudUtil;
@@ -144,19 +146,23 @@ namespace MareSynchronos.UI
         public void DrawFileScanState()
         {
             ImGui.Text("File Scanner Status");
-            if (_fileCacheManager.IsScanRunning)
+            ImGui.SameLine();
+            if (_cacheScanner.IsScanRunning)
             {
                 ImGui.Text("Scan is running");
                 ImGui.Text("Current Progress:");
                 ImGui.SameLine();
-                ImGui.Text(_fileCacheManager.TotalFiles <= 0
+                ImGui.Text(_cacheScanner.TotalFiles <= 1
                     ? "Collecting files"
-                    : $"Processing {_fileCacheManager.CurrentFileProgress} / {_fileCacheManager.TotalFiles} files");
+                    : $"Processing {_cacheScanner.CurrentFileProgress} / {_cacheScanner.TotalFiles} files");
+            }
+            else if (_pluginConfiguration.FileScanPaused)
+            {
+                ImGui.Text("File scanner is paused");
             }
             else
             {
-                ImGui.Text("Watching Penumbra Directory: " + _fileCacheManager.WatchedPenumbraDirectory);
+                ImGui.Text("Next scan in " + _cacheScanner.TimeUntilNextScan);
-                ImGui.Text("Watching Cache Directory: " + _fileCacheManager.WatchedCacheDirectory);
             }
         }
 
@@ -444,7 +450,7 @@ namespace MareSynchronos.UI
                     {
                         _pluginConfiguration.CacheFolder = path;
                         _pluginConfiguration.Save();
-                        _fileCacheManager.StartWatchers();
+                        _cacheScanner.StartWatchers();
                     }
                 });
             }
@@ -474,6 +480,7 @@ namespace MareSynchronos.UI
                 _pluginConfiguration.MaxLocalCacheInGiB = maxCacheSize;
                 _pluginConfiguration.Save();
             }
+            DrawHelpText("The cache is automatically governed by Mare. It will clear itself automatically once it reaches the set capacity by removing the oldest unused files. You typically do not need to clear it yourself.");
         }
 
         private bool _isDirectoryWritable = false;
@@ -503,14 +510,38 @@ namespace MareSynchronos.UI
             }
         }
 
+        public void RecalculateFileCacheSize()
+        {
+            _cacheScanner.InvokeScan();
+        }
+
         public void DrawParallelScansSetting()
         {
             var parallelScans = _pluginConfiguration.MaxParallelScan;
-            if (ImGui.SliderInt("Parallel File Scans##parallelism", ref parallelScans, 1, 20))
+            if (ImGui.SliderInt("File scan parallelism##parallelism", ref parallelScans, 1, 20))
             {
                 _pluginConfiguration.MaxParallelScan = parallelScans;
                 _pluginConfiguration.Save();
             }
+            DrawHelpText("Decrease to lessen load of file scans. File scans will take longer to execute with less parallel threads.");
+        }
+
+        public void DrawTimeSpanBetweenScansSetting()
+        {
+            var timeSpan = _pluginConfiguration.TimeSpanBetweenScansInSeconds;
+            if (ImGui.SliderInt("Seconds between scans##timespan", ref timeSpan, 20, 60))
+            {
+                _pluginConfiguration.TimeSpanBetweenScansInSeconds = timeSpan;
+                _pluginConfiguration.Save();
+            }
+            DrawHelpText("This is the time in seconds between file scans. Increase it to reduce system load. A too high setting can cause issues when manually fumbling about in the cache or Penumbra mods folders.");
+            var isPaused = _pluginConfiguration.FileScanPaused;
+            if (ImGui.Checkbox("Pause periodic file scan##filescanpause", ref isPaused))
+            {
+                _pluginConfiguration.FileScanPaused = isPaused;
+                _pluginConfiguration.Save();
+            }
+            DrawHelpText("This allows you to stop the periodic scans of your Penumbra and Mare cache directories. This setting will automatically revert itself on restart of Mare.");
         }
 
         public void Dispose()

MareSynchronos/WebAPI/ApIController.Functions.Files.cs

@@ -5,7 +5,6 @@ using System.Linq;
 using System.Net;
 using System.Runtime.CompilerServices;
 using System.Text;
-using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
 using LZ4;
@@ -59,7 +58,13 @@ namespace MareSynchronos.WebAPI
 
             string fileName = Path.GetTempFileName();
 
-            await wc.DownloadFileTaskAsync(downloadUri, fileName);
+            ct.Register(wc.CancelAsync);
+
+            try
+            {
+                await wc.DownloadFileTaskAsync(downloadUri, fileName);
+            }
+            catch { }
 
             CurrentDownloads[downloadId].Single(f => f.Hash == hash).Transferred = CurrentDownloads[downloadId].Single(f => f.Hash == hash).Total;
 
@@ -71,6 +76,7 @@ namespace MareSynchronos.WebAPI
 
         public async Task DownloadFiles(int currentDownloadId, List<FileReplacementDto> fileReplacementDto, CancellationToken ct)
         {
+            DownloadStarted?.Invoke();
             Logger.Debug("Downloading files (Download ID " + currentDownloadId + ")");
 
             List<DownloadFileDto> downloadFileInfoFromService = new List<DownloadFileDto>();
@@ -89,23 +95,29 @@ namespace MareSynchronos.WebAPI
                 }
             }
 
-            foreach (var file in CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred))
+            await Parallel.ForEachAsync(CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred), new ParallelOptions()
+            {
+                MaxDegreeOfParallelism = 5,
+                CancellationToken = ct
+            },
+            async (file, token) =>
             {
                 var hash = file.Hash;
-                var tempFile = await DownloadFile(currentDownloadId, file.Hash, file.DownloadUri, ct);
+                var tempFile = await DownloadFile(currentDownloadId, file.Hash, file.DownloadUri, token);
-                if (ct.IsCancellationRequested)
+                if (token.IsCancellationRequested)
                 {
                     File.Delete(tempFile);
                     Logger.Debug("Detected cancellation, removing " + currentDownloadId);
+                    DownloadFinished?.Invoke();
                     CancelDownload(currentDownloadId);
-                    break;
+                    return;
                 }
 
-                var tempFileData = await File.ReadAllBytesAsync(tempFile, ct);
+                var tempFileData = await File.ReadAllBytesAsync(tempFile, token);
                 var extractedFile = LZ4Codec.Unwrap(tempFileData);
                 File.Delete(tempFile);
                 var filePath = Path.Combine(_pluginConfiguration.CacheFolder, file.Hash);
-                await File.WriteAllBytesAsync(filePath, extractedFile, ct);
+                await File.WriteAllBytesAsync(filePath, extractedFile, token);
                 var fi = new FileInfo(filePath);
                 Func<DateTime> RandomDayFunc()
                 {
@@ -118,26 +130,20 @@ namespace MareSynchronos.WebAPI
                 fi.CreationTime = RandomDayFunc().Invoke();
                 fi.LastAccessTime = RandomDayFunc().Invoke();
                 fi.LastWriteTime = RandomDayFunc().Invoke();
-            }
+                try
-
-            var allFilesInDb = false;
-            while (!allFilesInDb && !ct.IsCancellationRequested)
-            {
-                await using (var db = new FileCacheContext())
                 {
-                    var fileCount = CurrentDownloads[currentDownloadId]
+                    _ = _fileDbManager.CreateFileCacheEntity(filePath);
-                        .Where(c => c.CanBeTransferred)
-                        .Count(h => db.FileCaches.Any(f => f.Hash == h.Hash));
-                    var totalFiles = CurrentDownloads[currentDownloadId].Count(c => c.CanBeTransferred);
-                    Logger.Debug("Waiting for files to be in the DB, added " + fileCount + " of " + totalFiles);
-
-                    allFilesInDb = fileCount == totalFiles;
                 }
-
+                catch (Exception ex)
-                await Task.Delay(250, ct);
+                {
-            }
+                    Logger.Warn("Issue adding file to the DB");
+                    Logger.Warn(ex.Message);
+                    Logger.Warn(ex.StackTrace);
+                }
+            });
 
             Logger.Debug("Download complete, removing " + currentDownloadId);
+            DownloadFinished?.Invoke();
             CancelDownload(currentDownloadId);
         }
 

MareSynchronos/WebAPI/ApiController.Connectivity.cs

@@ -6,6 +6,7 @@ using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using MareSynchronos.API;
+using MareSynchronos.Managers;
 using MareSynchronos.Utils;
 using MareSynchronos.WebAPI.Utils;
 using Microsoft.AspNetCore.Http.Connections;
@@ -35,7 +36,7 @@ namespace MareSynchronos.WebAPI
 
         private readonly Configuration _pluginConfiguration;
         private readonly DalamudUtil _dalamudUtil;
-
+        private readonly FileDbManager _fileDbManager;
         private CancellationTokenSource _connectionCancellationTokenSource;
 
         private HubConnection? _mareHub;
@@ -48,12 +49,13 @@ namespace MareSynchronos.WebAPI
 
         public bool IsAdmin => _connectionDto?.IsAdmin ?? false;
 
-        public ApiController(Configuration pluginConfiguration, DalamudUtil dalamudUtil)
+        public ApiController(Configuration pluginConfiguration, DalamudUtil dalamudUtil, FileDbManager fileDbManager)
         {
             Logger.Verbose("Creating " + nameof(ApiController));
 
             _pluginConfiguration = pluginConfiguration;
             _dalamudUtil = dalamudUtil;
+            _fileDbManager = fileDbManager;
             _connectionCancellationTokenSource = new CancellationTokenSource();
             _dalamudUtil.LogIn += DalamudUtilOnLogIn;
             _dalamudUtil.LogOut += DalamudUtilOnLogOut;
@@ -80,8 +82,6 @@ namespace MareSynchronos.WebAPI
 
         public event EventHandler<CharacterReceivedEventArgs>? CharacterReceived;
 
-        public event VoidDelegate? RegisterFinalized;
-
         public event VoidDelegate? Connected;
 
         public event VoidDelegate? Disconnected;
@@ -93,6 +93,8 @@ namespace MareSynchronos.WebAPI
         public event SimpleStringDelegate? PairedWithOther;
 
         public event SimpleStringDelegate? UnpairedFromOther;
+        public event VoidDelegate? DownloadStarted;
+        public event VoidDelegate? DownloadFinished;
 
         public ConcurrentDictionary<int, List<DownloadFileTransfer>> CurrentDownloads { get; } = new();
 
@@ -313,7 +315,7 @@ namespace MareSynchronos.WebAPI
                 .WithAutomaticReconnect(new ForeverRetryPolicy())
                 .ConfigureLogging(a => {
                     a.ClearProviders().AddProvider(new DalamudLoggingProvider());
-                    a.SetMinimumLevel(LogLevel.Trace);
+                    a.SetMinimumLevel(LogLevel.Warning);
                     })
                 .Build();
         }