From 0d7e173a973a5d24211d96994ec5b73d53489081 Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Sun, 25 Sep 2022 14:38:06 +0200 Subject: [PATCH] add periodic file scanner, parallelize downloads, fix transient files being readded when not necessary, fix disposal of players on plugin shutdown --- MareSynchronos/Configuration.cs | 9 + .../Factories/CharacterDataFactory.cs | 7 +- MareSynchronos/FileCacheDB/FileCache.cs | 34 +- .../FileCacheDB/FileCacheContext.cs | 6 +- MareSynchronos/FileCacheDB/FileCacheEntity.cs | 13 + MareSynchronos/FileCacheDB/FileDbManager.cs | 227 +++++++++++ .../FileCacheDB/PeriodicFileScanner.cs | 233 +++++++++++ MareSynchronos/Managers/CachedPlayer.cs | 13 +- MareSynchronos/Managers/FileCacheManager.cs | 381 ------------------ MareSynchronos/Managers/IpcManager.cs | 9 +- .../Managers/OnlinePlayerManager.cs | 7 +- .../Managers/TransientResourceManager.cs | 2 +- MareSynchronos/Models/FileReplacement.cs | 67 +-- MareSynchronos/Plugin.cs | 26 +- MareSynchronos/UI/CompactUI.cs | 1 + MareSynchronos/UI/IntroUI.cs | 8 +- MareSynchronos/UI/SettingsUi.cs | 4 + MareSynchronos/UI/UIShared.cs | 53 ++- .../WebAPI/ApIController.Functions.Files.cs | 54 +-- .../WebAPI/ApiController.Connectivity.cs | 12 +- 20 files changed, 641 insertions(+), 525 deletions(-) create mode 100644 MareSynchronos/FileCacheDB/FileCacheEntity.cs create mode 100644 MareSynchronos/FileCacheDB/FileDbManager.cs create mode 100644 MareSynchronos/FileCacheDB/PeriodicFileScanner.cs delete mode 100644 MareSynchronos/Managers/FileCacheManager.cs diff --git a/MareSynchronos/Configuration.cs b/MareSynchronos/Configuration.cs index e9371d8..dfed875 100644 --- a/MareSynchronos/Configuration.cs +++ b/MareSynchronos/Configuration.cs @@ -60,6 +60,9 @@ namespace MareSynchronos public bool ReverseUserSort { get; set; } = true; + public int TimeSpanBetweenScansInSeconds { get; set; } = 30; + public bool FileScanPaused { get; set; } = false; + public bool InitialScanComplete { get; set; } = false; public int MaxParallelScan { @@ -207,6 +210,12 @@ namespace MareSynchronos Version = 5; Save(); } + + if (FileScanPaused) + { + FileScanPaused = false; + Save(); + } } } } diff --git a/MareSynchronos/Factories/CharacterDataFactory.cs b/MareSynchronos/Factories/CharacterDataFactory.cs index 5106325..357ca4f 100644 --- a/MareSynchronos/Factories/CharacterDataFactory.cs +++ b/MareSynchronos/Factories/CharacterDataFactory.cs @@ -23,11 +23,12 @@ public class CharacterDataFactory private readonly DalamudUtil _dalamudUtil; private readonly IpcManager _ipcManager; private readonly TransientResourceManager transientResourceManager; + private readonly FileDbManager fileDbManager; - public CharacterDataFactory(DalamudUtil dalamudUtil, IpcManager ipcManager, TransientResourceManager transientResourceManager) + public CharacterDataFactory(DalamudUtil dalamudUtil, IpcManager ipcManager, TransientResourceManager transientResourceManager, FileDbManager fileDbManager) { Logger.Verbose("Creating " + nameof(CharacterDataFactory)); - + this.fileDbManager = fileDbManager; _dalamudUtil = dalamudUtil; _ipcManager = ipcManager; this.transientResourceManager = transientResourceManager; @@ -416,7 +417,7 @@ public class CharacterDataFactory private FileReplacement CreateFileReplacement(string path, bool doNotReverseResolve = false) { - var fileReplacement = new FileReplacement(_ipcManager.PenumbraModDirectory()!); + var fileReplacement = new FileReplacement(fileDbManager); if (!doNotReverseResolve) { fileReplacement.GamePaths = diff --git a/MareSynchronos/FileCacheDB/FileCache.cs b/MareSynchronos/FileCacheDB/FileCache.cs index c25a3fd..ce85835 100644 --- a/MareSynchronos/FileCacheDB/FileCache.cs +++ b/MareSynchronos/FileCacheDB/FileCache.cs @@ -1,12 +1,36 @@ #nullable disable + namespace MareSynchronos.FileCacheDB { - public partial class FileCache + + public class FileCache { - public string Hash { get; set; } - public string Filepath { get; set; } - public string LastModifiedDate { get; set; } - public int Version { get; set; } + private FileCacheEntity entity; + public string Filepath { get; private set; } + public string Hash { get; private set; } + public string OriginalFilepath => entity.Filepath; + public string OriginalHash => entity.Hash; + public long LastModifiedDateTicks => long.Parse(entity.LastModifiedDate); + + public FileCache(FileCacheEntity entity) + { + this.entity = entity; + } + + public void SetResolvedFilePath(string filePath) + { + Filepath = filePath.ToLowerInvariant(); + } + + public void SetHash(string hash) + { + Hash = hash; + } + + public void UpdateFileCache(FileCacheEntity entity) + { + this.entity = entity; + } } } diff --git a/MareSynchronos/FileCacheDB/FileCacheContext.cs b/MareSynchronos/FileCacheDB/FileCacheContext.cs index 133fc92..b8fb318 100644 --- a/MareSynchronos/FileCacheDB/FileCacheContext.cs +++ b/MareSynchronos/FileCacheDB/FileCacheContext.cs @@ -9,7 +9,7 @@ namespace MareSynchronos.FileCacheDB public partial class FileCacheContext : DbContext { private string DbPath { get; set; } - public FileCacheContext() + public FileCacheContext() { DbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "FileCache.db"); string oldDbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "FileCacheDebug.db"); @@ -36,7 +36,7 @@ namespace MareSynchronos.FileCacheDB { } - public virtual DbSet FileCaches { get; set; } + public virtual DbSet FileCaches { get; set; } protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { @@ -48,7 +48,7 @@ namespace MareSynchronos.FileCacheDB protected override void OnModelCreating(ModelBuilder modelBuilder) { - modelBuilder.Entity(entity => + modelBuilder.Entity(entity => { entity.HasKey(e => new { e.Hash, e.Filepath }); diff --git a/MareSynchronos/FileCacheDB/FileCacheEntity.cs b/MareSynchronos/FileCacheDB/FileCacheEntity.cs new file mode 100644 index 0000000..e0deffc --- /dev/null +++ b/MareSynchronos/FileCacheDB/FileCacheEntity.cs @@ -0,0 +1,13 @@ +#nullable disable + + +namespace MareSynchronos.FileCacheDB +{ + public partial class FileCacheEntity + { + public string Hash { get; set; } + public string Filepath { get; set; } + public string LastModifiedDate { get; set; } + public int Version { get; set; } + } +} diff --git a/MareSynchronos/FileCacheDB/FileDbManager.cs b/MareSynchronos/FileCacheDB/FileDbManager.cs new file mode 100644 index 0000000..558695a --- /dev/null +++ b/MareSynchronos/FileCacheDB/FileDbManager.cs @@ -0,0 +1,227 @@ +using MareSynchronos.FileCacheDB; +using MareSynchronos.Utils; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; + +namespace MareSynchronos.Managers; + +public class FileDbManager +{ + private const string PenumbraPrefix = "{penumbra}"; + private const string CachePrefix = "{cache}"; + private readonly IpcManager _ipcManager; + private readonly Configuration _configuration; + private static object _lock = new(); + + public FileDbManager(IpcManager ipcManager, Configuration configuration) + { + _ipcManager = ipcManager; + _configuration = configuration; + } + + public FileCache? GetFileCacheByHash(string hash) + { + List matchingEntries = new List(); + using (var db = new FileCacheContext()) + { + matchingEntries = db.FileCaches.Where(f => f.Hash.ToLower() == hash.ToLower()).ToList(); + } + + if (!matchingEntries.Any()) return null; + + if (matchingEntries.Any(f => f.Filepath.Contains(PenumbraPrefix) && matchingEntries.Any(f => f.Filepath.Contains(CachePrefix)))) + { + var cachedEntries = matchingEntries.Where(f => f.Filepath.Contains(CachePrefix)); + DeleteFromDatabase(cachedEntries.Select(f => new FileCache(f))); + foreach (var entry in cachedEntries) + { + matchingEntries.Remove(entry); + } + } + + return GetValidatedFileCache(matchingEntries.First()); + } + + public FileCache? ValidateFileCache(FileCacheEntity fileCacheEntity) + { + return GetValidatedFileCache(fileCacheEntity); + } + + public FileCache? GetFileCacheByPath(string path) + { + FileCacheEntity? matchingEntries = null; + var cleanedPath = path.Replace("/", "\\").ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), ""); + using (var db = new FileCacheContext()) + { + matchingEntries = db.FileCaches.FirstOrDefault(f => f.Filepath.EndsWith(cleanedPath)); + } + + if (matchingEntries == null) + { + return CreateFileCacheEntity(path); + } + + var validatedCacheEntry = GetValidatedFileCache(matchingEntries); + + return validatedCacheEntry; + } + + public FileCache? CreateFileCacheEntity(string path) + { + Logger.Verbose("Creating entry for " + path); + FileInfo fi = new FileInfo(path); + if (!fi.Exists) return null; + string prefixedPath = fi.FullName.ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), PenumbraPrefix + "\\") + .Replace(_configuration.CacheFolder.ToLowerInvariant(), CachePrefix + "\\").Replace("\\\\", "\\"); + var hash = Crypto.GetFileHash(path); + lock (_lock) + { + var entity = new FileCacheEntity(); + entity.Hash = hash; + entity.Filepath = prefixedPath; + entity.LastModifiedDate = fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture); + try + { + using var db = new FileCacheContext(); + db.FileCaches.Add(entity); + db.SaveChanges(); + } + catch (Exception ex) + { + Logger.Warn("Could not add " + path); + } + } + return GetFileCacheByPath(prefixedPath)!; + } + + private FileCache? GetValidatedFileCache(FileCacheEntity e) + { + var fileCache = new FileCache(e); + + var resulingFileCache = MigrateLegacy(fileCache); + + if (resulingFileCache == null) return null; + + resulingFileCache = ReplacePathPrefixes(resulingFileCache); + + resulingFileCache = Validate(resulingFileCache); + return resulingFileCache; + } + + private FileCache? Validate(FileCache fileCache) + { + var file = new FileInfo(fileCache.Filepath); + if (!file.Exists) + { + DeleteFromDatabase(new[] { fileCache }); + return null; + } + + if (file.LastWriteTimeUtc.Ticks != fileCache.LastModifiedDateTicks) + { + fileCache.SetHash(Crypto.GetFileHash(fileCache.Filepath)); + UpdateCacheHash(fileCache, file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture)); + } + + return fileCache; + } + + private FileCache? MigrateLegacy(FileCache fileCache) + { + if (fileCache.OriginalFilepath.Contains(PenumbraPrefix) || fileCache.OriginalFilepath.Contains(CachePrefix)) return fileCache; + + var fileInfo = new FileInfo(fileCache.OriginalFilepath); + var penumbraDir = _ipcManager.PenumbraModDirectory()!; + // check if it's a cache file + if (fileInfo.Exists && fileInfo.Name.Length == 40) + { + MigrateLegacyFilePath(fileCache, CachePrefix + "\\" + fileInfo.Name.ToLower()); + } + else if (fileInfo.Exists && fileInfo.FullName.ToLowerInvariant().Contains(penumbraDir)) + { + // attempt to replace penumbra mod folder path with {penumbra} + var newPath = PenumbraPrefix + fileCache.OriginalFilepath.ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!, string.Empty); + MigrateLegacyFilePath(fileCache, newPath); + } + else + { + DeleteFromDatabase(new[] { fileCache }); + return null; + } + + return fileCache; + } + + private FileCache ReplacePathPrefixes(FileCache fileCache) + { + if (fileCache.OriginalFilepath.Contains(PenumbraPrefix)) + { + fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory())); + } + else if (fileCache.OriginalFilepath.Contains(CachePrefix)) + { + fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(CachePrefix, _configuration.CacheFolder)); + } + + return fileCache; + } + + private void UpdateCacheHash(FileCache markedForUpdate, string lastModifiedDate) + { + lock (_lock) + { + Logger.Verbose("Updating Hash for " + markedForUpdate.OriginalFilepath); + using var db = new FileCacheContext(); + var cache = db.FileCaches.First(f => f.Filepath == markedForUpdate.OriginalFilepath && f.Hash == markedForUpdate.OriginalHash); + var newcache = new FileCacheEntity() + { + Filepath = cache.Filepath, + Hash = markedForUpdate.Hash, + LastModifiedDate = lastModifiedDate + }; + db.Remove(cache); + db.FileCaches.Add(newcache); + markedForUpdate.UpdateFileCache(newcache); + db.SaveChanges(); + } + } + + private void MigrateLegacyFilePath(FileCache fileCacheToMigrate, string newPath) + { + lock (_lock) + { + Logger.Verbose("Migrating legacy file path for " + fileCacheToMigrate.OriginalFilepath); + using var db = new FileCacheContext(); + var cache = db.FileCaches.First(f => f.Filepath == fileCacheToMigrate.OriginalFilepath && f.Hash == fileCacheToMigrate.OriginalHash); + var newcache = new FileCacheEntity() + { + Filepath = newPath, + Hash = cache.Hash, + LastModifiedDate = cache.LastModifiedDate + }; + db.Remove(cache); + db.FileCaches.Add(newcache); + fileCacheToMigrate.UpdateFileCache(newcache); + db.SaveChanges(); + } + } + + private void DeleteFromDatabase(IEnumerable markedForDeletion) + { + lock (_lock) + { + using var db = new FileCacheContext(); + foreach (var item in markedForDeletion) + { + Logger.Verbose("Removing " + item.OriginalFilepath); + var itemToRemove = db.FileCaches.FirstOrDefault(f => f.Hash == item.OriginalHash && f.Filepath == item.OriginalFilepath); + if (itemToRemove == null) continue; + db.FileCaches.Remove(itemToRemove); + } + db.SaveChanges(); + } + } +} diff --git a/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs b/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs new file mode 100644 index 0000000..ce1deae --- /dev/null +++ b/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs @@ -0,0 +1,233 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MareSynchronos.Managers; +using MareSynchronos.Utils; +using MareSynchronos.WebAPI; +using Microsoft.EntityFrameworkCore; + +namespace MareSynchronos.FileCacheDB; + +public class PeriodicFileScanner : IDisposable +{ + private readonly IpcManager _ipcManager; + private readonly Configuration _pluginConfiguration; + private readonly FileDbManager _fileDbManager; + private readonly ApiController _apiController; + private CancellationTokenSource? _scanCancellationTokenSource; + private Task? _fileScannerTask = null; + public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileDbManager fileDbManager, ApiController apiController) + { + Logger.Verbose("Creating " + nameof(PeriodicFileScanner)); + + _ipcManager = ipcManager; + _pluginConfiguration = pluginConfiguration; + _fileDbManager = fileDbManager; + _apiController = apiController; + _ipcManager.PenumbraInitialized += StartScan; + if (!string.IsNullOrEmpty(_ipcManager.PenumbraModDirectory())) + { + StartScan(); + } + _apiController.DownloadStarted += _apiController_DownloadStarted; + _apiController.DownloadFinished += _apiController_DownloadFinished; + } + + private void _apiController_DownloadFinished() + { + InvokeScan(); + } + + private void _apiController_DownloadStarted() + { + _scanCancellationTokenSource?.Cancel(); + } + + private long currentFileProgress = 0; + public long CurrentFileProgress => currentFileProgress; + + public long FileCacheSize { get; set; } + + public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0; + + public long TotalFiles { get; private set; } + + public string TimeUntilNextScan => _timeUntilNextScan.ToString(@"mm\:ss"); + private TimeSpan _timeUntilNextScan = TimeSpan.Zero; + private int timeBetweenScans => _pluginConfiguration.TimeSpanBetweenScansInSeconds; + + public void Dispose() + { + Logger.Verbose("Disposing " + nameof(PeriodicFileScanner)); + + _ipcManager.PenumbraInitialized -= StartScan; + _apiController.DownloadStarted -= _apiController_DownloadStarted; + _apiController.DownloadFinished -= _apiController_DownloadFinished; + _scanCancellationTokenSource?.Cancel(); + } + + public void InvokeScan() + { + _scanCancellationTokenSource?.Cancel(); + _scanCancellationTokenSource = new CancellationTokenSource(); + var token = _scanCancellationTokenSource.Token; + _fileScannerTask = Task.Run(async () => + { + while (!token.IsCancellationRequested) + { + RecalculateFileCacheSize(); + if (!_pluginConfiguration.FileScanPaused) + { + await PeriodicFileScan(token); + } + _timeUntilNextScan = TimeSpan.FromSeconds(timeBetweenScans); + while (_timeUntilNextScan.TotalSeconds >= 0) + { + await Task.Delay(TimeSpan.FromSeconds(1), token); + _timeUntilNextScan -= TimeSpan.FromSeconds(1); + } + } + }); + } + + internal void StartWatchers() + { + InvokeScan(); + } + + public void RecalculateFileCacheSize() + { + FileCacheSize = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder).Sum(f => + { + try + { + return new FileInfo(f).Length; + } + catch + { + return 0; + } + }); + + if (FileCacheSize < (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024) return; + + var allFiles = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder) + .Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList(); + while (FileCacheSize > (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024) + { + var oldestFile = allFiles.First(); + FileCacheSize -= oldestFile.Length; + File.Delete(oldestFile.FullName); + allFiles.Remove(oldestFile); + } + } + + private async Task PeriodicFileScan(CancellationToken ct) + { + TotalFiles = 1; + var penumbraDir = _ipcManager.PenumbraModDirectory(); + bool penDirExists = true; + bool cacheDirExists = true; + if (string.IsNullOrEmpty(penumbraDir) || !Directory.Exists(penumbraDir)) + { + penDirExists = false; + Logger.Warn("Penumbra directory is not set or does not exist."); + } + if (string.IsNullOrEmpty(_pluginConfiguration.CacheFolder) || !Directory.Exists(_pluginConfiguration.CacheFolder)) + { + cacheDirExists = false; + Logger.Warn("Mare Cache directory is not set or does not exist."); + } + if (!penDirExists || !cacheDirExists) + { + return; + } + + Logger.Debug("Getting files from " + penumbraDir + " and " + _pluginConfiguration.CacheFolder); + string[] ext = { ".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp" }; + var scannedFiles = new ConcurrentDictionary( + Directory.EnumerateFiles(penumbraDir, "*.*", SearchOption.AllDirectories) + .Select(s => new FileInfo(s)) + .Where(f => ext.Contains(f.Extension) && !f.FullName.Contains(@"\bg\") && !f.FullName.Contains(@"\bgcommon\") && !f.FullName.Contains(@"\ui\")) + .Select(f => f.FullName.ToLowerInvariant()) + .Concat(Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.AllDirectories) + .Where(f => new FileInfo(f).Name.Length == 40) + .Select(s => s.ToLowerInvariant())) + .Select(p => new KeyValuePair(p, false)).ToList()); + List fileDbEntries; + using (var db = new FileCacheContext()) + { + fileDbEntries = await db.FileCaches.ToListAsync(cancellationToken: ct); + } + + TotalFiles = scannedFiles.Count; + + Logger.Debug("Database contains " + fileDbEntries.Count + " files, local system contains " + TotalFiles); + // scan files from database + Parallel.ForEach(fileDbEntries.ToList(), new ParallelOptions() + { + MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan, + CancellationToken = ct, + }, + dbEntry => + { + if (ct.IsCancellationRequested) return; + try + { + var file = _fileDbManager.ValidateFileCache(dbEntry); + if (file != null && scannedFiles.ContainsKey(file.Filepath)) + { + scannedFiles[file.Filepath] = true; + } + } + catch (Exception ex) + { + Logger.Warn(ex.Message); + Logger.Warn(ex.StackTrace); + } + + Interlocked.Increment(ref currentFileProgress); + }); + + Logger.Debug("Scanner validated existing db files"); + + if (ct.IsCancellationRequested) return; + + // scan new files + Parallel.ForEach(scannedFiles.Where(c => c.Value == false), new ParallelOptions() + { + MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan, + CancellationToken = ct + }, + file => + { + if (ct.IsCancellationRequested) return; + + _ = _fileDbManager.CreateFileCacheEntity(file.Key); + Interlocked.Increment(ref currentFileProgress); + }); + + Logger.Debug("Scanner added new files to db"); + + Logger.Debug("Scan complete"); + TotalFiles = 0; + currentFileProgress = 0; + + if (!_pluginConfiguration.InitialScanComplete) + { + _pluginConfiguration.InitialScanComplete = true; + _pluginConfiguration.Save(); + } + } + + private void StartScan() + { + if (!_ipcManager.Initialized || !_pluginConfiguration.HasValidSetup()) return; + Logger.Verbose("Penumbra is active, configuration is valid, starting watchers and scan"); + InvokeScan(); + } +} diff --git a/MareSynchronos/Managers/CachedPlayer.cs b/MareSynchronos/Managers/CachedPlayer.cs index e4738b4..b12cc07 100644 --- a/MareSynchronos/Managers/CachedPlayer.cs +++ b/MareSynchronos/Managers/CachedPlayer.cs @@ -3,32 +3,30 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; -using Dalamud.Game.ClientState.Objects.SubKinds; using Dalamud.Logging; using FFXIVClientStructs.FFXIV.Client.Game.Character; using MareSynchronos.API; -using MareSynchronos.FileCacheDB; -using MareSynchronos.Interop; using MareSynchronos.Models; using MareSynchronos.Utils; using MareSynchronos.WebAPI; -using Newtonsoft.Json; namespace MareSynchronos.Managers; public class CachedPlayer { private readonly DalamudUtil _dalamudUtil; + private readonly FileDbManager fileDbManager; private readonly IpcManager _ipcManager; private readonly ApiController _apiController; private bool _isVisible; - public CachedPlayer(string nameHash, IpcManager ipcManager, ApiController apiController, DalamudUtil dalamudUtil) + public CachedPlayer(string nameHash, IpcManager ipcManager, ApiController apiController, DalamudUtil dalamudUtil, FileDbManager fileDbManager) { PlayerNameHash = nameHash; _ipcManager = ipcManager; _apiController = apiController; _dalamudUtil = dalamudUtil; + this.fileDbManager = fileDbManager; } public bool IsVisible @@ -196,12 +194,11 @@ public class CachedPlayer moddedDictionary = new Dictionary(); try { - using var db = new FileCacheContext(); foreach (var item in _cachedData.FileReplacements.SelectMany(k => k.Value.Where(v => string.IsNullOrEmpty(v.FileSwapPath))).ToList()) { foreach (var gamePath in item.GamePaths) { - var fileCache = db.FileCaches.FirstOrDefault(f => f.Hash == item.Hash); + var fileCache = fileDbManager.GetFileCacheByHash(item.Hash); if (fileCache != null) { moddedDictionary[gamePath] = fileCache.Filepath; @@ -311,7 +308,7 @@ public class CachedPlayer if (companion != IntPtr.Zero) { Logger.Debug("Request Redraw for Companion"); - _dalamudUtil.WaitWhileCharacterIsDrawing(PlayerName! + " companion", companion, 10000,ct); + _dalamudUtil.WaitWhileCharacterIsDrawing(PlayerName! + " companion", companion, 10000, ct); ct.ThrowIfCancellationRequested(); if (_ipcManager.CheckGlamourerApi() && !string.IsNullOrEmpty(glamourerData)) { diff --git a/MareSynchronos/Managers/FileCacheManager.cs b/MareSynchronos/Managers/FileCacheManager.cs deleted file mode 100644 index b8ffcac..0000000 --- a/MareSynchronos/Managers/FileCacheManager.cs +++ /dev/null @@ -1,381 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Dalamud.Logging; -using MareSynchronos.FileCacheDB; -using MareSynchronos.Utils; -using Microsoft.EntityFrameworkCore; - -namespace MareSynchronos.Managers -{ - public class FileCacheManager : IDisposable - { - private readonly IpcManager _ipcManager; - private readonly ConcurrentBag _modifiedFiles = new(); - private readonly Configuration _pluginConfiguration; - private FileSystemWatcher? _cacheDirWatcher; - private FileSystemWatcher? _penumbraDirWatcher; - private Task? _rescanTask; - private readonly CancellationTokenSource _rescanTaskCancellationTokenSource = new(); - private CancellationTokenSource _rescanTaskRunCancellationTokenSource = new(); - private CancellationTokenSource? _scanCancellationTokenSource; - private object modifiedFilesLock = new object(); - public FileCacheManager(IpcManager ipcManager, Configuration pluginConfiguration) - { - Logger.Verbose("Creating " + nameof(FileCacheManager)); - - _ipcManager = ipcManager; - _pluginConfiguration = pluginConfiguration; - - StartWatchersAndScan(); - - _ipcManager.PenumbraInitialized += StartWatchersAndScan; - _ipcManager.PenumbraDisposed += StopWatchersAndScan; - } - - public long CurrentFileProgress { get; private set; } - - public long FileCacheSize { get; set; } - - public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0; - - public long TotalFiles { get; private set; } - - public string WatchedCacheDirectory => (_cacheDirWatcher?.EnableRaisingEvents ?? false) ? _cacheDirWatcher!.Path : "Not watched"; - - public string WatchedPenumbraDirectory => (_penumbraDirWatcher?.EnableRaisingEvents ?? false) ? _penumbraDirWatcher!.Path : "Not watched"; - - public FileCache? Create(string file, CancellationToken? token) - { - FileInfo fileInfo = new(file); - int attempt = 0; - while (IsFileLocked(fileInfo) && attempt++ <= 10) - { - Thread.Sleep(1000); - Logger.Debug("Waiting for file release " + fileInfo.FullName + " attempt " + attempt); - token?.ThrowIfCancellationRequested(); - } - - if (attempt >= 10) return null; - - var sha1Hash = Crypto.GetFileHash(fileInfo.FullName); - return new FileCache() - { - Filepath = fileInfo.FullName.ToLowerInvariant(), - Hash = sha1Hash, - LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString(), - }; - } - - public void Dispose() - { - Logger.Verbose("Disposing " + nameof(FileCacheManager)); - - _ipcManager.PenumbraInitialized -= StartWatchersAndScan; - _ipcManager.PenumbraDisposed -= StopWatchersAndScan; - _rescanTaskCancellationTokenSource?.Cancel(); - _rescanTaskRunCancellationTokenSource?.Cancel(); - _scanCancellationTokenSource?.Cancel(); - - StopWatchersAndScan(); - } - - public void StartInitialScan() - { - _scanCancellationTokenSource = new CancellationTokenSource(); - Task.Run(() => StartFileScan(_scanCancellationTokenSource.Token)); - } - - public void StartWatchers() - { - if (!_ipcManager.Initialized || string.IsNullOrEmpty(_pluginConfiguration.CacheFolder)) return; - Logger.Verbose("Starting File System Watchers"); - _penumbraDirWatcher?.Dispose(); - _cacheDirWatcher?.Dispose(); - - _penumbraDirWatcher = new FileSystemWatcher(_ipcManager.PenumbraModDirectory()!) - { - IncludeSubdirectories = true, - }; - _penumbraDirWatcher.NotifyFilter = NotifyFilters.FileName | NotifyFilters.Size; - _penumbraDirWatcher.Deleted += OnModified; - _penumbraDirWatcher.Changed += OnModified; - _penumbraDirWatcher.Renamed += OnModified; - _penumbraDirWatcher.Filters.Add("*.mtrl"); - _penumbraDirWatcher.Filters.Add("*.mdl"); - _penumbraDirWatcher.Filters.Add("*.tex"); - _penumbraDirWatcher.Error += (sender, args) => PluginLog.Error(args.GetException(), "Error in Penumbra Dir Watcher"); - _penumbraDirWatcher.EnableRaisingEvents = true; - - _cacheDirWatcher = new FileSystemWatcher(_pluginConfiguration.CacheFolder) - { - IncludeSubdirectories = false, - }; - _cacheDirWatcher.NotifyFilter = NotifyFilters.FileName | NotifyFilters.Size; - _cacheDirWatcher.Deleted += OnModified; - _cacheDirWatcher.Changed += OnModified; - _cacheDirWatcher.Renamed += OnModified; - _cacheDirWatcher.Filters.Add("*"); - _cacheDirWatcher.Error += - (sender, args) => PluginLog.Error(args.GetException(), "Error in Cache Dir Watcher"); - _cacheDirWatcher.EnableRaisingEvents = true; - - Task.Run(RecalculateFileCacheSize); - } - - private bool IsFileLocked(FileInfo file) - { - try - { - using var fs = file.Open(FileMode.Open, FileAccess.Read, FileShare.Read); - } - catch - { - return true; - } - - return false; - } - - private void OnModified(object sender, FileSystemEventArgs e) - { - lock (modifiedFilesLock) - { - _modifiedFiles.Add(e.FullPath); - } - _ = StartRescan(); - } - - private void RecalculateFileCacheSize() - { - FileCacheSize = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder).Sum(f => - { - try - { - return new FileInfo(f).Length; - } - catch - { - return 0; - } - }); - - if (FileCacheSize < (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024) return; - - var allFiles = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder) - .Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList(); - while (FileCacheSize > (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024) - { - var oldestFile = allFiles.First(); - FileCacheSize -= oldestFile.Length; - File.Delete(oldestFile.FullName); - allFiles.Remove(oldestFile); - } - } - - public async Task StartRescan(bool force = false) - { - _rescanTaskRunCancellationTokenSource.Cancel(); - _rescanTaskRunCancellationTokenSource = new CancellationTokenSource(); - var token = _rescanTaskRunCancellationTokenSource.Token; - if (!force) - await Task.Delay(TimeSpan.FromSeconds(1), token); - while ((!_rescanTask?.IsCompleted ?? false) && !token.IsCancellationRequested) - { - await Task.Delay(TimeSpan.FromSeconds(1), token); - } - - if (token.IsCancellationRequested) return; - - Logger.Debug("File changes detected"); - - lock (modifiedFilesLock) - { - if (!_modifiedFiles.Any()) return; - } - - _rescanTask = Task.Run(async () => - { - List modifiedFilesCopy = new List(); - lock (modifiedFilesLock) - { - modifiedFilesCopy = _modifiedFiles.ToList(); - _modifiedFiles.Clear(); - } - await using var db = new FileCacheContext(); - foreach (var item in modifiedFilesCopy.Distinct()) - { - var fi = new FileInfo(item); - if (!fi.Exists) - { - PluginLog.Verbose("Removed: " + item); - - db.RemoveRange(db.FileCaches.Where(f => f.Filepath.ToLower() == item.ToLowerInvariant())); - } - else - { - PluginLog.Verbose("Changed :" + item); - var fileCache = Create(item, _rescanTaskCancellationTokenSource.Token); - if (fileCache != null) - { - db.RemoveRange(db.FileCaches.Where(f => f.Filepath.ToLower() == fileCache.Filepath.ToLowerInvariant())); - await db.AddAsync(fileCache, _rescanTaskCancellationTokenSource.Token); - } - } - } - - await db.SaveChangesAsync(_rescanTaskCancellationTokenSource.Token); - - RecalculateFileCacheSize(); - }, _rescanTaskCancellationTokenSource.Token); - } - - private async Task StartFileScan(CancellationToken ct) - { - TotalFiles = 1; - _scanCancellationTokenSource = new CancellationTokenSource(); - var penumbraDir = _ipcManager.PenumbraModDirectory()!; - Logger.Debug("Getting files from " + penumbraDir + " and " + _pluginConfiguration.CacheFolder); - var scannedFiles = new ConcurrentDictionary( - Directory.EnumerateFiles(penumbraDir, "*.*", SearchOption.AllDirectories) - .Select(s => s.ToLowerInvariant()) - .Where(f => f.Contains(@"\chara\")) - .Where(f => - (f.EndsWith(".tex", StringComparison.OrdinalIgnoreCase) - || f.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase) - || f.EndsWith(".mtrl", StringComparison.OrdinalIgnoreCase))) - .Concat(Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.AllDirectories) - .Where(f => new FileInfo(f).Name.Length == 40) - .Select(s => s.ToLowerInvariant())) - .Select(p => new KeyValuePair(p, false)).ToList()); - List fileCaches; - await using (var db = new FileCacheContext()) - fileCaches = db.FileCaches.ToList(); - - TotalFiles = scannedFiles.Count; - - var fileCachesToDelete = new ConcurrentBag(); - var fileCachesToAdd = new ConcurrentBag(); - - Logger.Debug("Database contains " + fileCaches.Count + " files, local system contains " + TotalFiles); - // scan files from database - Parallel.ForEach(fileCaches, new ParallelOptions() - { - MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan, - CancellationToken = ct, - }, - cache => - { - if (ct.IsCancellationRequested) return; - if (!File.Exists(cache.Filepath)) - { - fileCachesToDelete.Add(cache); - } - else - { - if (scannedFiles.ContainsKey(cache.Filepath)) - { - scannedFiles[cache.Filepath] = true; - } - FileInfo fileInfo = new(cache.Filepath); - if (fileInfo.LastWriteTimeUtc.Ticks == long.Parse(cache.LastModifiedDate)) return; - var newCache = Create(cache.Filepath, ct); - if (newCache != null) - { - fileCachesToAdd.Add(newCache); - fileCachesToDelete.Add(cache); - } - } - - var files = CurrentFileProgress; - Interlocked.Increment(ref files); - CurrentFileProgress = files; - }); - - if (ct.IsCancellationRequested) return; - - // scan new files - Parallel.ForEach(scannedFiles.Where(c => c.Value == false), new ParallelOptions() - { - MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan, - CancellationToken = ct - }, - file => - { - var newCache = Create(file.Key, ct); - if (newCache != null) - { - fileCachesToAdd.Add(newCache); - } - - var files = CurrentFileProgress; - Interlocked.Increment(ref files); - CurrentFileProgress = files; - }); - - if (fileCachesToAdd.Any() || fileCachesToDelete.Any()) - { - await using FileCacheContext db = new(); - - Logger.Debug("Found " + fileCachesToAdd.Count + " additions and " + fileCachesToDelete.Count + " deletions"); - try - { - foreach (var deletion in fileCachesToDelete) - { - var entries = db.FileCaches.Where(f => - f.Hash == deletion.Hash && f.Filepath.ToLower() == deletion.Filepath.ToLower()); - if (await entries.AnyAsync(ct)) - { - Logger.Verbose("Removing file from DB: " + deletion.Filepath); - db.FileCaches.RemoveRange(entries); - } - } - await db.SaveChangesAsync(ct); - foreach (var entry in fileCachesToAdd) - { - try - { - db.FileCaches.Add(entry); - } - catch - { - // ignored - } - } - await db.SaveChangesAsync(ct); - } - catch (Exception ex) - { - PluginLog.Error(ex, ex.Message); - } - } - - Logger.Debug("Scan complete"); - TotalFiles = 0; - CurrentFileProgress = 0; - - if (!_pluginConfiguration.InitialScanComplete) - { - _pluginConfiguration.InitialScanComplete = true; - _pluginConfiguration.Save(); - } - } - - private void StartWatchersAndScan() - { - if (!_ipcManager.Initialized || !_pluginConfiguration.HasValidSetup()) return; - Logger.Verbose("Penumbra is active, configuration is valid, starting watchers and scan"); - StartWatchers(); - StartInitialScan(); - } - - private void StopWatchersAndScan() - { - _cacheDirWatcher?.Dispose(); - } - } -} diff --git a/MareSynchronos/Managers/IpcManager.cs b/MareSynchronos/Managers/IpcManager.cs index 23e8be1..47186af 100644 --- a/MareSynchronos/Managers/IpcManager.cs +++ b/MareSynchronos/Managers/IpcManager.cs @@ -174,11 +174,16 @@ namespace MareSynchronos.Managers while (actionQueue.Count > 0 && totalSleepTime < 2000) { Logger.Verbose("Waiting for actionqueue to clear..."); + HandleActionQueue(); System.Threading.Thread.Sleep(16); totalSleepTime += 16; } - Logger.Verbose("Action queue clear or not, disposing"); + if (totalSleepTime >= 2000) + { + Logger.Verbose("Action queue clear or not, disposing"); + } + _dalamudUtil.FrameworkUpdate -= HandleActionQueue; _dalamudUtil.ZoneSwitchEnd -= ClearActionQueue; actionQueue.Clear(); @@ -304,7 +309,7 @@ namespace MareSynchronos.Managers public string? PenumbraModDirectory() { if (!CheckPenumbraApi()) return null; - return _penumbraResolveModDir!.InvokeFunc(); + return _penumbraResolveModDir!.InvokeFunc().ToLowerInvariant(); } public void PenumbraRedraw(IntPtr obj) diff --git a/MareSynchronos/Managers/OnlinePlayerManager.cs b/MareSynchronos/Managers/OnlinePlayerManager.cs index cc97c8d..e648584 100644 --- a/MareSynchronos/Managers/OnlinePlayerManager.cs +++ b/MareSynchronos/Managers/OnlinePlayerManager.cs @@ -17,6 +17,7 @@ public class OnlinePlayerManager : IDisposable private readonly DalamudUtil _dalamudUtil; private readonly IpcManager _ipcManager; private readonly PlayerManager _playerManager; + private readonly FileDbManager _fileDbManager; private readonly ConcurrentDictionary _onlineCachedPlayers = new(); private readonly ConcurrentDictionary _temporaryStoredCharacterCache = new(); private readonly ConcurrentDictionary _playerTokenDisposal = new(); @@ -24,7 +25,7 @@ public class OnlinePlayerManager : IDisposable private List OnlineVisiblePlayerHashes => _onlineCachedPlayers.Select(p => p.Value).Where(p => p.PlayerCharacter != IntPtr.Zero) .Select(p => p.PlayerNameHash).ToList(); - public OnlinePlayerManager(ApiController apiController, DalamudUtil dalamudUtil, IpcManager ipcManager, PlayerManager playerManager) + public OnlinePlayerManager(ApiController apiController, DalamudUtil dalamudUtil, IpcManager ipcManager, PlayerManager playerManager, FileDbManager fileDbManager) { Logger.Verbose("Creating " + nameof(OnlinePlayerManager)); @@ -32,7 +33,7 @@ public class OnlinePlayerManager : IDisposable _dalamudUtil = dalamudUtil; _ipcManager = ipcManager; _playerManager = playerManager; - + _fileDbManager = fileDbManager; _apiController.PairedClientOnline += ApiControllerOnPairedClientOnline; _apiController.PairedClientOffline += ApiControllerOnPairedClientOffline; _apiController.PairedWithOther += ApiControllerOnPairedWithOther; @@ -249,6 +250,6 @@ public class OnlinePlayerManager : IDisposable private CachedPlayer CreateCachedPlayer(string hashedName) { - return new CachedPlayer(hashedName, _ipcManager, _apiController, _dalamudUtil); + return new CachedPlayer(hashedName, _ipcManager, _apiController, _dalamudUtil, _fileDbManager); } } \ No newline at end of file diff --git a/MareSynchronos/Managers/TransientResourceManager.cs b/MareSynchronos/Managers/TransientResourceManager.cs index 63de47c..560355f 100644 --- a/MareSynchronos/Managers/TransientResourceManager.cs +++ b/MareSynchronos/Managers/TransientResourceManager.cs @@ -101,7 +101,7 @@ namespace MareSynchronos.Managers filePath = filePath.Split("|")[2]; } - filePath = filePath.ToLowerInvariant(); + filePath = filePath.ToLowerInvariant().Replace("\\", "/"); var replacedGamePath = gamePath.ToLowerInvariant().Replace("\\", "/"); diff --git a/MareSynchronos/Models/FileReplacement.cs b/MareSynchronos/Models/FileReplacement.cs index 7e27510..4d19864 100644 --- a/MareSynchronos/Models/FileReplacement.cs +++ b/MareSynchronos/Models/FileReplacement.cs @@ -9,16 +9,17 @@ using System.IO; using MareSynchronos.API; using MareSynchronos.Utils; using System.Text.RegularExpressions; +using MareSynchronos.Managers; namespace MareSynchronos.Models { public class FileReplacement { - private readonly string _penumbraDirectory; + private readonly FileDbManager fileDbManager; - public FileReplacement(string penumbraDirectory) + public FileReplacement(FileDbManager fileDbManager) { - _penumbraDirectory = penumbraDirectory; + this.fileDbManager = fileDbManager; } public bool Computed => IsFileSwap || !HasFileReplacement || !string.IsNullOrEmpty(Hash); @@ -35,38 +36,14 @@ namespace MareSynchronos.Models public void SetResolvedPath(string path) { - ResolvedPath = path.ToLowerInvariant().Replace('\\', '/');//.Replace('/', '\\').Replace(_penumbraDirectory, "").Replace('\\', '/'); + ResolvedPath = path.ToLowerInvariant().Replace('\\', '/'); if (!HasFileReplacement || IsFileSwap) return; _ = Task.Run(() => { - FileCache? fileCache; - using (FileCacheContext db = new()) - { - fileCache = db.FileCaches.FirstOrDefault(f => f.Filepath == path.Replace('/', '\\').ToLowerInvariant()); - } - - if (fileCache != null) - { - FileInfo fi = new(fileCache.Filepath); - if (fi.LastWriteTimeUtc.Ticks == long.Parse(fileCache.LastModifiedDate)) - { - Hash = fileCache.Hash; - } - else - { - Hash = ComputeHash(fi); - using var db = new FileCacheContext(); - var newTempCache = db.FileCaches.Single(f => f.Filepath == path.Replace('/', '\\').ToLowerInvariant()); - newTempCache.Hash = Hash; - db.Update(newTempCache); - db.SaveChanges(); - } - } - else - { - Hash = ComputeHash(new FileInfo(path.Replace('/', '\\').ToLowerInvariant())); - } + var cache = fileDbManager.GetFileCacheByPath(ResolvedPath); + cache ??= fileDbManager.CreateFileCacheEntity(ResolvedPath); + Hash = cache.OriginalHash; }); } @@ -85,33 +62,5 @@ namespace MareSynchronos.Models builder.AppendLine($"Modded: {HasFileReplacement} - {string.Join(",", GamePaths)} => {ResolvedPath}"); return builder.ToString(); } - - private string ComputeHash(FileInfo fi) - { - // compute hash if hash is not present - string hash = Crypto.GetFileHash(fi.FullName); - - using FileCacheContext db = new(); - var fileAddedDuringCompute = db.FileCaches.FirstOrDefault(f => f.Filepath == fi.FullName.ToLowerInvariant()); - if (fileAddedDuringCompute != null) return fileAddedDuringCompute.Hash; - - try - { - Logger.Debug("Adding new file to DB: " + fi.FullName + ", " + hash); - db.Add(new FileCache() - { - Hash = hash, - Filepath = fi.FullName.ToLowerInvariant(), - LastModifiedDate = fi.LastWriteTimeUtc.Ticks.ToString() - }); - db.SaveChanges(); - } - catch (Exception ex) - { - PluginLog.Error(ex, "Error adding files to database. Most likely not an issue though."); - } - - return hash; - } } } diff --git a/MareSynchronos/Plugin.cs b/MareSynchronos/Plugin.cs index 91d80fe..524dd61 100644 --- a/MareSynchronos/Plugin.cs +++ b/MareSynchronos/Plugin.cs @@ -25,7 +25,7 @@ namespace MareSynchronos private readonly CommandManager _commandManager; private readonly Framework _framework; private readonly Configuration _configuration; - private readonly FileCacheManager _fileCacheManager; + private readonly PeriodicFileScanner _fileCacheManager; private readonly IntroUi _introUi; private readonly IpcManager _ipcManager; public static DalamudPluginInterface PluginInterface { get; set; } @@ -37,6 +37,7 @@ namespace MareSynchronos private OnlinePlayerManager? _characterCacheManager; private readonly DownloadUi _downloadUi; private readonly FileDialogManager _fileDialogManager; + private readonly FileDbManager _fileDbManager; private readonly CompactUi _compactUi; private readonly UiShared _uiSharedComponent; private readonly Dalamud.Localization _localization; @@ -52,7 +53,7 @@ namespace MareSynchronos _configuration = PluginInterface.GetPluginConfig() as Configuration ?? new Configuration(); _configuration.Initialize(PluginInterface); _configuration.Migrate(); - + _localization = new Dalamud.Localization("MareSynchronos.Localization.", "", true); _localization.SetupWithLangCode("en"); @@ -63,15 +64,16 @@ namespace MareSynchronos // those can be initialized outside of game login _dalamudUtil = new DalamudUtil(clientState, objectTable, framework, condition); - _apiController = new ApiController(_configuration, _dalamudUtil); _ipcManager = new IpcManager(PluginInterface, _dalamudUtil); // Compatibility for FileSystemWatchers under OSX if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) Environment.SetEnvironmentVariable("MONO_MANAGED_WATCHER", "enabled"); - _fileCacheManager = new FileCacheManager(_ipcManager, _configuration); _fileDialogManager = new FileDialogManager(); + _fileDbManager = new FileDbManager(_ipcManager, _configuration); + _apiController = new ApiController(_configuration, _dalamudUtil, _fileDbManager); + _fileCacheManager = new PeriodicFileScanner(_ipcManager, _configuration, _fileDbManager, _apiController); _uiSharedComponent = new UiShared(_ipcManager, _apiController, _fileCacheManager, _fileDialogManager, _configuration, _dalamudUtil, PluginInterface, _localization); @@ -101,7 +103,6 @@ namespace MareSynchronos _dalamudUtil.LogIn += DalamudUtilOnLogIn; _dalamudUtil.LogOut += DalamudUtilOnLogOut; - _apiController.RegisterFinalized += ApiControllerOnRegisterFinalized; if (_dalamudUtil.IsLoggedIn) { @@ -109,23 +110,16 @@ namespace MareSynchronos } } - private void ApiControllerOnRegisterFinalized() - { - _introUi.IsOpen = false; - _compactUi.IsOpen = true; - } - public string Name => "Mare Synchronos"; public void Dispose() { Logger.Verbose("Disposing " + Name); - _apiController.RegisterFinalized -= ApiControllerOnRegisterFinalized; _apiController?.Dispose(); _commandManager.RemoveHandler(CommandName); _dalamudUtil.LogIn -= DalamudUtilOnLogIn; _dalamudUtil.LogOut -= DalamudUtilOnLogOut; - + _uiSharedComponent.Dispose(); _settingsUi?.Dispose(); _introUi?.Dispose(); @@ -133,9 +127,9 @@ namespace MareSynchronos _compactUi?.Dispose(); _fileCacheManager?.Dispose(); - _ipcManager?.Dispose(); _playerManager?.Dispose(); _characterCacheManager?.Dispose(); + _ipcManager?.Dispose(); _transientResourceManager?.Dispose(); _dalamudUtil.Dispose(); Logger.Debug("Shut down"); @@ -195,11 +189,11 @@ namespace MareSynchronos { _transientResourceManager = new TransientResourceManager(_ipcManager, _dalamudUtil); var characterCacheFactory = - new CharacterDataFactory(_dalamudUtil, _ipcManager, _transientResourceManager); + new CharacterDataFactory(_dalamudUtil, _ipcManager, _transientResourceManager, _fileDbManager); _playerManager = new PlayerManager(_apiController, _ipcManager, characterCacheFactory, _dalamudUtil, _transientResourceManager); _characterCacheManager = new OnlinePlayerManager(_apiController, - _dalamudUtil, _ipcManager, _playerManager); + _dalamudUtil, _ipcManager, _playerManager, _fileDbManager); } catch (Exception ex) { diff --git a/MareSynchronos/UI/CompactUI.cs b/MareSynchronos/UI/CompactUI.cs index 79bfc8e..6818fd2 100644 --- a/MareSynchronos/UI/CompactUI.cs +++ b/MareSynchronos/UI/CompactUI.cs @@ -52,6 +52,7 @@ namespace MareSynchronos.UI Logger.Warn(ex.StackTrace); } this.WindowName = "Mare Synchronos " + dateTime + "###MareSynchronosMainUI"; + Toggle(); #else this.WindowName = "Mare Synchronos " + Assembly.GetExecutingAssembly().GetName().Version; #endif diff --git a/MareSynchronos/UI/IntroUI.cs b/MareSynchronos/UI/IntroUI.cs index 053bf09..442fbb0 100644 --- a/MareSynchronos/UI/IntroUI.cs +++ b/MareSynchronos/UI/IntroUI.cs @@ -7,10 +7,10 @@ using System.Threading.Tasks; using Dalamud.Interface.Colors; using Dalamud.Interface.Windowing; using ImGuiNET; -using MareSynchronos.Managers; using MareSynchronos.Utils; using MareSynchronos.Localization; using Dalamud.Utility; +using MareSynchronos.FileCacheDB; namespace MareSynchronos.UI { @@ -18,7 +18,7 @@ namespace MareSynchronos.UI { private readonly UiShared _uiShared; private readonly Configuration _pluginConfiguration; - private readonly FileCacheManager _fileCacheManager; + private readonly PeriodicFileScanner _fileCacheManager; private readonly WindowSystem _windowSystem; private bool _readFirstPage; @@ -53,7 +53,7 @@ namespace MareSynchronos.UI } public IntroUi(WindowSystem windowSystem, UiShared uiShared, Configuration pluginConfiguration, - FileCacheManager fileCacheManager) : base("Mare Synchronos Setup") + PeriodicFileScanner fileCacheManager) : base("Mare Synchronos Setup") { Logger.Verbose("Creating " + nameof(IntroUi)); @@ -226,7 +226,7 @@ namespace MareSynchronos.UI if (ImGui.Button("Start Scan##startScan")) { - _fileCacheManager.StartInitialScan(); + _fileCacheManager.InvokeScan(); } } else diff --git a/MareSynchronos/UI/SettingsUi.cs b/MareSynchronos/UI/SettingsUi.cs index 56e8814..947e958 100644 --- a/MareSynchronos/UI/SettingsUi.cs +++ b/MareSynchronos/UI/SettingsUi.cs @@ -586,6 +586,8 @@ namespace MareSynchronos.UI private void DrawFileCacheSettings() { _uiShared.DrawFileScanState(); + _uiShared.DrawParallelScansSetting(); + _uiShared.DrawTimeSpanBetweenScansSetting(); _uiShared.DrawCacheDirectorySetting(); ImGui.Text($"Local cache size: {UiShared.ByteToString(_uiShared.FileCacheSize)}"); ImGui.SameLine(); @@ -597,6 +599,8 @@ namespace MareSynchronos.UI { File.Delete(file); } + + _uiShared.RecalculateFileCacheSize(); }); } } diff --git a/MareSynchronos/UI/UIShared.cs b/MareSynchronos/UI/UIShared.cs index e388bf7..dbb6ecb 100644 --- a/MareSynchronos/UI/UIShared.cs +++ b/MareSynchronos/UI/UIShared.cs @@ -11,6 +11,7 @@ using Dalamud.Interface.ImGuiFileDialog; using Dalamud.Plugin; using Dalamud.Utility; using ImGuiNET; +using MareSynchronos.FileCacheDB; using MareSynchronos.Localization; using MareSynchronos.Managers; using MareSynchronos.Utils; @@ -25,13 +26,13 @@ namespace MareSynchronos.UI private readonly IpcManager _ipcManager; private readonly ApiController _apiController; - private readonly FileCacheManager _fileCacheManager; + private readonly PeriodicFileScanner _cacheScanner; private readonly FileDialogManager _fileDialogManager; private readonly Configuration _pluginConfiguration; private readonly DalamudUtil _dalamudUtil; private readonly DalamudPluginInterface _pluginInterface; private readonly Dalamud.Localization _localization; - public long FileCacheSize => _fileCacheManager.FileCacheSize; + public long FileCacheSize => _cacheScanner.FileCacheSize; public string PlayerName => _dalamudUtil.PlayerName; public bool HasValidPenumbraModPath => !(_ipcManager.PenumbraModDirectory() ?? string.Empty).IsNullOrEmpty() && Directory.Exists(_ipcManager.PenumbraModDirectory()); public bool EditTrackerPosition { get; set; } @@ -42,11 +43,12 @@ namespace MareSynchronos.UI public ApiController ApiController => _apiController; - public UiShared(IpcManager ipcManager, ApiController apiController, FileCacheManager fileCacheManager, FileDialogManager fileDialogManager, Configuration pluginConfiguration, DalamudUtil dalamudUtil, DalamudPluginInterface pluginInterface, Dalamud.Localization localization) + public UiShared(IpcManager ipcManager, ApiController apiController, PeriodicFileScanner cacheScanner, FileDialogManager fileDialogManager, + Configuration pluginConfiguration, DalamudUtil dalamudUtil, DalamudPluginInterface pluginInterface, Dalamud.Localization localization) { _ipcManager = ipcManager; _apiController = apiController; - _fileCacheManager = fileCacheManager; + _cacheScanner = cacheScanner; _fileDialogManager = fileDialogManager; _pluginConfiguration = pluginConfiguration; _dalamudUtil = dalamudUtil; @@ -144,19 +146,23 @@ namespace MareSynchronos.UI public void DrawFileScanState() { ImGui.Text("File Scanner Status"); - if (_fileCacheManager.IsScanRunning) + ImGui.SameLine(); + if (_cacheScanner.IsScanRunning) { ImGui.Text("Scan is running"); ImGui.Text("Current Progress:"); ImGui.SameLine(); - ImGui.Text(_fileCacheManager.TotalFiles <= 0 + ImGui.Text(_cacheScanner.TotalFiles <= 1 ? "Collecting files" - : $"Processing {_fileCacheManager.CurrentFileProgress} / {_fileCacheManager.TotalFiles} files"); + : $"Processing {_cacheScanner.CurrentFileProgress} / {_cacheScanner.TotalFiles} files"); + } + else if (_pluginConfiguration.FileScanPaused) + { + ImGui.Text("File scanner is paused"); } else { - ImGui.Text("Watching Penumbra Directory: " + _fileCacheManager.WatchedPenumbraDirectory); - ImGui.Text("Watching Cache Directory: " + _fileCacheManager.WatchedCacheDirectory); + ImGui.Text("Next scan in " + _cacheScanner.TimeUntilNextScan); } } @@ -444,7 +450,7 @@ namespace MareSynchronos.UI { _pluginConfiguration.CacheFolder = path; _pluginConfiguration.Save(); - _fileCacheManager.StartWatchers(); + _cacheScanner.StartWatchers(); } }); } @@ -474,6 +480,7 @@ namespace MareSynchronos.UI _pluginConfiguration.MaxLocalCacheInGiB = maxCacheSize; _pluginConfiguration.Save(); } + DrawHelpText("The cache is automatically governed by Mare. It will clear itself automatically once it reaches the set capacity by removing the oldest unused files. You typically do not need to clear it yourself."); } private bool _isDirectoryWritable = false; @@ -503,14 +510,38 @@ namespace MareSynchronos.UI } } + public void RecalculateFileCacheSize() + { + _cacheScanner.InvokeScan(); + } + public void DrawParallelScansSetting() { var parallelScans = _pluginConfiguration.MaxParallelScan; - if (ImGui.SliderInt("Parallel File Scans##parallelism", ref parallelScans, 1, 20)) + if (ImGui.SliderInt("File scan parallelism##parallelism", ref parallelScans, 1, 20)) { _pluginConfiguration.MaxParallelScan = parallelScans; _pluginConfiguration.Save(); } + DrawHelpText("Decrease to lessen load of file scans. File scans will take longer to execute with less parallel threads."); + } + + public void DrawTimeSpanBetweenScansSetting() + { + var timeSpan = _pluginConfiguration.TimeSpanBetweenScansInSeconds; + if (ImGui.SliderInt("Seconds between scans##timespan", ref timeSpan, 20, 60)) + { + _pluginConfiguration.TimeSpanBetweenScansInSeconds = timeSpan; + _pluginConfiguration.Save(); + } + DrawHelpText("This is the time in seconds between file scans. Increase it to reduce system load. A too high setting can cause issues when manually fumbling about in the cache or Penumbra mods folders."); + var isPaused = _pluginConfiguration.FileScanPaused; + if (ImGui.Checkbox("Pause periodic file scan##filescanpause", ref isPaused)) + { + _pluginConfiguration.FileScanPaused = isPaused; + _pluginConfiguration.Save(); + } + DrawHelpText("This allows you to stop the periodic scans of your Penumbra and Mare cache directories. This setting will automatically revert itself on restart of Mare."); } public void Dispose() diff --git a/MareSynchronos/WebAPI/ApIController.Functions.Files.cs b/MareSynchronos/WebAPI/ApIController.Functions.Files.cs index 8d6c88f..217d0bc 100644 --- a/MareSynchronos/WebAPI/ApIController.Functions.Files.cs +++ b/MareSynchronos/WebAPI/ApIController.Functions.Files.cs @@ -5,7 +5,6 @@ using System.Linq; using System.Net; using System.Runtime.CompilerServices; using System.Text; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using LZ4; @@ -59,7 +58,13 @@ namespace MareSynchronos.WebAPI string fileName = Path.GetTempFileName(); - await wc.DownloadFileTaskAsync(downloadUri, fileName); + ct.Register(wc.CancelAsync); + + try + { + await wc.DownloadFileTaskAsync(downloadUri, fileName); + } + catch { } CurrentDownloads[downloadId].Single(f => f.Hash == hash).Transferred = CurrentDownloads[downloadId].Single(f => f.Hash == hash).Total; @@ -71,6 +76,7 @@ namespace MareSynchronos.WebAPI public async Task DownloadFiles(int currentDownloadId, List fileReplacementDto, CancellationToken ct) { + DownloadStarted?.Invoke(); Logger.Debug("Downloading files (Download ID " + currentDownloadId + ")"); List downloadFileInfoFromService = new List(); @@ -89,23 +95,29 @@ namespace MareSynchronos.WebAPI } } - foreach (var file in CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred)) + await Parallel.ForEachAsync(CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred), new ParallelOptions() + { + MaxDegreeOfParallelism = 5, + CancellationToken = ct + }, + async (file, token) => { var hash = file.Hash; - var tempFile = await DownloadFile(currentDownloadId, file.Hash, file.DownloadUri, ct); - if (ct.IsCancellationRequested) + var tempFile = await DownloadFile(currentDownloadId, file.Hash, file.DownloadUri, token); + if (token.IsCancellationRequested) { File.Delete(tempFile); Logger.Debug("Detected cancellation, removing " + currentDownloadId); + DownloadFinished?.Invoke(); CancelDownload(currentDownloadId); - break; + return; } - var tempFileData = await File.ReadAllBytesAsync(tempFile, ct); + var tempFileData = await File.ReadAllBytesAsync(tempFile, token); var extractedFile = LZ4Codec.Unwrap(tempFileData); File.Delete(tempFile); var filePath = Path.Combine(_pluginConfiguration.CacheFolder, file.Hash); - await File.WriteAllBytesAsync(filePath, extractedFile, ct); + await File.WriteAllBytesAsync(filePath, extractedFile, token); var fi = new FileInfo(filePath); Func RandomDayFunc() { @@ -118,26 +130,20 @@ namespace MareSynchronos.WebAPI fi.CreationTime = RandomDayFunc().Invoke(); fi.LastAccessTime = RandomDayFunc().Invoke(); fi.LastWriteTime = RandomDayFunc().Invoke(); - } - - var allFilesInDb = false; - while (!allFilesInDb && !ct.IsCancellationRequested) - { - await using (var db = new FileCacheContext()) + try { - var fileCount = CurrentDownloads[currentDownloadId] - .Where(c => c.CanBeTransferred) - .Count(h => db.FileCaches.Any(f => f.Hash == h.Hash)); - var totalFiles = CurrentDownloads[currentDownloadId].Count(c => c.CanBeTransferred); - Logger.Debug("Waiting for files to be in the DB, added " + fileCount + " of " + totalFiles); - - allFilesInDb = fileCount == totalFiles; + _ = _fileDbManager.CreateFileCacheEntity(filePath); } - - await Task.Delay(250, ct); - } + catch (Exception ex) + { + Logger.Warn("Issue adding file to the DB"); + Logger.Warn(ex.Message); + Logger.Warn(ex.StackTrace); + } + }); Logger.Debug("Download complete, removing " + currentDownloadId); + DownloadFinished?.Invoke(); CancelDownload(currentDownloadId); } diff --git a/MareSynchronos/WebAPI/ApiController.Connectivity.cs b/MareSynchronos/WebAPI/ApiController.Connectivity.cs index 3fbb1c1..5a22302 100644 --- a/MareSynchronos/WebAPI/ApiController.Connectivity.cs +++ b/MareSynchronos/WebAPI/ApiController.Connectivity.cs @@ -6,6 +6,7 @@ using System.Net.Http; using System.Threading; using System.Threading.Tasks; using MareSynchronos.API; +using MareSynchronos.Managers; using MareSynchronos.Utils; using MareSynchronos.WebAPI.Utils; using Microsoft.AspNetCore.Http.Connections; @@ -35,7 +36,7 @@ namespace MareSynchronos.WebAPI private readonly Configuration _pluginConfiguration; private readonly DalamudUtil _dalamudUtil; - + private readonly FileDbManager _fileDbManager; private CancellationTokenSource _connectionCancellationTokenSource; private HubConnection? _mareHub; @@ -48,12 +49,13 @@ namespace MareSynchronos.WebAPI public bool IsAdmin => _connectionDto?.IsAdmin ?? false; - public ApiController(Configuration pluginConfiguration, DalamudUtil dalamudUtil) + public ApiController(Configuration pluginConfiguration, DalamudUtil dalamudUtil, FileDbManager fileDbManager) { Logger.Verbose("Creating " + nameof(ApiController)); _pluginConfiguration = pluginConfiguration; _dalamudUtil = dalamudUtil; + _fileDbManager = fileDbManager; _connectionCancellationTokenSource = new CancellationTokenSource(); _dalamudUtil.LogIn += DalamudUtilOnLogIn; _dalamudUtil.LogOut += DalamudUtilOnLogOut; @@ -80,8 +82,6 @@ namespace MareSynchronos.WebAPI public event EventHandler? CharacterReceived; - public event VoidDelegate? RegisterFinalized; - public event VoidDelegate? Connected; public event VoidDelegate? Disconnected; @@ -93,6 +93,8 @@ namespace MareSynchronos.WebAPI public event SimpleStringDelegate? PairedWithOther; public event SimpleStringDelegate? UnpairedFromOther; + public event VoidDelegate? DownloadStarted; + public event VoidDelegate? DownloadFinished; public ConcurrentDictionary> CurrentDownloads { get; } = new(); @@ -313,7 +315,7 @@ namespace MareSynchronos.WebAPI .WithAutomaticReconnect(new ForeverRetryPolicy()) .ConfigureLogging(a => { a.ClearProviders().AddProvider(new DalamudLoggingProvider()); - a.SetMinimumLevel(LogLevel.Trace); + a.SetMinimumLevel(LogLevel.Warning); }) .Build(); }