From a75ff627b8b72d4b9beeb5f57da2b61880639631 Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Mon, 26 Sep 2022 00:16:05 +0200 Subject: [PATCH] performance optimizations --- MareSynchronos/Configuration.cs | 14 ---- .../FileCacheDB/FileCacheContext.cs | 21 +---- MareSynchronos/FileCacheDB/FileDbManager.cs | 7 +- .../FileCacheDB/PeriodicFileScanner.cs | 84 +++++++++---------- MareSynchronos/Plugin.cs | 7 -- MareSynchronos/UI/IntroUI.cs | 4 - MareSynchronos/UI/SettingsUi.cs | 1 - MareSynchronos/UI/UIShared.cs | 11 --- 8 files changed, 45 insertions(+), 104 deletions(-) diff --git a/MareSynchronos/Configuration.cs b/MareSynchronos/Configuration.cs index f63b734..017ef81 100644 --- a/MareSynchronos/Configuration.cs +++ b/MareSynchronos/Configuration.cs @@ -57,26 +57,12 @@ namespace MareSynchronos public Dictionary ClientSecret { get; set; } = new(); public Dictionary CustomServerList { get; set; } = new(); public int MaxLocalCacheInGiB { get; set; } = 20; - public bool ReverseUserSort { get; set; } = true; public int TimeSpanBetweenScansInSeconds { get; set; } = 30; public bool FileScanPaused { get; set; } = false; public bool InitialScanComplete { get; set; } = false; - public int MaxParallelScan - { - get => _maxParallelScan; - set - { - _maxParallelScan = value switch - { - < 0 => 1, - > 20 => 10, - _ => value - }; - } - } public bool FullPause { get; set; } = false; public Dictionary> UidServerComments { get; set; } = new(); diff --git a/MareSynchronos/FileCacheDB/FileCacheContext.cs b/MareSynchronos/FileCacheDB/FileCacheContext.cs index b8fb318..9ccc9a5 100644 --- a/MareSynchronos/FileCacheDB/FileCacheContext.cs +++ b/MareSynchronos/FileCacheDB/FileCacheContext.cs @@ -1,5 +1,4 @@ using System.IO; -using MareSynchronos.Utils; using Microsoft.EntityFrameworkCore; #nullable disable @@ -12,22 +11,6 @@ namespace MareSynchronos.FileCacheDB public FileCacheContext() { DbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "FileCache.db"); - string oldDbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "FileCacheDebug.db"); - if (!Directory.Exists(Plugin.PluginInterface.ConfigDirectory.FullName)) - { - Directory.CreateDirectory(Plugin.PluginInterface.ConfigDirectory.FullName); - } - var veryOldDbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "..", "FileCacheDebug.db"); - if (File.Exists(veryOldDbPath)) - { - Logger.Debug("Migrated old path to new path"); - File.Move(veryOldDbPath, oldDbPath, true); - } - if (File.Exists(oldDbPath)) - { - File.Move(oldDbPath, DbPath, true); - } - Database.EnsureCreated(); } @@ -42,7 +25,7 @@ namespace MareSynchronos.FileCacheDB { if (!optionsBuilder.IsConfigured) { - optionsBuilder.UseSqlite("Data Source=" + DbPath); + optionsBuilder.UseSqlite("Data Source=" + DbPath+";Cache=Shared"); } } @@ -53,8 +36,6 @@ namespace MareSynchronos.FileCacheDB entity.HasKey(e => new { e.Hash, e.Filepath }); entity.ToTable("FileCache"); - - entity.Property(c => c.Version).HasDefaultValue(0).IsRowVersion(); }); OnModelCreatingPartial(modelBuilder); diff --git a/MareSynchronos/FileCacheDB/FileDbManager.cs b/MareSynchronos/FileCacheDB/FileDbManager.cs index 7392e8b..de97485 100644 --- a/MareSynchronos/FileCacheDB/FileDbManager.cs +++ b/MareSynchronos/FileCacheDB/FileDbManager.cs @@ -115,13 +115,10 @@ public class FileDbManager private FileCache? GetValidatedFileCache(FileCacheEntity e) { var fileCache = new FileCache(e); - var resulingFileCache = MigrateLegacy(fileCache); - if (resulingFileCache == null) return null; resulingFileCache = ReplacePathPrefixes(resulingFileCache); - resulingFileCache = Validate(resulingFileCache); return resulingFileCache; } @@ -178,11 +175,11 @@ public class FileDbManager private FileCache ReplacePathPrefixes(FileCache fileCache) { - if (fileCache.OriginalFilepath.Contains(PenumbraPrefix)) + if (fileCache.OriginalFilepath.StartsWith(PenumbraPrefix)) { fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory())); } - else if (fileCache.OriginalFilepath.Contains(CachePrefix)) + else if (fileCache.OriginalFilepath.StartsWith(CachePrefix)) { fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(CachePrefix, _configuration.CacheFolder)); } diff --git a/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs b/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs index efdef4f..0d38c5d 100644 --- a/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs +++ b/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -84,7 +83,7 @@ public class PeriodicFileScanner : IDisposable if (!_pluginConfiguration.FileScanPaused || isForced) { isForced = false; - await PeriodicFileScan(token); + PeriodicFileScan(token); } _timeUntilNextScan = TimeSpan.FromSeconds(timeBetweenScans); while (_timeUntilNextScan.TotalSeconds >= 0) @@ -130,7 +129,7 @@ public class PeriodicFileScanner : IDisposable return true; } - private async Task PeriodicFileScan(CancellationToken ct) + private void PeriodicFileScan(CancellationToken ct) { TotalFiles = 1; var penumbraDir = _ipcManager.PenumbraModDirectory(); @@ -153,68 +152,69 @@ public class PeriodicFileScanner : IDisposable Logger.Debug("Getting files from " + penumbraDir + " and " + _pluginConfiguration.CacheFolder); string[] ext = { ".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp" }; - var scannedFiles = new ConcurrentDictionary( - Directory.EnumerateFiles(penumbraDir, "*.*", SearchOption.AllDirectories) - .Select(s => new FileInfo(s)) - .Where(f => ext.Contains(f.Extension) && !f.FullName.Contains(@"\bg\") && !f.FullName.Contains(@"\bgcommon\") && !f.FullName.Contains(@"\ui\")) - .Select(f => f.FullName.ToLowerInvariant()) - .Concat(Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.AllDirectories) + var penumbraFiles = Directory.EnumerateDirectories(penumbraDir) + .SelectMany(d => Directory.EnumerateFiles(d, "*.*", SearchOption.AllDirectories) + .Select(s => new FileInfo(s)) + .Where(f => ext.Contains(f.Extension) && !f.FullName.Contains(@"\bg\") && !f.FullName.Contains(@"\bgcommon\") && !f.FullName.Contains(@"\ui\")) + .Select(f => f.FullName.ToLowerInvariant())).ToList(); + + var cacheFiles = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.TopDirectoryOnly) .Where(f => new FileInfo(f).Name.Length == 40) - .Select(s => s.ToLowerInvariant())) - .Select(p => new KeyValuePair(p, false)).ToList()); + .Select(s => s.ToLowerInvariant()); + + var scannedFiles = new Dictionary(penumbraFiles.Concat(cacheFiles).Select(c => new KeyValuePair(c, false))); + List fileDbEntries; using (var db = new FileCacheContext()) { - fileDbEntries = await db.FileCaches.ToListAsync(cancellationToken: ct); + fileDbEntries = db.FileCaches.AsNoTracking().ToList(); } - TotalFiles = scannedFiles.Count; Logger.Debug("Database contains " + fileDbEntries.Count + " files, local system contains " + TotalFiles); // scan files from database - Parallel.ForEach(fileDbEntries.ToList(), new ParallelOptions() + var cpuCount = (int)(Environment.ProcessorCount / 2.0f); + foreach (var chunk in fileDbEntries.Chunk(cpuCount)) { - MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan, - CancellationToken = ct, - }, - dbEntry => - { - if (ct.IsCancellationRequested) return; - try + Task[] tasks = chunk.Select(c => Task.Run(() => { - var file = _fileDbManager.ValidateFileCache(dbEntry); - if (file != null && scannedFiles.ContainsKey(file.Filepath)) + var file = _fileDbManager.ValidateFileCache(c); + if (file != null) { scannedFiles[file.Filepath] = true; } - } - catch (Exception ex) - { - Logger.Warn(ex.Message); - Logger.Warn(ex.StackTrace); - } - Interlocked.Increment(ref currentFileProgress); - }); + Interlocked.Increment(ref currentFileProgress); + })).ToArray(); + + Task.WaitAll(tasks, ct); + + Thread.Sleep(3); + + if (ct.IsCancellationRequested) return; + } Logger.Debug("Scanner validated existing db files"); if (ct.IsCancellationRequested) return; // scan new files - Parallel.ForEach(scannedFiles.Where(c => c.Value == false), new ParallelOptions() + foreach (var chunk in scannedFiles.Where(c => c.Value == false).Chunk(cpuCount)) { - MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan, - CancellationToken = ct - }, - file => - { - if (ct.IsCancellationRequested) return; + Task[] tasks = chunk.Select(c => Task.Run(() => + { + var entry = _fileDbManager.CreateFileEntry(c.Key); + if (entry == null) _ = _fileDbManager.CreateCacheEntry(c.Key); - var entry = _fileDbManager.CreateFileEntry(file.Key); - if (entry == null) _ = _fileDbManager.CreateCacheEntry(file.Key); - Interlocked.Increment(ref currentFileProgress); - }); + Interlocked.Increment(ref currentFileProgress); + })).ToArray(); + + Task.WaitAll(tasks, ct); + + Thread.Sleep(3); + + if (ct.IsCancellationRequested) return; + } Logger.Debug("Scanner added new files to db"); diff --git a/MareSynchronos/Plugin.cs b/MareSynchronos/Plugin.cs index 524dd61..f157cb5 100644 --- a/MareSynchronos/Plugin.cs +++ b/MareSynchronos/Plugin.cs @@ -23,7 +23,6 @@ namespace MareSynchronos private const string CommandName = "/mare"; private readonly ApiController _apiController; private readonly CommandManager _commandManager; - private readonly Framework _framework; private readonly Configuration _configuration; private readonly PeriodicFileScanner _fileCacheManager; private readonly IntroUi _introUi; @@ -49,7 +48,6 @@ namespace MareSynchronos Logger.Debug("Launching " + Name); PluginInterface = pluginInterface; _commandManager = commandManager; - _framework = framework; _configuration = PluginInterface.GetPluginConfig() as Configuration ?? new Configuration(); _configuration.Initialize(PluginInterface); _configuration.Migrate(); @@ -65,11 +63,6 @@ namespace MareSynchronos _dalamudUtil = new DalamudUtil(clientState, objectTable, framework, condition); _ipcManager = new IpcManager(PluginInterface, _dalamudUtil); - - // Compatibility for FileSystemWatchers under OSX - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - Environment.SetEnvironmentVariable("MONO_MANAGED_WATCHER", "enabled"); - _fileDialogManager = new FileDialogManager(); _fileDbManager = new FileDbManager(_ipcManager, _configuration); _apiController = new ApiController(_configuration, _dalamudUtil, _fileDbManager); diff --git a/MareSynchronos/UI/IntroUI.cs b/MareSynchronos/UI/IntroUI.cs index b4a48b7..187e95f 100644 --- a/MareSynchronos/UI/IntroUI.cs +++ b/MareSynchronos/UI/IntroUI.cs @@ -220,10 +220,6 @@ namespace MareSynchronos.UI if (!_fileCacheManager.IsScanRunning && !string.IsNullOrEmpty(_pluginConfiguration.CacheFolder) && _uiShared.HasValidPenumbraModPath && Directory.Exists(_pluginConfiguration.CacheFolder)) { - UiShared.TextWrapped("You can adjust how many parallel threads will be used for scanning. Mind that ultimately it will depend on the amount of mods, your disk speed and your CPU. " + - "More is not necessarily better, the default of 10 should be fine for most cases."); - _uiShared.DrawParallelScansSetting(); - if (ImGui.Button("Start Scan##startScan")) { _fileCacheManager.InvokeScan(true); diff --git a/MareSynchronos/UI/SettingsUi.cs b/MareSynchronos/UI/SettingsUi.cs index 947e958..295f9cc 100644 --- a/MareSynchronos/UI/SettingsUi.cs +++ b/MareSynchronos/UI/SettingsUi.cs @@ -586,7 +586,6 @@ namespace MareSynchronos.UI private void DrawFileCacheSettings() { _uiShared.DrawFileScanState(); - _uiShared.DrawParallelScansSetting(); _uiShared.DrawTimeSpanBetweenScansSetting(); _uiShared.DrawCacheDirectorySetting(); ImGui.Text($"Local cache size: {UiShared.ByteToString(_uiShared.FileCacheSize)}"); diff --git a/MareSynchronos/UI/UIShared.cs b/MareSynchronos/UI/UIShared.cs index c840cb4..39aeb84 100644 --- a/MareSynchronos/UI/UIShared.cs +++ b/MareSynchronos/UI/UIShared.cs @@ -520,17 +520,6 @@ namespace MareSynchronos.UI _cacheScanner.InvokeScan(true); } - public void DrawParallelScansSetting() - { - var parallelScans = _pluginConfiguration.MaxParallelScan; - if (ImGui.SliderInt("File scan parallelism##parallelism", ref parallelScans, 1, 20)) - { - _pluginConfiguration.MaxParallelScan = parallelScans; - _pluginConfiguration.Save(); - } - DrawHelpText("Decrease to lessen load of file scans. File scans will take longer to execute with less parallel threads."); - } - public void DrawTimeSpanBetweenScansSetting() { var timeSpan = _pluginConfiguration.TimeSpanBetweenScansInSeconds;