performance optimizations

MareSynchronos/Configuration.cs

@@ -57,26 +57,12 @@ namespace MareSynchronos
         public Dictionary<string, string> ClientSecret { get; set; } = new();
         public Dictionary<string, string> CustomServerList { get; set; } = new();
         public int MaxLocalCacheInGiB { get; set; } = 20;
-
         public bool ReverseUserSort { get; set; } = true;
 
         public int TimeSpanBetweenScansInSeconds { get; set; } = 30;
         public bool FileScanPaused { get; set; } = false;
 
         public bool InitialScanComplete { get; set; } = false;
-        public int MaxParallelScan
-        {
-            get => _maxParallelScan;
-            set
-            {
-                _maxParallelScan = value switch
-                {
-                    < 0 => 1,
-                    > 20 => 10,
-                    _ => value
-                };
-            }
-        }
 
         public bool FullPause { get; set; } = false;
         public Dictionary<string, Dictionary<string, string>> UidServerComments { get; set; } = new();

MareSynchronos/FileCacheDB/FileCacheContext.cs

@@ -1,5 +1,4 @@
 using System.IO;
-using MareSynchronos.Utils;
 using Microsoft.EntityFrameworkCore;
 
 #nullable disable
@@ -12,22 +11,6 @@ namespace MareSynchronos.FileCacheDB
         public FileCacheContext()
         {
             DbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "FileCache.db");
-            string oldDbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "FileCacheDebug.db");
-            if (!Directory.Exists(Plugin.PluginInterface.ConfigDirectory.FullName))
-            {
-                Directory.CreateDirectory(Plugin.PluginInterface.ConfigDirectory.FullName);
-            }
-            var veryOldDbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "..", "FileCacheDebug.db");
-            if (File.Exists(veryOldDbPath))
-            {
-                Logger.Debug("Migrated old path to new path");
-                File.Move(veryOldDbPath, oldDbPath, true);
-            }
-            if (File.Exists(oldDbPath))
-            {
-                File.Move(oldDbPath, DbPath, true);
-            }
-
             Database.EnsureCreated();
         }
 
@@ -42,7 +25,7 @@ namespace MareSynchronos.FileCacheDB
         {
             if (!optionsBuilder.IsConfigured)
             {
-                optionsBuilder.UseSqlite("Data Source=" + DbPath);
+                optionsBuilder.UseSqlite("Data Source=" + DbPath+";Cache=Shared");
             }
         }
 
@@ -53,8 +36,6 @@ namespace MareSynchronos.FileCacheDB
                 entity.HasKey(e => new { e.Hash, e.Filepath });
 
                 entity.ToTable("FileCache");
-
-                entity.Property(c => c.Version).HasDefaultValue(0).IsRowVersion();
             });
 
             OnModelCreatingPartial(modelBuilder);

MareSynchronos/FileCacheDB/FileDbManager.cs

@@ -115,13 +115,10 @@ public class FileDbManager
     private FileCache? GetValidatedFileCache(FileCacheEntity e)
     {
         var fileCache = new FileCache(e);
-
         var resulingFileCache = MigrateLegacy(fileCache);
-
         if (resulingFileCache == null) return null;
 
         resulingFileCache = ReplacePathPrefixes(resulingFileCache);
-
         resulingFileCache = Validate(resulingFileCache);
         return resulingFileCache;
     }
@@ -178,11 +175,11 @@ public class FileDbManager
 
     private FileCache ReplacePathPrefixes(FileCache fileCache)
     {
-        if (fileCache.OriginalFilepath.Contains(PenumbraPrefix))
+        if (fileCache.OriginalFilepath.StartsWith(PenumbraPrefix))
         {
             fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory()));
         }
-        else if (fileCache.OriginalFilepath.Contains(CachePrefix))
+        else if (fileCache.OriginalFilepath.StartsWith(CachePrefix))
         {
             fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(CachePrefix, _configuration.CacheFolder));
         }

MareSynchronos/FileCacheDB/PeriodicFileScanner.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -84,7 +83,7 @@ public class PeriodicFileScanner : IDisposable
                 if (!_pluginConfiguration.FileScanPaused || isForced)
                 {
                     isForced = false;
-                    await PeriodicFileScan(token);
+                    PeriodicFileScan(token);
                 }
                 _timeUntilNextScan = TimeSpan.FromSeconds(timeBetweenScans);
                 while (_timeUntilNextScan.TotalSeconds >= 0)
@@ -130,7 +129,7 @@ public class PeriodicFileScanner : IDisposable
         return true;
     }
 
-    private async Task PeriodicFileScan(CancellationToken ct)
+    private void PeriodicFileScan(CancellationToken ct)
     {
         TotalFiles = 1;
         var penumbraDir = _ipcManager.PenumbraModDirectory();
@@ -153,68 +152,69 @@ public class PeriodicFileScanner : IDisposable
 
         Logger.Debug("Getting files from " + penumbraDir + " and " + _pluginConfiguration.CacheFolder);
         string[] ext = { ".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp" };
-        var scannedFiles = new ConcurrentDictionary<string, bool>(
+        var penumbraFiles = Directory.EnumerateDirectories(penumbraDir)
-            Directory.EnumerateFiles(penumbraDir, "*.*", SearchOption.AllDirectories)
+                            .SelectMany(d => Directory.EnumerateFiles(d, "*.*", SearchOption.AllDirectories)
-                            .Select(s => new FileInfo(s))
+                                                .Select(s => new FileInfo(s))
-                            .Where(f => ext.Contains(f.Extension) && !f.FullName.Contains(@"\bg\") && !f.FullName.Contains(@"\bgcommon\") && !f.FullName.Contains(@"\ui\"))
+                                                .Where(f => ext.Contains(f.Extension) && !f.FullName.Contains(@"\bg\") && !f.FullName.Contains(@"\bgcommon\") && !f.FullName.Contains(@"\ui\"))
-                            .Select(f => f.FullName.ToLowerInvariant())
+                                                .Select(f => f.FullName.ToLowerInvariant())).ToList();
-                            .Concat(Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.AllDirectories)
+
+        var cacheFiles = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.TopDirectoryOnly)
                                 .Where(f => new FileInfo(f).Name.Length == 40)
-                                .Select(s => s.ToLowerInvariant()))
+                                .Select(s => s.ToLowerInvariant());
-                            .Select(p => new KeyValuePair<string, bool>(p, false)).ToList());
+
+        var scannedFiles = new Dictionary<string, bool>(penumbraFiles.Concat(cacheFiles).Select(c => new KeyValuePair<string, bool>(c, false)));
+
         List<FileCacheEntity> fileDbEntries;
         using (var db = new FileCacheContext())
         {
-            fileDbEntries = await db.FileCaches.ToListAsync(cancellationToken: ct);
+            fileDbEntries = db.FileCaches.AsNoTracking().ToList();
         }
-
         TotalFiles = scannedFiles.Count;
 
         Logger.Debug("Database contains " + fileDbEntries.Count + " files, local system contains " + TotalFiles);
         // scan files from database
-        Parallel.ForEach(fileDbEntries.ToList(), new ParallelOptions()
+        var cpuCount = (int)(Environment.ProcessorCount / 2.0f);
+        foreach (var chunk in fileDbEntries.Chunk(cpuCount))
         {
-            MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
+            Task[] tasks = chunk.Select(c => Task.Run(() =>
-            CancellationToken = ct,
-        },
-        dbEntry =>
-        {
-            if (ct.IsCancellationRequested) return;
-            try
             {
-                var file = _fileDbManager.ValidateFileCache(dbEntry);
+                var file = _fileDbManager.ValidateFileCache(c);
-                if (file != null && scannedFiles.ContainsKey(file.Filepath))
+                if (file != null)
                 {
                     scannedFiles[file.Filepath] = true;
                 }
-            }
-            catch (Exception ex)
-            {
-                Logger.Warn(ex.Message);
-                Logger.Warn(ex.StackTrace);
-            }
 
-            Interlocked.Increment(ref currentFileProgress);
+                Interlocked.Increment(ref currentFileProgress);
-        });
+            })).ToArray();
+
+            Task.WaitAll(tasks, ct);
+
+            Thread.Sleep(3);
+
+            if (ct.IsCancellationRequested) return;
+        }
 
         Logger.Debug("Scanner validated existing db files");
 
         if (ct.IsCancellationRequested) return;
 
         // scan new files
-        Parallel.ForEach(scannedFiles.Where(c => c.Value == false), new ParallelOptions()
+        foreach (var chunk in scannedFiles.Where(c => c.Value == false).Chunk(cpuCount))
         {
-            MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
+            Task[] tasks = chunk.Select(c => Task.Run(() =>
-            CancellationToken = ct
+            {
-        },
+                var entry = _fileDbManager.CreateFileEntry(c.Key);
-        file =>
+                if (entry == null) _ = _fileDbManager.CreateCacheEntry(c.Key);
-        {
-            if (ct.IsCancellationRequested) return;
 
-            var entry = _fileDbManager.CreateFileEntry(file.Key);
+                Interlocked.Increment(ref currentFileProgress);
-            if (entry == null) _ = _fileDbManager.CreateCacheEntry(file.Key);
+            })).ToArray();
-            Interlocked.Increment(ref currentFileProgress);
+
-        });
+            Task.WaitAll(tasks, ct);
+
+            Thread.Sleep(3);
+
+            if (ct.IsCancellationRequested) return;
+        }
 
         Logger.Debug("Scanner added new files to db");
 

MareSynchronos/Plugin.cs

@@ -23,7 +23,6 @@ namespace MareSynchronos
         private const string CommandName = "/mare";
         private readonly ApiController _apiController;
         private readonly CommandManager _commandManager;
-        private readonly Framework _framework;
         private readonly Configuration _configuration;
         private readonly PeriodicFileScanner _fileCacheManager;
         private readonly IntroUi _introUi;
@@ -49,7 +48,6 @@ namespace MareSynchronos
             Logger.Debug("Launching " + Name);
             PluginInterface = pluginInterface;
             _commandManager = commandManager;
-            _framework = framework;
             _configuration = PluginInterface.GetPluginConfig() as Configuration ?? new Configuration();
             _configuration.Initialize(PluginInterface);
             _configuration.Migrate();
@@ -65,11 +63,6 @@ namespace MareSynchronos
             _dalamudUtil = new DalamudUtil(clientState, objectTable, framework, condition);
 
             _ipcManager = new IpcManager(PluginInterface, _dalamudUtil);
-
-            // Compatibility for FileSystemWatchers under OSX
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
-                Environment.SetEnvironmentVariable("MONO_MANAGED_WATCHER", "enabled");
-
             _fileDialogManager = new FileDialogManager();
             _fileDbManager = new FileDbManager(_ipcManager, _configuration);
             _apiController = new ApiController(_configuration, _dalamudUtil, _fileDbManager);

MareSynchronos/UI/IntroUI.cs

@@ -220,10 +220,6 @@ namespace MareSynchronos.UI
 
                 if (!_fileCacheManager.IsScanRunning && !string.IsNullOrEmpty(_pluginConfiguration.CacheFolder) && _uiShared.HasValidPenumbraModPath && Directory.Exists(_pluginConfiguration.CacheFolder))
                 {
-                    UiShared.TextWrapped("You can adjust how many parallel threads will be used for scanning. Mind that ultimately it will depend on the amount of mods, your disk speed and your CPU. " +
-                                      "More is not necessarily better, the default of 10 should be fine for most cases.");
-                    _uiShared.DrawParallelScansSetting();
-
                     if (ImGui.Button("Start Scan##startScan"))
                     {
                         _fileCacheManager.InvokeScan(true);

MareSynchronos/UI/SettingsUi.cs

@@ -586,7 +586,6 @@ namespace MareSynchronos.UI
         private void DrawFileCacheSettings()
         {
             _uiShared.DrawFileScanState();
-            _uiShared.DrawParallelScansSetting();
             _uiShared.DrawTimeSpanBetweenScansSetting();
             _uiShared.DrawCacheDirectorySetting();
             ImGui.Text($"Local cache size: {UiShared.ByteToString(_uiShared.FileCacheSize)}");

MareSynchronos/UI/UIShared.cs

@@ -520,17 +520,6 @@ namespace MareSynchronos.UI
             _cacheScanner.InvokeScan(true);
         }
 
-        public void DrawParallelScansSetting()
-        {
-            var parallelScans = _pluginConfiguration.MaxParallelScan;
-            if (ImGui.SliderInt("File scan parallelism##parallelism", ref parallelScans, 1, 20))
-            {
-                _pluginConfiguration.MaxParallelScan = parallelScans;
-                _pluginConfiguration.Save();
-            }
-            DrawHelpText("Decrease to lessen load of file scans. File scans will take longer to execute with less parallel threads.");
-        }
-
         public void DrawTimeSpanBetweenScansSetting()
         {
             var timeSpan = _pluginConfiguration.TimeSpanBetweenScansInSeconds;