add more resilency, performance adjustments

MareSynchronos/FileCacheDB/FileDbManager.cs

@@ -47,7 +47,7 @@ public class FileDbManager
 
     public FileCache? ValidateFileCacheEntity(FileCacheEntity fileCacheEntity)
     {
-        return GetValidatedFileCache(fileCacheEntity);
+        return GetValidatedFileCache(fileCacheEntity, false);
     }
 
     public FileCache? GetFileCacheByPath(string path)
@@ -112,23 +112,26 @@ public class FileDbManager
         return result;
     }
 
-    private FileCache? GetValidatedFileCache(FileCacheEntity e)
+    private FileCache? GetValidatedFileCache(FileCacheEntity e, bool removeOnNonExistence = true)
     {
         var fileCache = new FileCache(e);
         var resulingFileCache = MigrateLegacy(fileCache);
         if (resulingFileCache == null) return null;
 
         resulingFileCache = ReplacePathPrefixes(resulingFileCache);
-        resulingFileCache = Validate(resulingFileCache);
+        resulingFileCache = Validate(resulingFileCache, removeOnNonExistence);
         return resulingFileCache;
     }
 
-    private FileCache? Validate(FileCache fileCache)
+    private FileCache? Validate(FileCache fileCache, bool removeOnNonExistence = true)
     {
         var file = new FileInfo(fileCache.Filepath);
         if (!file.Exists)
         {
-            DeleteFromDatabase(new[] { fileCache });
+            if (removeOnNonExistence)
+            {
+                DeleteFromDatabase(new[] { fileCache });
+            }
             return null;
         }
 
@@ -141,7 +144,7 @@ public class FileDbManager
         return fileCache;
     }
 
-    private FileCache? MigrateLegacy(FileCache fileCache)
+    private FileCache? MigrateLegacy(FileCache fileCache, bool removeOnNonExistence = true)
     {
         if (fileCache.OriginalFilepath.Contains(PenumbraPrefix + "\\") || fileCache.OriginalFilepath.Contains(CachePrefix)) return fileCache;
 
@@ -166,7 +169,10 @@ public class FileDbManager
         }
         else
         {
-            DeleteFromDatabase(new[] { fileCache });
+            if (removeOnNonExistence)
+            {
+                DeleteFromDatabase(new[] { fileCache });
+            }
             return null;
         }
 

MareSynchronos/FileCacheDB/PeriodicFileScanner.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -38,14 +39,23 @@ public class PeriodicFileScanner : IDisposable
 
     private void _apiController_DownloadFinished()
     {
-        InvokeScan();
+        if (fileScanWasRunning)
+        {
+            fileScanWasRunning = false;
+            InvokeScan(true);
+        }
     }
 
     private void _apiController_DownloadStarted()
     {
-        _scanCancellationTokenSource?.Cancel();
+        if (IsScanRunning)
+        {
+            _scanCancellationTokenSource?.Cancel();
+            fileScanWasRunning = true;
+        }
     }
 
+    private bool fileScanWasRunning = false;
     private long currentFileProgress = 0;
     public long CurrentFileProgress => currentFileProgress;
 
@@ -72,6 +82,8 @@ public class PeriodicFileScanner : IDisposable
     public void InvokeScan(bool forced = false)
     {
         bool isForced = forced;
+        TotalFiles = 0;
+        currentFileProgress = 0;
         _scanCancellationTokenSource?.Cancel();
         _scanCancellationTokenSource = new CancellationTokenSource();
         var token = _scanCancellationTokenSource.Token;
@@ -83,6 +95,8 @@ public class PeriodicFileScanner : IDisposable
                 if (!_pluginConfiguration.FileScanPaused || isForced)
                 {
                     isForced = false;
+                    TotalFiles = 0;
+                    currentFileProgress = 0;
                     PeriodicFileScan(token);
                 }
                 _timeUntilNextScan = TimeSpan.FromSeconds(timeBetweenScans);
@@ -163,45 +177,76 @@ public class PeriodicFileScanner : IDisposable
                                 .Select(s => s.ToLowerInvariant());
 
         var scannedFiles = new Dictionary<string, bool>(penumbraFiles.Concat(cacheFiles).Select(c => new KeyValuePair<string, bool>(c, false)));
-
-        List<FileCacheEntity> fileDbEntries;
-        using (var db = new FileCacheContext())
-        {
-            fileDbEntries = db.FileCaches.AsNoTracking().ToList();
-        }
         TotalFiles = scannedFiles.Count;
 
-        Logger.Debug("Database contains " + fileDbEntries.Count + " files, local system contains " + TotalFiles);
+
         // scan files from database
         var cpuCount = (int)(Environment.ProcessorCount / 2.0f);
-        foreach (var chunk in fileDbEntries.Chunk(cpuCount))
+        Task[] dbTasks = Enumerable.Range(0, cpuCount).Select(c => Task.CompletedTask).ToArray();
+        using (var db = new FileCacheContext())
         {
-            Task[] tasks = chunk.Select(c => Task.Run(() =>
+            Logger.Debug("Database contains " + db.FileCaches.Count() + " files, local system contains " + TotalFiles);
+            ConcurrentBag<FileCacheEntity> entitiesToRemove = new();
+            try
             {
-                try
+                foreach (var entry in db.FileCaches.AsNoTracking())
                 {
-                    var file = _fileDbManager.ValidateFileCacheEntity(c);
+                    var idx = Task.WaitAny(dbTasks, ct);
-                    if (file != null)
+                    dbTasks[idx] = Task.Run(() =>
                     {
-                        scannedFiles[file.Filepath] = true;
+                        try
-                    }
+                        {
+                            var file = _fileDbManager.ValidateFileCacheEntity(entry);
+                            if (file != null)
+                            {
+                                scannedFiles[file.Filepath] = true;
+                            }
+                            else
+                            {
+                                entitiesToRemove.Add(entry);
+                            }
+                        }
+                        catch (Exception ex)
+                        {
+                            Logger.Warn("Failed validating " + entry.Filepath);
+                            Logger.Warn(ex.Message);
+                            entitiesToRemove.Add(entry);
+                        }
+
+                        Interlocked.Increment(ref currentFileProgress);
+                        Thread.Sleep(1);
+                    }, ct);
+
+                    if (ct.IsCancellationRequested) return;
                 }
-                catch (Exception ex)
+            }
+            catch (Exception ex)
+            {
+                Logger.Warn("Error during enumerating FileCaches: " + ex.Message);
+            }
+
+            try
+            {
+                if (entitiesToRemove.Any())
                 {
-                    Logger.Warn("Failed validating " + c.Filepath);
+                    foreach (var entry in entitiesToRemove)
-                    Logger.Warn(ex.Message);
+                    {
+                        Logger.Debug("Removing " + entry.Filepath);
+                        var toRemove = db.FileCaches.First(f => f.Filepath == entry.Filepath && f.Hash == entry.Hash);
+                        db.FileCaches.Remove(toRemove);
+                    }
+
+                    db.SaveChanges();
                 }
-
+            }
-                Interlocked.Increment(ref currentFileProgress);
+            catch (Exception ex)
-            })).ToArray();
+            {
-
+                Logger.Warn(ex.Message);
-            Task.WaitAll(tasks, ct);
+            }
-
-            Thread.Sleep(3);
-
-            if (ct.IsCancellationRequested) return;
         }
 
+        Task.WaitAll(dbTasks);
+
         Logger.Debug("Scanner validated existing db files");
 
         if (ct.IsCancellationRequested) return;

MareSynchronos/MareSynchronos.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <Authors></Authors>
     <Company></Company>
-    <Version>0.4.15</Version>
+    <Version>0.4.16</Version>
     <Description></Description>
     <Copyright></Copyright>
     <PackageProjectUrl>https://github.com/Penumbra-Sync/client</PackageProjectUrl>

MareSynchronos/UI/UIShared.cs

@@ -152,7 +152,7 @@ namespace MareSynchronos.UI
                 ImGui.Text("Scan is running");
                 ImGui.Text("Current Progress:");
                 ImGui.SameLine();
-                ImGui.Text(_cacheScanner.TotalFiles <= 1
+                ImGui.Text(_cacheScanner.TotalFiles == 1
                     ? "Collecting files"
                     : $"Processing {_cacheScanner.CurrentFileProgress} / {_cacheScanner.TotalFiles} files");
             }