From e40ea323e4c7ae91ba4b4255d3ca5b3be01881d8 Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Mon, 26 Sep 2022 14:15:11 +0200 Subject: [PATCH] add more resilency, performance adjustments --- MareSynchronos/FileCacheDB/FileDbManager.cs | 20 ++-- .../FileCacheDB/PeriodicFileScanner.cs | 101 +++++++++++++----- MareSynchronos/MareSynchronos.csproj | 2 +- MareSynchronos/UI/UIShared.cs | 2 +- 4 files changed, 88 insertions(+), 37 deletions(-) diff --git a/MareSynchronos/FileCacheDB/FileDbManager.cs b/MareSynchronos/FileCacheDB/FileDbManager.cs index 33fcb94..0b75212 100644 --- a/MareSynchronos/FileCacheDB/FileDbManager.cs +++ b/MareSynchronos/FileCacheDB/FileDbManager.cs @@ -47,7 +47,7 @@ public class FileDbManager public FileCache? ValidateFileCacheEntity(FileCacheEntity fileCacheEntity) { - return GetValidatedFileCache(fileCacheEntity); + return GetValidatedFileCache(fileCacheEntity, false); } public FileCache? GetFileCacheByPath(string path) @@ -112,23 +112,26 @@ public class FileDbManager return result; } - private FileCache? GetValidatedFileCache(FileCacheEntity e) + private FileCache? GetValidatedFileCache(FileCacheEntity e, bool removeOnNonExistence = true) { var fileCache = new FileCache(e); var resulingFileCache = MigrateLegacy(fileCache); if (resulingFileCache == null) return null; resulingFileCache = ReplacePathPrefixes(resulingFileCache); - resulingFileCache = Validate(resulingFileCache); + resulingFileCache = Validate(resulingFileCache, removeOnNonExistence); return resulingFileCache; } - private FileCache? Validate(FileCache fileCache) + private FileCache? Validate(FileCache fileCache, bool removeOnNonExistence = true) { var file = new FileInfo(fileCache.Filepath); if (!file.Exists) { - DeleteFromDatabase(new[] { fileCache }); + if (removeOnNonExistence) + { + DeleteFromDatabase(new[] { fileCache }); + } return null; } @@ -141,7 +144,7 @@ public class FileDbManager return fileCache; } - private FileCache? MigrateLegacy(FileCache fileCache) + private FileCache? MigrateLegacy(FileCache fileCache, bool removeOnNonExistence = true) { if (fileCache.OriginalFilepath.Contains(PenumbraPrefix + "\\") || fileCache.OriginalFilepath.Contains(CachePrefix)) return fileCache; @@ -166,7 +169,10 @@ public class FileDbManager } else { - DeleteFromDatabase(new[] { fileCache }); + if (removeOnNonExistence) + { + DeleteFromDatabase(new[] { fileCache }); + } return null; } diff --git a/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs b/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs index bf63086..c9fcccf 100644 --- a/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs +++ b/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -38,14 +39,23 @@ public class PeriodicFileScanner : IDisposable private void _apiController_DownloadFinished() { - InvokeScan(); + if (fileScanWasRunning) + { + fileScanWasRunning = false; + InvokeScan(true); + } } private void _apiController_DownloadStarted() { - _scanCancellationTokenSource?.Cancel(); + if (IsScanRunning) + { + _scanCancellationTokenSource?.Cancel(); + fileScanWasRunning = true; + } } + private bool fileScanWasRunning = false; private long currentFileProgress = 0; public long CurrentFileProgress => currentFileProgress; @@ -72,6 +82,8 @@ public class PeriodicFileScanner : IDisposable public void InvokeScan(bool forced = false) { bool isForced = forced; + TotalFiles = 0; + currentFileProgress = 0; _scanCancellationTokenSource?.Cancel(); _scanCancellationTokenSource = new CancellationTokenSource(); var token = _scanCancellationTokenSource.Token; @@ -83,6 +95,8 @@ public class PeriodicFileScanner : IDisposable if (!_pluginConfiguration.FileScanPaused || isForced) { isForced = false; + TotalFiles = 0; + currentFileProgress = 0; PeriodicFileScan(token); } _timeUntilNextScan = TimeSpan.FromSeconds(timeBetweenScans); @@ -163,45 +177,76 @@ public class PeriodicFileScanner : IDisposable .Select(s => s.ToLowerInvariant()); var scannedFiles = new Dictionary(penumbraFiles.Concat(cacheFiles).Select(c => new KeyValuePair(c, false))); - - List fileDbEntries; - using (var db = new FileCacheContext()) - { - fileDbEntries = db.FileCaches.AsNoTracking().ToList(); - } TotalFiles = scannedFiles.Count; - Logger.Debug("Database contains " + fileDbEntries.Count + " files, local system contains " + TotalFiles); + // scan files from database var cpuCount = (int)(Environment.ProcessorCount / 2.0f); - foreach (var chunk in fileDbEntries.Chunk(cpuCount)) + Task[] dbTasks = Enumerable.Range(0, cpuCount).Select(c => Task.CompletedTask).ToArray(); + using (var db = new FileCacheContext()) { - Task[] tasks = chunk.Select(c => Task.Run(() => + Logger.Debug("Database contains " + db.FileCaches.Count() + " files, local system contains " + TotalFiles); + ConcurrentBag entitiesToRemove = new(); + try { - try + foreach (var entry in db.FileCaches.AsNoTracking()) { - var file = _fileDbManager.ValidateFileCacheEntity(c); - if (file != null) + var idx = Task.WaitAny(dbTasks, ct); + dbTasks[idx] = Task.Run(() => { - scannedFiles[file.Filepath] = true; - } + try + { + var file = _fileDbManager.ValidateFileCacheEntity(entry); + if (file != null) + { + scannedFiles[file.Filepath] = true; + } + else + { + entitiesToRemove.Add(entry); + } + } + catch (Exception ex) + { + Logger.Warn("Failed validating " + entry.Filepath); + Logger.Warn(ex.Message); + entitiesToRemove.Add(entry); + } + + Interlocked.Increment(ref currentFileProgress); + Thread.Sleep(1); + }, ct); + + if (ct.IsCancellationRequested) return; } - catch (Exception ex) + } + catch (Exception ex) + { + Logger.Warn("Error during enumerating FileCaches: " + ex.Message); + } + + try + { + if (entitiesToRemove.Any()) { - Logger.Warn("Failed validating " + c.Filepath); - Logger.Warn(ex.Message); + foreach (var entry in entitiesToRemove) + { + Logger.Debug("Removing " + entry.Filepath); + var toRemove = db.FileCaches.First(f => f.Filepath == entry.Filepath && f.Hash == entry.Hash); + db.FileCaches.Remove(toRemove); + } + + db.SaveChanges(); } - - Interlocked.Increment(ref currentFileProgress); - })).ToArray(); - - Task.WaitAll(tasks, ct); - - Thread.Sleep(3); - - if (ct.IsCancellationRequested) return; + } + catch (Exception ex) + { + Logger.Warn(ex.Message); + } } + Task.WaitAll(dbTasks); + Logger.Debug("Scanner validated existing db files"); if (ct.IsCancellationRequested) return; diff --git a/MareSynchronos/MareSynchronos.csproj b/MareSynchronos/MareSynchronos.csproj index f6b27b5..9d3a5f2 100644 --- a/MareSynchronos/MareSynchronos.csproj +++ b/MareSynchronos/MareSynchronos.csproj @@ -3,7 +3,7 @@ - 0.4.15 + 0.4.16 https://github.com/Penumbra-Sync/client diff --git a/MareSynchronos/UI/UIShared.cs b/MareSynchronos/UI/UIShared.cs index 39aeb84..d5a42f6 100644 --- a/MareSynchronos/UI/UIShared.cs +++ b/MareSynchronos/UI/UIShared.cs @@ -152,7 +152,7 @@ namespace MareSynchronos.UI ImGui.Text("Scan is running"); ImGui.Text("Current Progress:"); ImGui.SameLine(); - ImGui.Text(_cacheScanner.TotalFiles <= 1 + ImGui.Text(_cacheScanner.TotalFiles == 1 ? "Collecting files" : $"Processing {_cacheScanner.CurrentFileProgress} / {_cacheScanner.TotalFiles} files"); }