improve file cache manager performance

MareSynchronos/FileCache/FileCacheManager.cs

@@ -15,7 +15,7 @@ public sealed class FileCacheManager : IDisposable
     private const string _penumbraPrefix = "{penumbra}";
     private readonly MareConfigService _configService;
     private readonly string _csvPath;
-    private readonly ConcurrentDictionary<string, FileCacheEntity> _fileCaches = new(StringComparer.Ordinal);
+    private readonly ConcurrentDictionary<string, List<FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
     private readonly object _fileWriteLock = new();
     private readonly IpcManager _ipcManager;
     private readonly ILogger<FileCacheManager> _logger;
@@ -39,8 +39,15 @@ public sealed class FileCacheManager : IDisposable
             catch (Exception ex)
             {
                 _logger.LogWarning(ex, "Failed to move BAK to ORG, deleting BAK");
-                if (File.Exists(CsvBakPath))
+                try
-                    File.Delete(CsvBakPath);
+                {
+                    if (File.Exists(CsvBakPath))
+                        File.Delete(CsvBakPath);
+                }
+                catch (Exception ex1)
+                {
+                    _logger.LogWarning(ex1, "Could not delete bak file");
+                }
             }
         }
 
@@ -55,7 +62,7 @@ public sealed class FileCacheManager : IDisposable
                     var hash = splittedEntry[0];
                     var path = splittedEntry[1];
                     var time = splittedEntry[2];
-                    _fileCaches[path] = ReplacePathPrefixes(new FileCacheEntity(hash, path, time));
+                    AddHashedFile(ReplacePathPrefixes(new FileCacheEntity(hash, path, time)));
                 }
                 catch (Exception)
                 {
@@ -96,7 +103,7 @@ public sealed class FileCacheManager : IDisposable
         GC.SuppressFinalize(this);
     }
 
-    public List<FileCacheEntity> GetAllFileCaches() => _fileCaches.Values.ToList();
+    public List<FileCacheEntity> GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v).ToList();
 
     public string GetCacheFilePath(string hash, bool isTemporaryFile)
     {
@@ -105,19 +112,18 @@ public sealed class FileCacheManager : IDisposable
 
     public FileCacheEntity? GetFileCacheByHash(string hash)
     {
-        var entry = _fileCaches.FirstOrDefault(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal));
+        if (_fileCaches.TryGetValue(hash, out var hashes))
-        if (!EqualityComparer<KeyValuePair<string, FileCacheEntity>>.Default.Equals(entry, default))
         {
-            return GetValidatedFileCache(entry.Value);
+            var item = hashes.FirstOrDefault();
+            if (item != null) return GetValidatedFileCache(item);
         }
-
         return null;
     }
 
     public FileCacheEntity? GetFileCacheByPath(string path)
     {
         var cleanedPath = path.Replace("/", "\\", StringComparison.OrdinalIgnoreCase).ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), "", StringComparison.OrdinalIgnoreCase);
-        var entry = _fileCaches.Values.FirstOrDefault(f => f.ResolvedFilepath.EndsWith(cleanedPath, StringComparison.OrdinalIgnoreCase));
+        var entry = _fileCaches.SelectMany(v => v.Value).FirstOrDefault(f => f.ResolvedFilepath.EndsWith(cleanedPath, StringComparison.OrdinalIgnoreCase));
 
         if (entry == null)
         {
@@ -130,22 +136,26 @@ public sealed class FileCacheManager : IDisposable
         return validatedCacheEntry;
     }
 
-    public void RemoveHash(FileCacheEntity? entity)
+    public void RemoveHashedFile(FileCacheEntity? fileCache)
     {
-        if (entity != null)
+        if (fileCache == null) return;
+        if (_fileCaches.TryGetValue(fileCache.Hash, out var caches))
         {
-            _logger.LogTrace("Removing {path}", entity.ResolvedFilepath);
+            caches?.RemoveAll(c => string.Equals(c.PrefixedFilePath, fileCache.PrefixedFilePath, StringComparison.Ordinal));
-            _fileCaches.Remove(entity.PrefixedFilePath, out _);
+            if (_fileCaches.Count == 0)
+            {
+                _fileCaches.Remove(fileCache.Hash, out _);
+            }
         }
     }
 
-    public void UpdateHash(FileCacheEntity fileCache)
+    public void UpdateHashedFile(FileCacheEntity fileCache)
     {
         _logger.LogTrace("Updating hash for {path}", fileCache.ResolvedFilepath);
         fileCache.Hash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
         fileCache.LastModifiedDateTicks = new FileInfo(fileCache.ResolvedFilepath).LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
-        _fileCaches.Remove(fileCache.PrefixedFilePath, out _);
+        RemoveHashedFile(fileCache);
-        _fileCaches[fileCache.PrefixedFilePath] = fileCache;
+        AddHashedFile(fileCache);
     }
 
     public (FileState, FileCacheEntity) ValidateFileCacheEntity(FileCacheEntity fileCache)
@@ -167,9 +177,9 @@ public sealed class FileCacheManager : IDisposable
     public void WriteOutFullCsv()
     {
         StringBuilder sb = new();
-        foreach (var entry in _fileCaches.OrderBy(f => f.Value.PrefixedFilePath, StringComparer.OrdinalIgnoreCase))
+        foreach (var entry in _fileCaches.SelectMany(k => k.Value).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase))
         {
-            sb.AppendLine(entry.Value.CsvEntry);
+            sb.AppendLine(entry.CsvEntry);
         }
         if (File.Exists(_csvPath))
         {
@@ -189,12 +199,22 @@ public sealed class FileCacheManager : IDisposable
         }
     }
 
+    private void AddHashedFile(FileCacheEntity fileCache)
+    {
+        if (!_fileCaches.TryGetValue(fileCache.Hash, out var entries))
+        {
+            _fileCaches[fileCache.Hash] = entries = new();
+        }
+
+        entries.Add(fileCache);
+    }
+
     private FileCacheEntity? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath, string? hash = null)
     {
         hash ??= Crypto.GetFileHash(fileInfo.FullName);
         var entity = new FileCacheEntity(hash, prefixedPath, fileInfo.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture));
         entity = ReplacePathPrefixes(entity);
-        _fileCaches[prefixedPath] = entity;
+        AddHashedFile(entity);
         lock (_fileWriteLock)
         {
             File.AppendAllLines(_csvPath, new[] { entity.CsvEntry });
@@ -230,13 +250,13 @@ public sealed class FileCacheManager : IDisposable
         var file = new FileInfo(fileCache.ResolvedFilepath);
         if (!file.Exists)
         {
-            _fileCaches.Remove(fileCache.PrefixedFilePath, out _);
+            RemoveHashedFile(fileCache);
             return null;
         }
 
         if (!string.Equals(file.LastWriteTimeUtc.Ticks.ToString(), fileCache.LastModifiedDateTicks, StringComparison.Ordinal))
         {
-            UpdateHash(fileCache);
+            UpdateHashedFile(fileCache);
         }
 
         return fileCache;

MareSynchronos/FileCache/PeriodicFileScanner.cs

@@ -258,12 +258,12 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
         {
             foreach (var entity in entitiesToUpdate)
             {
-                _fileDbManager.UpdateHash(entity);
+                _fileDbManager.UpdateHashedFile(entity);
             }
 
             foreach (var entity in entitiesToRemove)
             {
-                _fileDbManager.RemoveHash(entity);
+                _fileDbManager.RemoveHashedFile(entity);
             }
 
             _fileDbManager.WriteOutFullCsv();

MareSynchronos/MareSynchronos.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <Authors></Authors>
     <Company></Company>
-    <Version>0.8.22</Version>
+    <Version>0.8.23</Version>
     <Description></Description>
     <Copyright></Copyright>
     <PackageProjectUrl>https://github.com/Penumbra-Sync/client</PackageProjectUrl>

MareSynchronos/PlayerData/Pairs/CachedPlayer.cs

@@ -12,6 +12,8 @@ using MareSynchronos.Utils;
 using MareSynchronos.WebAPI.Files;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
+using System.Collections.Concurrent;
+using System.Diagnostics;
 using ObjectKind = MareSynchronos.API.Data.Enum.ObjectKind;
 
 namespace MareSynchronos.PlayerData.Pairs;
@@ -422,6 +424,8 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase
                 }
             }
 
+            downloadToken.ThrowIfCancellationRequested();
+
             var appToken = _applicationCancellationTokenSource?.Token;
             while ((!_applicationTask?.IsCompleted ?? false)
                    && !downloadToken.IsCancellationRequested
@@ -577,12 +581,19 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase
 
     private List<FileReplacementData> TryCalculateModdedDictionary(CharacterData charaData, out Dictionary<string, string> moddedDictionary, CancellationToken token)
     {
+        Stopwatch st = Stopwatch.StartNew();
         List<FileReplacementData> missingFiles = new();
         moddedDictionary = new Dictionary<string, string>(StringComparer.Ordinal);
+        ConcurrentDictionary<string, string> outputDict = new ConcurrentDictionary<string, string>(StringComparer.Ordinal);
         try
         {
             var replacementList = charaData.FileReplacements.SelectMany(k => k.Value.Where(v => string.IsNullOrEmpty(v.FileSwapPath))).DistinctBy(p => p.Hash).ToList();
-            foreach (var item in replacementList)
+            Parallel.ForEach(replacementList, new ParallelOptions()
+            {
+                CancellationToken = token,
+                MaxDegreeOfParallelism = 4
+            },
+            (item) =>
             {
                 token.ThrowIfCancellationRequested();
                 var fileCache = _fileDbManager.GetFileCacheByHash(item.Hash);
@@ -590,7 +601,7 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase
                 {
                     foreach (var gamePath in item.GamePaths)
                     {
-                        moddedDictionary[gamePath] = fileCache.ResolvedFilepath;
+                        outputDict[gamePath] = fileCache.ResolvedFilepath;
                     }
                 }
                 else
@@ -598,7 +609,9 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase
                     Logger.LogTrace("Missing file: {hash}", item.Hash);
                     missingFiles.Add(item);
                 }
-            }
+            });
+
+            moddedDictionary = outputDict.ToDictionary(k => k.Key, k => k.Value, StringComparer.Ordinal);
 
             foreach (var item in charaData.FileReplacements.SelectMany(k => k.Value.Where(v => !string.IsNullOrEmpty(v.FileSwapPath))).ToList())
             {
@@ -613,7 +626,8 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase
         {
             PluginLog.Error(ex, "Something went wrong during calculation replacements");
         }
-        Logger.LogDebug("ModdedPaths calculated, missing files: {count}", missingFiles.Count);
+        st.Stop();
+        Logger.LogDebug("ModdedPaths calculated in {time}ms, missing files: {count}", st.ElapsedMilliseconds, missingFiles.Count);
         return missingFiles;
     }
 }

MareSynchronos/WebAPI/Files/FileDownloadManager.cs

@@ -259,7 +259,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
                     {
                         Logger.LogError("Hash mismatch after extracting, got {hash}, expected {expectedHash}, deleting file", entry?.Hash, file.Hash);
                         File.Delete(filePath);
-                        _fileDbManager.RemoveHash(entry);
+                        _fileDbManager.RemoveHashedFile(entry);
                     }
                 }
                 catch (Exception ex)