diff --git a/MareSynchronos/FileCache/FileCacheManager.cs b/MareSynchronos/FileCache/FileCacheManager.cs index 34a05bb..808286e 100644 --- a/MareSynchronos/FileCache/FileCacheManager.cs +++ b/MareSynchronos/FileCache/FileCacheManager.cs @@ -15,7 +15,7 @@ public sealed class FileCacheManager : IDisposable private const string _penumbraPrefix = "{penumbra}"; private readonly MareConfigService _configService; private readonly string _csvPath; - private readonly ConcurrentDictionary _fileCaches = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary> _fileCaches = new(StringComparer.Ordinal); private readonly object _fileWriteLock = new(); private readonly IpcManager _ipcManager; private readonly ILogger _logger; @@ -39,8 +39,15 @@ public sealed class FileCacheManager : IDisposable catch (Exception ex) { _logger.LogWarning(ex, "Failed to move BAK to ORG, deleting BAK"); - if (File.Exists(CsvBakPath)) - File.Delete(CsvBakPath); + try + { + if (File.Exists(CsvBakPath)) + File.Delete(CsvBakPath); + } + catch (Exception ex1) + { + _logger.LogWarning(ex1, "Could not delete bak file"); + } } } @@ -55,7 +62,7 @@ public sealed class FileCacheManager : IDisposable var hash = splittedEntry[0]; var path = splittedEntry[1]; var time = splittedEntry[2]; - _fileCaches[path] = ReplacePathPrefixes(new FileCacheEntity(hash, path, time)); + AddHashedFile(ReplacePathPrefixes(new FileCacheEntity(hash, path, time))); } catch (Exception) { @@ -96,7 +103,7 @@ public sealed class FileCacheManager : IDisposable GC.SuppressFinalize(this); } - public List GetAllFileCaches() => _fileCaches.Values.ToList(); + public List GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v).ToList(); public string GetCacheFilePath(string hash, bool isTemporaryFile) { @@ -105,19 +112,18 @@ public sealed class FileCacheManager : IDisposable public FileCacheEntity? GetFileCacheByHash(string hash) { - var entry = _fileCaches.FirstOrDefault(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal)); - if (!EqualityComparer>.Default.Equals(entry, default)) + if (_fileCaches.TryGetValue(hash, out var hashes)) { - return GetValidatedFileCache(entry.Value); + var item = hashes.FirstOrDefault(); + if (item != null) return GetValidatedFileCache(item); } - return null; } public FileCacheEntity? GetFileCacheByPath(string path) { var cleanedPath = path.Replace("/", "\\", StringComparison.OrdinalIgnoreCase).ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), "", StringComparison.OrdinalIgnoreCase); - var entry = _fileCaches.Values.FirstOrDefault(f => f.ResolvedFilepath.EndsWith(cleanedPath, StringComparison.OrdinalIgnoreCase)); + var entry = _fileCaches.SelectMany(v => v.Value).FirstOrDefault(f => f.ResolvedFilepath.EndsWith(cleanedPath, StringComparison.OrdinalIgnoreCase)); if (entry == null) { @@ -130,22 +136,26 @@ public sealed class FileCacheManager : IDisposable return validatedCacheEntry; } - public void RemoveHash(FileCacheEntity? entity) + public void RemoveHashedFile(FileCacheEntity? fileCache) { - if (entity != null) + if (fileCache == null) return; + if (_fileCaches.TryGetValue(fileCache.Hash, out var caches)) { - _logger.LogTrace("Removing {path}", entity.ResolvedFilepath); - _fileCaches.Remove(entity.PrefixedFilePath, out _); + caches?.RemoveAll(c => string.Equals(c.PrefixedFilePath, fileCache.PrefixedFilePath, StringComparison.Ordinal)); + if (_fileCaches.Count == 0) + { + _fileCaches.Remove(fileCache.Hash, out _); + } } } - public void UpdateHash(FileCacheEntity fileCache) + public void UpdateHashedFile(FileCacheEntity fileCache) { _logger.LogTrace("Updating hash for {path}", fileCache.ResolvedFilepath); fileCache.Hash = Crypto.GetFileHash(fileCache.ResolvedFilepath); fileCache.LastModifiedDateTicks = new FileInfo(fileCache.ResolvedFilepath).LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture); - _fileCaches.Remove(fileCache.PrefixedFilePath, out _); - _fileCaches[fileCache.PrefixedFilePath] = fileCache; + RemoveHashedFile(fileCache); + AddHashedFile(fileCache); } public (FileState, FileCacheEntity) ValidateFileCacheEntity(FileCacheEntity fileCache) @@ -167,9 +177,9 @@ public sealed class FileCacheManager : IDisposable public void WriteOutFullCsv() { StringBuilder sb = new(); - foreach (var entry in _fileCaches.OrderBy(f => f.Value.PrefixedFilePath, StringComparer.OrdinalIgnoreCase)) + foreach (var entry in _fileCaches.SelectMany(k => k.Value).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase)) { - sb.AppendLine(entry.Value.CsvEntry); + sb.AppendLine(entry.CsvEntry); } if (File.Exists(_csvPath)) { @@ -189,12 +199,22 @@ public sealed class FileCacheManager : IDisposable } } + private void AddHashedFile(FileCacheEntity fileCache) + { + if (!_fileCaches.TryGetValue(fileCache.Hash, out var entries)) + { + _fileCaches[fileCache.Hash] = entries = new(); + } + + entries.Add(fileCache); + } + private FileCacheEntity? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath, string? hash = null) { hash ??= Crypto.GetFileHash(fileInfo.FullName); var entity = new FileCacheEntity(hash, prefixedPath, fileInfo.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture)); entity = ReplacePathPrefixes(entity); - _fileCaches[prefixedPath] = entity; + AddHashedFile(entity); lock (_fileWriteLock) { File.AppendAllLines(_csvPath, new[] { entity.CsvEntry }); @@ -230,13 +250,13 @@ public sealed class FileCacheManager : IDisposable var file = new FileInfo(fileCache.ResolvedFilepath); if (!file.Exists) { - _fileCaches.Remove(fileCache.PrefixedFilePath, out _); + RemoveHashedFile(fileCache); return null; } if (!string.Equals(file.LastWriteTimeUtc.Ticks.ToString(), fileCache.LastModifiedDateTicks, StringComparison.Ordinal)) { - UpdateHash(fileCache); + UpdateHashedFile(fileCache); } return fileCache; diff --git a/MareSynchronos/FileCache/PeriodicFileScanner.cs b/MareSynchronos/FileCache/PeriodicFileScanner.cs index 2d539a6..bdd8af3 100644 --- a/MareSynchronos/FileCache/PeriodicFileScanner.cs +++ b/MareSynchronos/FileCache/PeriodicFileScanner.cs @@ -258,12 +258,12 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase { foreach (var entity in entitiesToUpdate) { - _fileDbManager.UpdateHash(entity); + _fileDbManager.UpdateHashedFile(entity); } foreach (var entity in entitiesToRemove) { - _fileDbManager.RemoveHash(entity); + _fileDbManager.RemoveHashedFile(entity); } _fileDbManager.WriteOutFullCsv(); diff --git a/MareSynchronos/MareSynchronos.csproj b/MareSynchronos/MareSynchronos.csproj index def9c30..619485f 100644 --- a/MareSynchronos/MareSynchronos.csproj +++ b/MareSynchronos/MareSynchronos.csproj @@ -3,7 +3,7 @@ - 0.8.22 + 0.8.23 https://github.com/Penumbra-Sync/client diff --git a/MareSynchronos/PlayerData/Pairs/CachedPlayer.cs b/MareSynchronos/PlayerData/Pairs/CachedPlayer.cs index 7c22c2e..b5de7ce 100644 --- a/MareSynchronos/PlayerData/Pairs/CachedPlayer.cs +++ b/MareSynchronos/PlayerData/Pairs/CachedPlayer.cs @@ -12,6 +12,8 @@ using MareSynchronos.Utils; using MareSynchronos.WebAPI.Files; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using System.Collections.Concurrent; +using System.Diagnostics; using ObjectKind = MareSynchronos.API.Data.Enum.ObjectKind; namespace MareSynchronos.PlayerData.Pairs; @@ -422,6 +424,8 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase } } + downloadToken.ThrowIfCancellationRequested(); + var appToken = _applicationCancellationTokenSource?.Token; while ((!_applicationTask?.IsCompleted ?? false) && !downloadToken.IsCancellationRequested @@ -577,12 +581,19 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase private List TryCalculateModdedDictionary(CharacterData charaData, out Dictionary moddedDictionary, CancellationToken token) { + Stopwatch st = Stopwatch.StartNew(); List missingFiles = new(); moddedDictionary = new Dictionary(StringComparer.Ordinal); + ConcurrentDictionary outputDict = new ConcurrentDictionary(StringComparer.Ordinal); try { var replacementList = charaData.FileReplacements.SelectMany(k => k.Value.Where(v => string.IsNullOrEmpty(v.FileSwapPath))).DistinctBy(p => p.Hash).ToList(); - foreach (var item in replacementList) + Parallel.ForEach(replacementList, new ParallelOptions() + { + CancellationToken = token, + MaxDegreeOfParallelism = 4 + }, + (item) => { token.ThrowIfCancellationRequested(); var fileCache = _fileDbManager.GetFileCacheByHash(item.Hash); @@ -590,7 +601,7 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase { foreach (var gamePath in item.GamePaths) { - moddedDictionary[gamePath] = fileCache.ResolvedFilepath; + outputDict[gamePath] = fileCache.ResolvedFilepath; } } else @@ -598,7 +609,9 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase Logger.LogTrace("Missing file: {hash}", item.Hash); missingFiles.Add(item); } - } + }); + + moddedDictionary = outputDict.ToDictionary(k => k.Key, k => k.Value, StringComparer.Ordinal); foreach (var item in charaData.FileReplacements.SelectMany(k => k.Value.Where(v => !string.IsNullOrEmpty(v.FileSwapPath))).ToList()) { @@ -613,7 +626,8 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase { PluginLog.Error(ex, "Something went wrong during calculation replacements"); } - Logger.LogDebug("ModdedPaths calculated, missing files: {count}", missingFiles.Count); + st.Stop(); + Logger.LogDebug("ModdedPaths calculated in {time}ms, missing files: {count}", st.ElapsedMilliseconds, missingFiles.Count); return missingFiles; } } \ No newline at end of file diff --git a/MareSynchronos/WebAPI/Files/FileDownloadManager.cs b/MareSynchronos/WebAPI/Files/FileDownloadManager.cs index 67585f1..cc486f5 100644 --- a/MareSynchronos/WebAPI/Files/FileDownloadManager.cs +++ b/MareSynchronos/WebAPI/Files/FileDownloadManager.cs @@ -259,7 +259,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase { Logger.LogError("Hash mismatch after extracting, got {hash}, expected {expectedHash}, deleting file", entry?.Hash, file.Hash); File.Delete(filePath); - _fileDbManager.RemoveHash(entry); + _fileDbManager.RemoveHashedFile(entry); } } catch (Exception ex)