t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve file cache manager performance

Browse Source
This commit is contained in:
rootdarkarchon
2023-04-11 13:50:55 +02:00
parent a1d5f18215
commit 4b278bd456
5 changed files with 64 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
MareSynchronos/FileCache/FileCacheManager.cs
View File

@@ -15,7 +15,7 @@ public sealed class FileCacheManager : IDisposable
private const string _penumbraPrefix = "{penumbra}";
private readonly MareConfigService _configService;
private readonly string _csvPath;
private readonly ConcurrentDictionary<string, FileCacheEntity> _fileCaches = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, List<FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
private readonly object _fileWriteLock = new();
private readonly IpcManager _ipcManager;
private readonly ILogger<FileCacheManager> _logger;
@@ -39,9 +39,16 @@ public sealed class FileCacheManager : IDisposable
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to move BAK to ORG, deleting BAK");
try
{
if (File.Exists(CsvBakPath))
File.Delete(CsvBakPath);
}
catch (Exception ex1)
{
_logger.LogWarning(ex1, "Could not delete bak file");
}
}
}
if (File.Exists(_csvPath))
@@ -55,7 +62,7 @@ public sealed class FileCacheManager : IDisposable
var hash = splittedEntry[0];
var path = splittedEntry[1];
var time = splittedEntry[2];
_fileCaches[path] = ReplacePathPrefixes(new FileCacheEntity(hash, path, time));
AddHashedFile(ReplacePathPrefixes(new FileCacheEntity(hash, path, time)));
}
catch (Exception)
{
@@ -96,7 +103,7 @@ public sealed class FileCacheManager : IDisposable
GC.SuppressFinalize(this);
}
public List<FileCacheEntity> GetAllFileCaches() => _fileCaches.Values.ToList();
public List<FileCacheEntity> GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v).ToList();
public string GetCacheFilePath(string hash, bool isTemporaryFile)
{
@@ -105,19 +112,18 @@ public sealed class FileCacheManager : IDisposable
public FileCacheEntity? GetFileCacheByHash(string hash)
{
var entry = _fileCaches.FirstOrDefault(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal));
if (!EqualityComparer<KeyValuePair<string, FileCacheEntity>>.Default.Equals(entry, default))
if (_fileCaches.TryGetValue(hash, out var hashes))
{
return GetValidatedFileCache(entry.Value);
var item = hashes.FirstOrDefault();
if (item != null) return GetValidatedFileCache(item);
}
return null;
}
public FileCacheEntity? GetFileCacheByPath(string path)
{
var cleanedPath = path.Replace("/", "\\", StringComparison.OrdinalIgnoreCase).ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), "", StringComparison.OrdinalIgnoreCase);
var entry = _fileCaches.Values.FirstOrDefault(f => f.ResolvedFilepath.EndsWith(cleanedPath, StringComparison.OrdinalIgnoreCase));
var entry = _fileCaches.SelectMany(v => v.Value).FirstOrDefault(f => f.ResolvedFilepath.EndsWith(cleanedPath, StringComparison.OrdinalIgnoreCase));
if (entry == null)
{
@@ -130,22 +136,26 @@ public sealed class FileCacheManager : IDisposable
return validatedCacheEntry;
}
public void RemoveHash(FileCacheEntity? entity)
public void RemoveHashedFile(FileCacheEntity? fileCache)
{
if (entity != null)
if (fileCache == null) return;
if (_fileCaches.TryGetValue(fileCache.Hash, out var caches))
{
_logger.LogTrace("Removing {path}", entity.ResolvedFilepath);
_fileCaches.Remove(entity.PrefixedFilePath, out _);
caches?.RemoveAll(c => string.Equals(c.PrefixedFilePath, fileCache.PrefixedFilePath, StringComparison.Ordinal));
if (_fileCaches.Count == 0)
{
_fileCaches.Remove(fileCache.Hash, out _);
}
}
}
public void UpdateHash(FileCacheEntity fileCache)
public void UpdateHashedFile(FileCacheEntity fileCache)
{
_logger.LogTrace("Updating hash for {path}", fileCache.ResolvedFilepath);
fileCache.Hash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
fileCache.LastModifiedDateTicks = new FileInfo(fileCache.ResolvedFilepath).LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
_fileCaches.Remove(fileCache.PrefixedFilePath, out _);
_fileCaches[fileCache.PrefixedFilePath] = fileCache;
RemoveHashedFile(fileCache);
AddHashedFile(fileCache);
}
public (FileState, FileCacheEntity) ValidateFileCacheEntity(FileCacheEntity fileCache)
@@ -167,9 +177,9 @@ public sealed class FileCacheManager : IDisposable
public void WriteOutFullCsv()
{
StringBuilder sb = new();
foreach (var entry in _fileCaches.OrderBy(f => f.Value.PrefixedFilePath, StringComparer.OrdinalIgnoreCase))
foreach (var entry in _fileCaches.SelectMany(k => k.Value).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase))
{
sb.AppendLine(entry.Value.CsvEntry);
sb.AppendLine(entry.CsvEntry);
}
if (File.Exists(_csvPath))
{
@@ -189,12 +199,22 @@ public sealed class FileCacheManager : IDisposable
}
}
private void AddHashedFile(FileCacheEntity fileCache)
{
if (!_fileCaches.TryGetValue(fileCache.Hash, out var entries))
{
_fileCaches[fileCache.Hash] = entries = new();
}
entries.Add(fileCache);
}
private FileCacheEntity? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath, string? hash = null)
{
hash ??= Crypto.GetFileHash(fileInfo.FullName);
var entity = new FileCacheEntity(hash, prefixedPath, fileInfo.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture));
entity = ReplacePathPrefixes(entity);
_fileCaches[prefixedPath] = entity;
AddHashedFile(entity);
lock (_fileWriteLock)
{
File.AppendAllLines(_csvPath, new[] { entity.CsvEntry });
@@ -230,13 +250,13 @@ public sealed class FileCacheManager : IDisposable
var file = new FileInfo(fileCache.ResolvedFilepath);
if (!file.Exists)
{
_fileCaches.Remove(fileCache.PrefixedFilePath, out _);
RemoveHashedFile(fileCache);
return null;
}
if (!string.Equals(file.LastWriteTimeUtc.Ticks.ToString(), fileCache.LastModifiedDateTicks, StringComparison.Ordinal))
{
UpdateHash(fileCache);
UpdateHashedFile(fileCache);
}
return fileCache;

4
MareSynchronos/FileCache/PeriodicFileScanner.cs
View File

@@ -258,12 +258,12 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
{
foreach (var entity in entitiesToUpdate)
{
_fileDbManager.UpdateHash(entity);
_fileDbManager.UpdateHashedFile(entity);
}
foreach (var entity in entitiesToRemove)
{
_fileDbManager.RemoveHash(entity);
_fileDbManager.RemoveHashedFile(entity);
}
_fileDbManager.WriteOutFullCsv();

2
MareSynchronos/MareSynchronos.csproj
View File

@@ -3,7 +3,7 @@
<PropertyGroup>
<Authors></Authors>
<Company></Company>
<Version>0.8.22</Version>
<Version>0.8.23</Version>
<Description></Description>
<Copyright></Copyright>
<PackageProjectUrl>https://github.com/Penumbra-Sync/client</PackageProjectUrl>

22
MareSynchronos/PlayerData/Pairs/CachedPlayer.cs
View File

@@ -12,6 +12,8 @@ using MareSynchronos.Utils;
using MareSynchronos.WebAPI.Files;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
using System.Diagnostics;
using ObjectKind = MareSynchronos.API.Data.Enum.ObjectKind;
namespace MareSynchronos.PlayerData.Pairs;
@@ -422,6 +424,8 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase
}
}
downloadToken.ThrowIfCancellationRequested();
var appToken = _applicationCancellationTokenSource?.Token;
while ((!_applicationTask?.IsCompleted ?? false)
&& !downloadToken.IsCancellationRequested
@@ -577,12 +581,19 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase
private List<FileReplacementData> TryCalculateModdedDictionary(CharacterData charaData, out Dictionary<string, string> moddedDictionary, CancellationToken token)
{
Stopwatch st = Stopwatch.StartNew();
List<FileReplacementData> missingFiles = new();
moddedDictionary = new Dictionary<string, string>(StringComparer.Ordinal);
ConcurrentDictionary<string, string> outputDict = new ConcurrentDictionary<string, string>(StringComparer.Ordinal);
try
{
var replacementList = charaData.FileReplacements.SelectMany(k => k.Value.Where(v => string.IsNullOrEmpty(v.FileSwapPath))).DistinctBy(p => p.Hash).ToList();
foreach (var item in replacementList)
Parallel.ForEach(replacementList, new ParallelOptions()
{
CancellationToken = token,
MaxDegreeOfParallelism = 4
},
(item) =>
{
token.ThrowIfCancellationRequested();
var fileCache = _fileDbManager.GetFileCacheByHash(item.Hash);
@@ -590,7 +601,7 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase
{
foreach (var gamePath in item.GamePaths)
{
moddedDictionary[gamePath] = fileCache.ResolvedFilepath;
outputDict[gamePath] = fileCache.ResolvedFilepath;
}
}
else
@@ -598,7 +609,9 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase
Logger.LogTrace("Missing file: {hash}", item.Hash);
missingFiles.Add(item);
}
}
});
moddedDictionary = outputDict.ToDictionary(k => k.Key, k => k.Value, StringComparer.Ordinal);
foreach (var item in charaData.FileReplacements.SelectMany(k => k.Value.Where(v => !string.IsNullOrEmpty(v.FileSwapPath))).ToList())
{
@@ -613,7 +626,8 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase
{
PluginLog.Error(ex, "Something went wrong during calculation replacements");
}
Logger.LogDebug("ModdedPaths calculated, missing files: {count}", missingFiles.Count);
st.Stop();
Logger.LogDebug("ModdedPaths calculated in {time}ms, missing files: {count}", st.ElapsedMilliseconds, missingFiles.Count);
return missingFiles;
}
}

2
MareSynchronos/WebAPI/Files/FileDownloadManager.cs
View File

@@ -259,7 +259,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
{
Logger.LogError("Hash mismatch after extracting, got {hash}, expected {expectedHash}, deleting file", entry?.Hash, file.Hash);
File.Delete(filePath);
_fileDbManager.RemoveHash(entry);
_fileDbManager.RemoveHashedFile(entry);
}
}
catch (Exception ex)