t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reduce hitches and do some other changes to FSW

Browse Source
This commit is contained in:
Stanley Dimant
2024-02-15 11:22:18 +01:00
committed by Loporrit
parent e98d5ce1ea
commit f9ca55db43
17 changed files with 140 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
MareSynchronos/FileCache/CacheMonitor.cs
View File

@@ -68,6 +68,11 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
{
try
{
while (_dalamudUtil.IsOnFrameworkThread && !token.IsCancellationRequested)
{
await Task.Delay(1).ConfigureAwait(false);
}
RecalculateFileCacheSize(token);
}
catch
@@ -272,18 +277,41 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
}
}
if (changes.Any(c => c.Value.ChangeType == WatcherChangeTypes.Deleted))
{
lock (_fileDbManager)
{
foreach (var change in changes)
{
Logger.LogDebug("FSW Change: {change} = {val}", change.Key, change.Value);
_ = _fileDbManager.GetFileCacheByPath(change.Key);
}
HandleChanges(changes);
}
_fileDbManager.WriteOutFullCsv();
private void HandleChanges(Dictionary<string, WatcherChange> changes)
{
lock (_fileDbManager)
{
var deletedEntries = changes.Where(c => c.Value.ChangeType == WatcherChangeTypes.Deleted).Select(c => c.Key);
var renamedEntries = changes.Where(c => c.Value.ChangeType == WatcherChangeTypes.Renamed);
var remainingEntries = changes.Where(c => c.Value.ChangeType != WatcherChangeTypes.Deleted).Select(c => c.Key);
foreach (var entry in deletedEntries)
{
Logger.LogDebug("FSW Change: Deletion - {val}", entry);
}
foreach (var entry in renamedEntries)
{
Logger.LogDebug("FSW Change: Renamed - {oldVal} => {val}", entry.Value.OldPath, entry.Key);
}
foreach (var entry in remainingEntries)
{
Logger.LogDebug("FSW Change: Creation or Change - {val}", entry);
}
var allChanges = deletedEntries
.Concat(renamedEntries.Select(c => c.Value.OldPath!))
.Concat(renamedEntries.Select(c => c.Key))
.Concat(remainingEntries)
.ToArray();
_ = _fileDbManager.GetFileCachesByPaths(allChanges);
_fileDbManager.WriteOutFullCsv();
}
}
@@ -315,24 +343,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
}
}
lock (_fileDbManager)
{
foreach (var change in changes)
{
Logger.LogDebug("FSW Change: {change} = {val}", change.Key, change.Value);
if (change.Value.ChangeType == WatcherChangeTypes.Deleted)
{
_fileDbManager.GetFileCacheByPath(change.Key);
}
else
{
if (change.Value.OldPath != null) _fileDbManager.GetFileCacheByPath(change.Value.OldPath);
_fileDbManager.CreateFileEntry(change.Key);
}
}
_fileDbManager.WriteOutFullCsv();
}
HandleChanges(changes);
}
public void InvokeScan()
@@ -397,7 +408,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
}
FileCacheSize = Directory.EnumerateFiles(_configService.Current.CacheFolder)
.AsParallel().Sum(f =>
.Sum(f =>
{
token.ThrowIfCancellationRequested();

73
MareSynchronos/FileCache/FileCacheManager.cs
View File

@@ -20,6 +20,7 @@ public sealed class FileCacheManager : IHostedService
private readonly MareMediator _mareMediator;
private readonly string _csvPath;
private readonly ConcurrentDictionary<string, List<FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
private readonly SemaphoreSlim _getCachesByPathsSemaphore = new(1, 1);
private readonly object _fileWriteLock = new();
private readonly IpcManager _ipcManager;
private readonly ILogger<FileCacheManager> _logger;
@@ -64,9 +65,9 @@ public sealed class FileCacheManager : IHostedService
List<FileCacheEntity> output = [];
if (_fileCaches.TryGetValue(hash, out var fileCacheEntities))
{
foreach (var filecache in fileCacheEntities.ToList())
foreach (var fileCache in fileCacheEntities.ToList())
{
var validated = GetValidatedFileCache(filecache);
var validated = GetValidatedFileCache(fileCache);
if (validated != null) output.Add(validated);
}
}
@@ -79,7 +80,7 @@ public sealed class FileCacheManager : IHostedService
_mareMediator.Publish(new HaltScanMessage(nameof(ValidateLocalIntegrity)));
_logger.LogInformation("Validating local storage");
var cacheEntries = _fileCaches.SelectMany(v => v.Value).Where(v => v.IsCacheEntry).ToList();
List<FileCacheEntity> brokenEntities = new();
List<FileCacheEntity> brokenEntities = [];
int i = 0;
foreach (var fileCache in cacheEntries)
{
@@ -176,32 +177,44 @@ public sealed class FileCacheManager : IHostedService
public Dictionary<string, FileCacheEntity?> GetFileCachesByPaths(string[] paths)
{
var cleanedPaths = paths.Distinct(StringComparer.OrdinalIgnoreCase).ToDictionary(p => p,
p => p.Replace("/", "\\", StringComparison.OrdinalIgnoreCase)
.Replace(_ipcManager.PenumbraModDirectory!,
_ipcManager.PenumbraModDirectory!.EndsWith('\\') ? (PenumbraPrefix + "\\") : PenumbraPrefix,
StringComparison.OrdinalIgnoreCase),
StringComparer.OrdinalIgnoreCase);
_getCachesByPathsSemaphore.Wait();
Dictionary<string, FileCacheEntity?> result = new(StringComparer.OrdinalIgnoreCase);
var dict = _fileCaches.SelectMany(f => f.Value).Where(f => f.PrefixedFilePath.StartsWith(PenumbraPrefix, StringComparison.Ordinal))
.ToDictionary(d => d.PrefixedFilePath, d => d, StringComparer.Ordinal);
foreach (var entry in cleanedPaths)
try
{
if (dict.TryGetValue(entry.Value, out var entity))
{
var validatedCache = GetValidatedFileCache(entity);
result.Add(entry.Key, validatedCache);
}
else
{
result.Add(entry.Key, CreateFileEntry(entry.Key));
}
}
var cleanedPaths = paths.Distinct(StringComparer.OrdinalIgnoreCase).ToDictionary(p => p,
p => p.Replace("/", "\\", StringComparison.OrdinalIgnoreCase)
.Replace(_ipcManager.PenumbraModDirectory!, PenumbraPrefix, StringComparison.OrdinalIgnoreCase)
.Replace(_configService.Current.CacheFolder, CachePrefix, StringComparison.OrdinalIgnoreCase)
.Replace("\\\\", "\\", StringComparison.Ordinal),
StringComparer.OrdinalIgnoreCase);
return result;
Dictionary<string, FileCacheEntity?> result = new(StringComparer.OrdinalIgnoreCase);
var dict = _fileCaches.SelectMany(f => f.Value)
.ToDictionary(d => d.PrefixedFilePath, d => d, StringComparer.Ordinal);
foreach (var entry in cleanedPaths)
{
if (dict.TryGetValue(entry.Value, out var entity))
{
var validatedCache = GetValidatedFileCache(entity);
result.Add(entry.Key, validatedCache);
}
else
{
if (!entry.Value.Contains(CachePrefix, StringComparison.Ordinal))
result.Add(entry.Key, CreateFileEntry(entry.Key));
else
result.Add(entry.Key, CreateCacheEntry(entry.Key));
}
}
return result;
}
finally
{
_getCachesByPathsSemaphore.Release();
}
}
public void RemoveHashedFile(string hash, string prefixedFilePath)
@@ -281,8 +294,6 @@ public sealed class FileCacheManager : IHostedService
try
{
RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath);
FileInfo fileInfo = new(fileCache.ResolvedFilepath);
FileInfo oldCache = fileInfo;
var extensionPath = fileCache.ResolvedFilepath.ToUpper(CultureInfo.InvariantCulture) + "." + ext;
File.Move(fileCache.ResolvedFilepath, extensionPath, overwrite: true);
var newHashedEntity = new FileCacheEntity(fileCache.Hash, fileCache.PrefixedFilePath + "." + ext, DateTime.UtcNow.Ticks.ToString(CultureInfo.InvariantCulture));
@@ -329,9 +340,9 @@ public sealed class FileCacheManager : IHostedService
private FileCacheEntity? GetValidatedFileCache(FileCacheEntity fileCache)
{
var resulingFileCache = ReplacePathPrefixes(fileCache);
resulingFileCache = Validate(resulingFileCache);
return resulingFileCache;
var resultingFileCache = ReplacePathPrefixes(fileCache);
resultingFileCache = Validate(resultingFileCache);
return resultingFileCache;
}
private FileCacheEntity ReplacePathPrefixes(FileCacheEntity fileCache)