t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Draft] Update 0.8 (#46)

Browse Source 
* move stuff out into file transfer manager

* obnoxious unsupported version text, adjustments to filetransfermanager

* add back file upload transfer progress

* restructure code

* cleanup some more stuff I guess

* downloadids by playername

* individual anim/sound bs

* fix migration stuff, finalize impl of individual sound/anim pause

* fixes with logging stuff

* move download manager to transient

* rework dl ui first iteration

* some refactoring and cleanup

* more code cleanup

* refactoring

* switch to hostbuilder

* some more rework I guess

* more refactoring

* clean up mediator calls and disposal

* fun code cleanup

* push error message when log level is set to anything but information in non-debug builds

* remove notificationservice

* move message to after login

* add download bars to gameworld

* fixes download progress bar

* set gpose ui min and max size

* remove unnecessary usings

* adjustments to reconnection logic

* add options to set visible/offline groups visibility

* add impl of uploading display, transfer list in settings ui

* attempt to fix issues with server selection

* add back download status to compact ui

* make dl bar fixed size based

* some fixes for upload/download handling

* adjust text from Syncing back to Uploading

---------

Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>
Co-authored-by: Stanley Dimant <stanley.dimant@varian.com>
This commit is contained in:
rootdarkarchon
2023-03-14 19:48:35 +01:00
committed by GitHub
parent 0824ba434b
commit 0c87e84f25
109 changed files with 7323 additions and 6488 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
MareSynchronos/FileCache/FileCacheEntity.cs
View File

@@ -1,17 +1,9 @@
#nullable disable
using System.Globalization;
namespace MareSynchronos.FileCache;
public class FileCacheEntity
{
public string ResolvedFilepath { get; private set; } = string.Empty;
public string Hash { get; set; }
public string PrefixedFilePath { get; init; }
public string LastModifiedDateTicks { get; set; }
public FileCacheEntity(string hash, string path, string lastModifiedDateTicks)
{
Hash = hash;
@@ -19,10 +11,14 @@ public class FileCacheEntity
LastModifiedDateTicks = lastModifiedDateTicks;
}
public string CsvEntry => $"{Hash}{FileCacheManager.CsvSplit}{PrefixedFilePath}{FileCacheManager.CsvSplit}{LastModifiedDateTicks}";
public string Hash { get; set; }
public string LastModifiedDateTicks { get; set; }
public string PrefixedFilePath { get; init; }
public string ResolvedFilepath { get; private set; } = string.Empty;
public void SetResolvedFilePath(string filePath)
{
ResolvedFilepath = filePath.ToLowerInvariant().Replace("\\\\", "\\", StringComparison.Ordinal);
}
public string CsvEntry => $"{Hash}{FileCacheManager.CsvSplit}{PrefixedFilePath}{FileCacheManager.CsvSplit}{LastModifiedDateTicks.ToString(CultureInfo.InvariantCulture)}";
}
}

243
MareSynchronos/FileCache/FileCacheManager.cs
View File

@@ -1,4 +1,4 @@
using MareSynchronos.Managers;
using MareSynchronos.Interop;
using MareSynchronos.MareConfiguration;
using MareSynchronos.Utils;
using Microsoft.Extensions.Logging;
@@ -8,18 +8,17 @@ using System.Text;
namespace MareSynchronos.FileCache;
public class FileCacheManager : IDisposable
public sealed class FileCacheManager : IDisposable
{
private const string _penumbraPrefix = "{penumbra}";
public const string CsvSplit = "|";
private const string _cachePrefix = "{cache}";
private readonly ILogger<FileCacheManager> _logger;
private readonly IpcManager _ipcManager;
private const string _penumbraPrefix = "{penumbra}";
private readonly MareConfigService _configService;
private readonly string _csvPath;
private string CsvBakPath => _csvPath + ".bak";
private readonly ConcurrentDictionary<string, FileCacheEntity> _fileCaches = new(StringComparer.Ordinal);
public const string CsvSplit = "|";
private readonly object _fileWriteLock = new();
private readonly IpcManager _ipcManager;
private readonly ILogger<FileCacheManager> _logger;
public FileCacheManager(ILogger<FileCacheManager> logger, IpcManager ipcManager, MareConfigService configService)
{
@@ -51,12 +50,114 @@ public class FileCacheManager : IDisposable
}
catch (Exception)
{
_logger.LogWarning($"Failed to initialize entry {entry}, ignoring");
_logger.LogWarning("Failed to initialize entry {entry}, ignoring", entry);
}
}
}
}
private string CsvBakPath => _csvPath + ".bak";
public FileCacheEntity? CreateCacheEntry(string path)
{
_logger.LogTrace("Creating cache entry for {path}", path);
FileInfo fi = new(path);
if (!fi.Exists) return null;
var fullName = fi.FullName.ToLowerInvariant();
if (!fullName.Contains(_configService.Current.CacheFolder.ToLowerInvariant(), StringComparison.Ordinal)) return null;
string prefixedPath = fullName.Replace(_configService.Current.CacheFolder.ToLowerInvariant(), _cachePrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
return CreateFileCacheEntity(fi, prefixedPath, fi.Name.ToUpper(CultureInfo.InvariantCulture));
}
public FileCacheEntity? CreateFileEntry(string path)
{
_logger.LogTrace("Creating file entry for {path}", path);
FileInfo fi = new(path);
if (!fi.Exists) return null;
var fullName = fi.FullName.ToLowerInvariant();
if (!fullName.Contains(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), StringComparison.Ordinal)) return null;
string prefixedPath = fullName.Replace(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), _penumbraPrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
return CreateFileCacheEntity(fi, prefixedPath);
}
public void Dispose()
{
_logger.LogTrace("Disposing {type}", GetType());
WriteOutFullCsv();
GC.SuppressFinalize(this);
}
public List<FileCacheEntity> GetAllFileCaches() => _fileCaches.Values.ToList();
public string GetCacheFilePath(string hash, bool isTemporaryFile)
{
return Path.Combine(_configService.Current.CacheFolder, hash + (isTemporaryFile ? ".tmp" : string.Empty));
}
public FileCacheEntity? GetFileCacheByHash(string hash)
{
if (_fileCaches.Any(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal)))
{
return GetValidatedFileCache(_fileCaches.Where(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal))
.OrderByDescending(f => f.Value.PrefixedFilePath.Length)
.FirstOrDefault(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal)).Value);
}
return null;
}
public FileCacheEntity? GetFileCacheByPath(string path)
{
var cleanedPath = path.Replace("/", "\\", StringComparison.OrdinalIgnoreCase).ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), "", StringComparison.OrdinalIgnoreCase);
var entry = _fileCaches.Values.FirstOrDefault(f => f.ResolvedFilepath.EndsWith(cleanedPath, StringComparison.OrdinalIgnoreCase));
if (entry == null)
{
_logger.LogDebug("Found no entries for {path}", cleanedPath);
return CreateFileEntry(path);
}
var validatedCacheEntry = GetValidatedFileCache(entry);
return validatedCacheEntry;
}
public void RemoveHash(FileCacheEntity entity)
{
_logger.LogTrace("Removing {path}", entity.ResolvedFilepath);
_fileCaches.Remove(entity.PrefixedFilePath, out _);
}
public string ResolveFileReplacement(string gamePath)
{
return _ipcManager.PenumbraResolvePath(gamePath);
}
public void UpdateHash(FileCacheEntity fileCache)
{
_logger.LogTrace("Updating hash for {path}", fileCache.ResolvedFilepath);
fileCache.Hash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
fileCache.LastModifiedDateTicks = new FileInfo(fileCache.ResolvedFilepath).LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
_fileCaches.Remove(fileCache.PrefixedFilePath, out _);
_fileCaches[fileCache.PrefixedFilePath] = fileCache;
}
public (FileState, FileCacheEntity) ValidateFileCacheEntity(FileCacheEntity fileCache)
{
fileCache = ReplacePathPrefixes(fileCache);
FileInfo fi = new(fileCache.ResolvedFilepath);
if (!fi.Exists)
{
return (FileState.RequireDeletion, fileCache);
}
if (!string.Equals(fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), fileCache.LastModifiedDateTicks, StringComparison.Ordinal))
{
return (FileState.RequireUpdate, fileCache);
}
return (FileState.Valid, fileCache);
}
public void WriteOutFullCsv()
{
StringBuilder sb = new();
@@ -82,74 +183,6 @@ public class FileCacheManager : IDisposable
}
}
public List<FileCacheEntity> GetAllFileCaches() => _fileCaches.Values.ToList();
public FileCacheEntity? GetFileCacheByHash(string hash)
{
if (_fileCaches.Any(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal)))
{
return GetValidatedFileCache(_fileCaches.Where(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal))
.OrderByDescending(f => f.Value.PrefixedFilePath.Length)
.FirstOrDefault(f => string.Equals(f.Value.Hash, hash, StringComparison.Ordinal)).Value);
}
return null;
}
public (FileState, FileCacheEntity) ValidateFileCacheEntity(FileCacheEntity fileCache)
{
fileCache = ReplacePathPrefixes(fileCache);
FileInfo fi = new(fileCache.ResolvedFilepath);
if (!fi.Exists)
{
return (FileState.RequireDeletion, fileCache);
}
if (!string.Equals(fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), fileCache.LastModifiedDateTicks, StringComparison.Ordinal))
{
return (FileState.RequireUpdate, fileCache);
}
return (FileState.Valid, fileCache);
}
public FileCacheEntity? GetFileCacheByPath(string path)
{
var cleanedPath = path.Replace("/", "\\", StringComparison.OrdinalIgnoreCase).ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), "", StringComparison.OrdinalIgnoreCase);
var entry = _fileCaches.Values.FirstOrDefault(f => f.ResolvedFilepath.EndsWith(cleanedPath, StringComparison.OrdinalIgnoreCase));
if (entry == null)
{
_logger.LogDebug("Found no entries for " + cleanedPath);
return CreateFileEntry(path);
}
var validatedCacheEntry = GetValidatedFileCache(entry);
return validatedCacheEntry;
}
public FileCacheEntity? CreateCacheEntry(string path)
{
_logger.LogTrace("Creating cache entry for " + path);
FileInfo fi = new(path);
if (!fi.Exists) return null;
var fullName = fi.FullName.ToLowerInvariant();
if (!fullName.Contains(_configService.Current.CacheFolder.ToLowerInvariant(), StringComparison.Ordinal)) return null;
string prefixedPath = fullName.Replace(_configService.Current.CacheFolder.ToLowerInvariant(), _cachePrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
return CreateFileCacheEntity(fi, prefixedPath, fi.Name.ToUpper(CultureInfo.InvariantCulture));
}
public FileCacheEntity? CreateFileEntry(string path)
{
_logger.LogTrace("Creating file entry for " + path);
FileInfo fi = new(path);
if (!fi.Exists) return null;
var fullName = fi.FullName.ToLowerInvariant();
if (!fullName.Contains(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), StringComparison.Ordinal)) return null;
string prefixedPath = fullName.Replace(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), _penumbraPrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
return CreateFileCacheEntity(fi, prefixedPath);
}
private FileCacheEntity? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath, string? hash = null)
{
hash ??= Crypto.GetFileHash(fileInfo.FullName);
@@ -161,7 +194,7 @@ public class FileCacheManager : IDisposable
File.AppendAllLines(_csvPath, new[] { entity.CsvEntry });
}
var result = GetFileCacheByPath(fileInfo.FullName);
_logger.LogDebug("Creating file cache for " + fileInfo.FullName + " success: " + (result != null));
_logger.LogDebug("Creating file cache for {name} success: {success}", fileInfo.FullName, (result != null));
return result;
}
@@ -172,6 +205,20 @@ public class FileCacheManager : IDisposable
return resulingFileCache;
}
private FileCacheEntity ReplacePathPrefixes(FileCacheEntity fileCache)
{
if (fileCache.PrefixedFilePath.StartsWith(_penumbraPrefix, StringComparison.OrdinalIgnoreCase))
{
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(_penumbraPrefix, _ipcManager.PenumbraModDirectory, StringComparison.Ordinal));
}
else if (fileCache.PrefixedFilePath.StartsWith(_cachePrefix, StringComparison.OrdinalIgnoreCase))
{
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(_cachePrefix, _configService.Current.CacheFolder, StringComparison.Ordinal));
}
return fileCache;
}
private FileCacheEntity? Validate(FileCacheEntity fileCache)
{
var file = new FileInfo(fileCache.ResolvedFilepath);
@@ -188,44 +235,4 @@ public class FileCacheManager : IDisposable
return fileCache;
}
public void RemoveHash(FileCacheEntity entity)
{
_logger.LogTrace("Removing " + entity.ResolvedFilepath);
_fileCaches.Remove(entity.PrefixedFilePath, out _);
}
public void UpdateHash(FileCacheEntity fileCache)
{
_logger.LogTrace("Updating hash for " + fileCache.ResolvedFilepath);
fileCache.Hash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
fileCache.LastModifiedDateTicks = new FileInfo(fileCache.ResolvedFilepath).LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
_fileCaches.Remove(fileCache.PrefixedFilePath, out _);
_fileCaches[fileCache.PrefixedFilePath] = fileCache;
}
private FileCacheEntity ReplacePathPrefixes(FileCacheEntity fileCache)
{
if (fileCache.PrefixedFilePath.StartsWith(_penumbraPrefix, StringComparison.OrdinalIgnoreCase))
{
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(_penumbraPrefix, _ipcManager.PenumbraModDirectory, StringComparison.Ordinal));
}
else if (fileCache.PrefixedFilePath.StartsWith(_cachePrefix, StringComparison.OrdinalIgnoreCase))
{
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(_cachePrefix, _configService.Current.CacheFolder, StringComparison.Ordinal));
}
return fileCache;
}
public string ResolveFileReplacement(string gamePath)
{
return _ipcManager.PenumbraResolvePath(gamePath);
}
public void Dispose()
{
_logger.LogTrace($"Disposing {GetType()}");
WriteOutFullCsv();
}
}
}

154
MareSynchronos/FileCache/PeriodicFileScanner.cs
View File

@@ -1,89 +1,60 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using MareSynchronos.Managers;
using MareSynchronos.Interop;
using MareSynchronos.MareConfiguration;
using MareSynchronos.Mediator;
using MareSynchronos.Utils;
using MareSynchronos.Services;
using MareSynchronos.Services.Mediator;
using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
namespace MareSynchronos.FileCache;
public class PeriodicFileScanner : MediatorSubscriberBase, IDisposable
public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
{
private readonly IpcManager _ipcManager;
private readonly MareConfigService _configService;
private readonly FileCacheManager _fileDbManager;
private readonly PerformanceCollector _performanceCollector;
private readonly IpcManager _ipcManager;
private readonly PerformanceCollectorService _performanceCollector;
private long _currentFileProgress = 0;
private bool _fileScanWasRunning = false;
private CancellationTokenSource? _scanCancellationTokenSource;
private Task? _fileScannerTask = null;
public ConcurrentDictionary<string, int> haltScanLocks = new(StringComparer.Ordinal);
private TimeSpan _timeUntilNextScan = TimeSpan.Zero;
public PeriodicFileScanner(ILogger<PeriodicFileScanner> logger, IpcManager ipcManager, MareConfigService configService,
FileCacheManager fileDbManager, MareMediator mediator, PerformanceCollector performanceCollector) : base(logger, mediator)
FileCacheManager fileDbManager, MareMediator mediator, PerformanceCollectorService performanceCollector) : base(logger, mediator)
{
_logger.LogTrace("Creating " + nameof(PeriodicFileScanner));
_ipcManager = ipcManager;
_configService = configService;
_fileDbManager = fileDbManager;
_performanceCollector = performanceCollector;
Mediator.Subscribe<PenumbraInitializedMessage>(this, (_) => StartScan());
Mediator.Subscribe<HaltScanMessage>(this, (msg) => HaltScan(((HaltScanMessage)msg).Source));
Mediator.Subscribe<ResumeScanMessage>(this, (msg) => ResumeScan(((ResumeScanMessage)msg).Source));
Mediator.Subscribe<HaltScanMessage>(this, (msg) => HaltScan(msg.Source));
Mediator.Subscribe<ResumeScanMessage>(this, (msg) => ResumeScan(msg.Source));
Mediator.Subscribe<SwitchToMainUiMessage>(this, (_) => StartScan());
Mediator.Subscribe<DalamudLoginMessage>(this, (_) => StartScan());
}
public void ResetLocks()
{
haltScanLocks.Clear();
}
public long CurrentFileProgress => _currentFileProgress;
public long FileCacheSize { get; set; }
public ConcurrentDictionary<string, int> HaltScanLocks { get; set; } = new(StringComparer.Ordinal);
public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
public void ResumeScan(string source)
{
if (!haltScanLocks.ContainsKey(source)) haltScanLocks[source] = 0;
public string TimeUntilNextScan => _timeUntilNextScan.ToString(@"mm\:ss");
haltScanLocks[source]--;
if (haltScanLocks[source] < 0) haltScanLocks[source] = 0;
public long TotalFiles { get; private set; }
if (_fileScanWasRunning && haltScanLocks.All(f => f.Value == 0))
{
_fileScanWasRunning = false;
InvokeScan(forced: true);
}
}
private int TimeBetweenScans => _configService.Current.TimeSpanBetweenScansInSeconds;
public void HaltScan(string source)
{
if (!haltScanLocks.ContainsKey(source)) haltScanLocks[source] = 0;
haltScanLocks[source]++;
if (!HaltScanLocks.ContainsKey(source)) HaltScanLocks[source] = 0;
HaltScanLocks[source]++;
if (IsScanRunning && haltScanLocks.Any(f => f.Value > 0))
if (IsScanRunning && HaltScanLocks.Any(f => f.Value > 0))
{
_scanCancellationTokenSource?.Cancel();
_fileScanWasRunning = true;
}
}
private bool _fileScanWasRunning = false;
private long _currentFileProgress = 0;
public long CurrentFileProgress => _currentFileProgress;
public long FileCacheSize { get; set; }
public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
public long TotalFiles { get; private set; }
public string TimeUntilNextScan => _timeUntilNextScan.ToString(@"mm\:ss");
private TimeSpan _timeUntilNextScan = TimeSpan.Zero;
private int TimeBetweenScans => _configService.Current.TimeSpanBetweenScansInSeconds;
public override void Dispose()
{
base.Dispose();
_scanCancellationTokenSource?.Cancel();
}
public void InvokeScan(bool forced = false)
{
bool isForced = forced;
@@ -92,11 +63,11 @@ public class PeriodicFileScanner : MediatorSubscriberBase, IDisposable
_scanCancellationTokenSource?.Cancel();
_scanCancellationTokenSource = new CancellationTokenSource();
var token = _scanCancellationTokenSource.Token;
_fileScannerTask = Task.Run(async () =>
Task.Run(async () =>
{
while (!token.IsCancellationRequested)
{
while (haltScanLocks.Any(f => f.Value > 0) || !_ipcManager.CheckPenumbraApi())
while (HaltScanLocks.Any(f => f.Value > 0) || !_ipcManager.CheckPenumbraApi())
{
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
@@ -150,6 +121,38 @@ public class PeriodicFileScanner : MediatorSubscriberBase, IDisposable
return true;
}
public void ResetLocks()
{
HaltScanLocks.Clear();
}
public void ResumeScan(string source)
{
if (!HaltScanLocks.ContainsKey(source)) HaltScanLocks[source] = 0;
HaltScanLocks[source]--;
if (HaltScanLocks[source] < 0) HaltScanLocks[source] = 0;
if (_fileScanWasRunning && HaltScanLocks.All(f => f.Value == 0))
{
_fileScanWasRunning = false;
InvokeScan(forced: true);
}
}
public void StartScan()
{
if (!_ipcManager.Initialized || !_configService.Current.HasValidSetup()) return;
Logger.LogTrace("Penumbra is active, configuration is valid, scan");
InvokeScan(forced: true);
}
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
_scanCancellationTokenSource?.Cancel();
}
private void PeriodicFileScan(CancellationToken ct)
{
TotalFiles = 1;
@@ -159,19 +162,19 @@ public class PeriodicFileScanner : MediatorSubscriberBase, IDisposable
if (string.IsNullOrEmpty(penumbraDir) || !Directory.Exists(penumbraDir))
{
penDirExists = false;
_logger.LogWarning("Penumbra directory is not set or does not exist.");
Logger.LogWarning("Penumbra directory is not set or does not exist.");
}
if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder))
{
cacheDirExists = false;
_logger.LogWarning("Mare Cache directory is not set or does not exist.");
Logger.LogWarning("Mare Cache directory is not set or does not exist.");
}
if (!penDirExists || !cacheDirExists)
{
return;
}
_logger.LogDebug("Getting files from " + penumbraDir + " and " + _configService.Current.CacheFolder);
Logger.LogDebug("Getting files from {penumbra} and {storage}", penumbraDir, _configService.Current.CacheFolder);
string[] ext = { ".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp", ".shpk" };
var scannedFiles = new ConcurrentDictionary<string, bool>(Directory.EnumerateFiles(penumbraDir!, "*.*", SearchOption.AllDirectories)
@@ -207,18 +210,18 @@ public class PeriodicFileScanner : MediatorSubscriberBase, IDisposable
scannedFiles[validatedCacheResult.Item2.ResolvedFilepath] = true;
if (validatedCacheResult.Item1 == FileState.RequireUpdate)
{
_logger.LogTrace("To update: {path}", validatedCacheResult.Item2.ResolvedFilepath);
Logger.LogTrace("To update: {path}", validatedCacheResult.Item2.ResolvedFilepath);
entitiesToUpdate.Add(validatedCacheResult.Item2);
}
else if (validatedCacheResult.Item1 == FileState.RequireDeletion)
{
_logger.LogTrace("To delete: {path}", validatedCacheResult.Item2.ResolvedFilepath);
Logger.LogTrace("To delete: {path}", validatedCacheResult.Item2.ResolvedFilepath);
entitiesToRemove.Add(validatedCacheResult.Item2);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed validating {path}", cache.ResolvedFilepath);
Logger.LogWarning(ex, "Failed validating {path}", cache.ResolvedFilepath);
}
Interlocked.Increment(ref _currentFileProgress);
@@ -227,7 +230,7 @@ public class PeriodicFileScanner : MediatorSubscriberBase, IDisposable
if (!_ipcManager.CheckPenumbraApi())
{
_logger.LogWarning("Penumbra not available");
Logger.LogWarning("Penumbra not available");
return;
}
@@ -240,14 +243,14 @@ public class PeriodicFileScanner : MediatorSubscriberBase, IDisposable
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error during enumerating FileCaches");
Logger.LogWarning(ex, "Error during enumerating FileCaches");
}
Task.WaitAll(dbTasks);
if (!_ipcManager.CheckPenumbraApi())
{
_logger.LogWarning("Penumbra not available");
Logger.LogWarning("Penumbra not available");
return;
}
@@ -266,18 +269,18 @@ public class PeriodicFileScanner : MediatorSubscriberBase, IDisposable
_fileDbManager.WriteOutFullCsv();
}
_logger.LogTrace("Scanner validated existing db files");
Logger.LogTrace("Scanner validated existing db files");
if (!_ipcManager.CheckPenumbraApi())
{
_logger.LogWarning("Penumbra not available");
Logger.LogWarning("Penumbra not available");
return;
}
if (ct.IsCancellationRequested) return;
// scan new files
foreach (var c in scannedFiles.Where(c => c.Value == false))
foreach (var c in scannedFiles.Where(c => !c.Value))
{
var idx = Task.WaitAny(dbTasks, ct);
dbTasks[idx] = Task.Run(() =>
@@ -289,7 +292,7 @@ public class PeriodicFileScanner : MediatorSubscriberBase, IDisposable
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed adding {file}", c.Key);
Logger.LogWarning(ex, "Failed adding {file}", c.Key);
}
Interlocked.Increment(ref _currentFileProgress);
@@ -298,7 +301,7 @@ public class PeriodicFileScanner : MediatorSubscriberBase, IDisposable
if (!_ipcManager.CheckPenumbraApi())
{
_logger.LogWarning("Penumbra not available");
Logger.LogWarning("Penumbra not available");
return;
}
@@ -307,9 +310,9 @@ public class PeriodicFileScanner : MediatorSubscriberBase, IDisposable
Task.WaitAll(dbTasks);
_logger.LogTrace("Scanner added new files to db");
Logger.LogTrace("Scanner added new files to db");
_logger.LogDebug("Scan complete");
Logger.LogDebug("Scan complete");
TotalFiles = 0;
_currentFileProgress = 0;
entitiesToRemove.Clear();
@@ -322,11 +325,4 @@ public class PeriodicFileScanner : MediatorSubscriberBase, IDisposable
_configService.Save();
}
}
public void StartScan()
{
if (!_ipcManager.Initialized || !_configService.Current.HasValidSetup()) return;
_logger.LogTrace("Penumbra is active, configuration is valid, scan");
InvokeScan(forced: true);
}
}
}

233
MareSynchronos/FileCache/TransientResourceManager.cs Normal file
View File

@@ -0,0 +1,233 @@
using MareSynchronos.API.Data.Enum;
using MareSynchronos.MareConfiguration;
using MareSynchronos.PlayerData.Data;
using MareSynchronos.PlayerData.Handlers;
using MareSynchronos.Services;
using MareSynchronos.Services.Mediator;
using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
namespace MareSynchronos.FileCache;
public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
{
private readonly TransientConfigService _configurationService;
private readonly DalamudUtilService _dalamudUtil;
private readonly string[] _fileTypesToHandle = new[] { "tmb", "pap", "avfx", "atex", "sklb", "eid", "phyb", "scd", "skp", "shpk" };
private readonly HashSet<GameObjectHandler> _playerRelatedPointers = new();
public TransientResourceManager(ILogger<TransientResourceManager> logger, TransientConfigService configurationService,
DalamudUtilService dalamudUtil, MareMediator mediator) : base(logger, mediator)
{
_configurationService = configurationService;
_dalamudUtil = dalamudUtil;
SemiTransientResources.TryAdd(ObjectKind.Player, new HashSet<string>(StringComparer.Ordinal));
if (_configurationService.Current.PlayerPersistentTransientCache.TryGetValue(PlayerPersistentDataKey, out var gamePaths))
{
int restored = 0;
foreach (var gamePath in gamePaths)
{
if (string.IsNullOrEmpty(gamePath)) continue;
try
{
Logger.LogDebug("Loaded persistent transient resource {path}", gamePath);
SemiTransientResources[ObjectKind.Player].Add(gamePath);
restored++;
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Error during loading persistent transient resource {path}", gamePath);
}
}
Logger.LogDebug("Restored {restored}/{total} semi persistent resources", restored, gamePaths.Count);
}
Mediator.Subscribe<PenumbraResourceLoadMessage>(this, Manager_PenumbraResourceLoadEvent);
Mediator.Subscribe<PenumbraModSettingChangedMessage>(this, (_) => Manager_PenumbraModSettingChanged());
Mediator.Subscribe<FrameworkUpdateMessage>(this, (_) => DalamudUtil_FrameworkUpdate());
Mediator.Subscribe<ClassJobChangedMessage>(this, (_) => DalamudUtil_ClassJobChanged());
Mediator.Subscribe<AddWatchedGameObjectHandler>(this, (msg) =>
{
_playerRelatedPointers.Add(msg.Handler);
});
Mediator.Subscribe<RemoveWatchedGameObjectHandler>(this, (msg) =>
{
_playerRelatedPointers.Remove(msg.Handler);
});
}
private string PlayerPersistentDataKey => _dalamudUtil.PlayerName + "_" + _dalamudUtil.WorldId;
private ConcurrentDictionary<ObjectKind, HashSet<string>> SemiTransientResources { get; } = new();
private ConcurrentDictionary<IntPtr, HashSet<string>> TransientResources { get; } = new();
public void CleanUpSemiTransientResources(ObjectKind objectKind, List<FileReplacement>? fileReplacement = null)
{
if (SemiTransientResources.ContainsKey(objectKind))
{
if (fileReplacement == null)
{
SemiTransientResources[objectKind].Clear();
return;
}
foreach (var replacement in fileReplacement.Where(p => !p.HasFileReplacement).SelectMany(p => p.GamePaths).ToList())
{
SemiTransientResources[objectKind].RemoveWhere(p => string.Equals(p, replacement, StringComparison.OrdinalIgnoreCase));
}
}
}
public HashSet<string> GetSemiTransientResources(ObjectKind objectKind)
{
if (SemiTransientResources.TryGetValue(objectKind, out var result))
{
return result ?? new HashSet<string>(StringComparer.Ordinal);
}
return new HashSet<string>(StringComparer.Ordinal);
}
public List<string> GetTransientResources(IntPtr gameObject)
{
if (TransientResources.TryGetValue(gameObject, out var result))
{
return result.ToList();
}
return new List<string>();
}
public void PersistTransientResources(IntPtr gameObject, ObjectKind objectKind)
{
if (!SemiTransientResources.ContainsKey(objectKind))
{
SemiTransientResources[objectKind] = new HashSet<string>(StringComparer.Ordinal);
}
if (!TransientResources.TryGetValue(gameObject, out var resources))
{
return;
}
var transientResources = resources.ToList();
Logger.LogDebug("Persisting {count} transient resources", transientResources.Count);
foreach (var gamePath in transientResources)
{
SemiTransientResources[objectKind].Add(gamePath);
}
if (objectKind == ObjectKind.Player && SemiTransientResources.TryGetValue(ObjectKind.Player, out var fileReplacements))
{
_configurationService.Current.PlayerPersistentTransientCache[PlayerPersistentDataKey] = fileReplacements.Where(f => !string.IsNullOrEmpty(f)).ToHashSet(StringComparer.Ordinal);
_configurationService.Save();
}
TransientResources[gameObject].Clear();
}
internal void AddSemiTransientResource(ObjectKind objectKind, string item)
{
if (!SemiTransientResources.ContainsKey(objectKind))
{
SemiTransientResources[objectKind] = new HashSet<string>(StringComparer.Ordinal);
}
SemiTransientResources[objectKind].Add(item.ToLowerInvariant());
}
internal void ClearTransientPaths(IntPtr ptr, List<string> list)
{
if (TransientResources.TryGetValue(ptr, out var set))
{
set.RemoveWhere(p => list.Contains(p, StringComparer.OrdinalIgnoreCase));
}
}
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
TransientResources.Clear();
SemiTransientResources.Clear();
if (SemiTransientResources.ContainsKey(ObjectKind.Player))
{
_configurationService.Current.PlayerPersistentTransientCache[PlayerPersistentDataKey] = SemiTransientResources[ObjectKind.Player];
_configurationService.Save();
}
}
private void DalamudUtil_ClassJobChanged()
{
if (SemiTransientResources.TryGetValue(ObjectKind.Pet, out HashSet<string>? value))
{
value?.Clear();
}
}
private void DalamudUtil_FrameworkUpdate()
{
foreach (var item in TransientResources.Where(item => !_dalamudUtil.IsGameObjectPresent(item.Key)).Select(i => i.Key).ToList())
{
Logger.LogDebug("Object not present anymore: {addr}", item.ToString("X"));
TransientResources.TryRemove(item, out _);
}
}
private void Manager_PenumbraModSettingChanged()
{
Task.Run(() =>
{
Logger.LogDebug("Penumbra Mod Settings changed, verifying SemiTransientResources");
foreach (var item in SemiTransientResources)
{
Mediator.Publish(new TransientResourceChangedMessage(_dalamudUtil.PlayerPointer));
}
});
}
private void Manager_PenumbraResourceLoadEvent(PenumbraResourceLoadMessage msg)
{
var gamePath = msg.GamePath.ToLowerInvariant();
var gameObject = msg.GameObject;
var filePath = msg.FilePath;
if (!_fileTypesToHandle.Any(type => gamePath.EndsWith(type, StringComparison.OrdinalIgnoreCase)))
{
return;
}
if (!_playerRelatedPointers.Select(p => p.CurrentAddress).Contains(gameObject))
{
//_logger.LogDebug("Got resource " + gamePath + " for ptr " + gameObject.ToString("X"));
return;
}
if (!TransientResources.ContainsKey(gameObject))
{
TransientResources[gameObject] = new(StringComparer.OrdinalIgnoreCase);
}
if (filePath.StartsWith("|", StringComparison.OrdinalIgnoreCase))
{
filePath = filePath.Split("|")[2];
}
filePath = filePath.ToLowerInvariant().Replace("\\", "/", StringComparison.OrdinalIgnoreCase);
var replacedGamePath = gamePath.ToLowerInvariant().Replace("\\", "/", StringComparison.OrdinalIgnoreCase);
if (string.Equals(filePath, replacedGamePath, StringComparison.OrdinalIgnoreCase)) return;
if (TransientResources[gameObject].Contains(replacedGamePath) ||
SemiTransientResources.Any(r => r.Value.Any(f => string.Equals(f, gamePath, StringComparison.OrdinalIgnoreCase))))
{
Logger.LogTrace("Not adding {replacedPath} : {filePath}", replacedGamePath, filePath);
}
else
{
TransientResources[gameObject].Add(replacedGamePath);
Logger.LogDebug("Adding {replacedGamePath} for {gameObject} ({filePath})", replacedGamePath, gameObject.ToString("X"), filePath);
Mediator.Publish(new TransientResourceChangedMessage(gameObject));
}
}
}