From 00c59fce3ba8fb25769d7a0702684f7835e3579b Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Mon, 9 Sep 2024 21:06:08 +0200 Subject: [PATCH] rework transient files to work on a per job/global basis fix removal of stale paths fix some shit fix some bugs fix pet spaghetti --- .../FileCache/TransientResourceManager.cs | 226 ++++++++++-------- .../Configurations/TransientConfig.cs | 65 ++++- .../PlayerData/Factories/PlayerDataFactory.cs | 19 +- MareSynchronos/Services/DalamudUtilService.cs | 4 +- 4 files changed, 206 insertions(+), 108 deletions(-) diff --git a/MareSynchronos/FileCache/TransientResourceManager.cs b/MareSynchronos/FileCache/TransientResourceManager.cs index e59e26d..b25e13b 100644 --- a/MareSynchronos/FileCache/TransientResourceManager.cs +++ b/MareSynchronos/FileCache/TransientResourceManager.cs @@ -1,5 +1,6 @@ using MareSynchronos.API.Data.Enum; using MareSynchronos.MareConfiguration; +using MareSynchronos.MareConfiguration.Configurations; using MareSynchronos.PlayerData.Data; using MareSynchronos.PlayerData.Handlers; using MareSynchronos.Services; @@ -18,9 +19,11 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase private readonly string[] _fileTypesToHandle = ["tmb", "pap", "avfx", "atex", "sklb", "eid", "phyb", "scd", "skp", "shpk"]; private readonly HashSet _playerRelatedPointers = []; private ConcurrentDictionary _cachedFrameAddresses = []; + private ConcurrentDictionary>? _semiTransientResources = null; + private uint _lastClassJobId = uint.MaxValue; public TransientResourceManager(ILogger logger, TransientConfigService configurationService, - DalamudUtilService dalamudUtil, MareMediator mediator) : base(logger, mediator) + DalamudUtilService dalamudUtil, MareMediator mediator) : base(logger, mediator) { _configurationService = configurationService; _dalamudUtil = dalamudUtil; @@ -28,13 +31,6 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase Mediator.Subscribe(this, Manager_PenumbraResourceLoadEvent); Mediator.Subscribe(this, (_) => Manager_PenumbraModSettingChanged()); Mediator.Subscribe(this, (_) => DalamudUtil_FrameworkUpdate()); - Mediator.Subscribe(this, (msg) => - { - if (_playerRelatedPointers.Contains(msg.GameObjectHandler)) - { - DalamudUtil_ClassJobChanged(); - } - }); Mediator.Subscribe(this, (msg) => { if (!msg.OwnedObject) return; @@ -47,8 +43,20 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase }); } + private TransientConfig.TransientPlayerConfig PlayerConfig + { + get + { + if (!_configurationService.Current.TransientConfigs.TryGetValue(PlayerPersistentDataKey, out var transientConfig)) + { + _configurationService.Current.TransientConfigs[PlayerPersistentDataKey] = transientConfig = new(); + } + + return transientConfig; + } + } + private string PlayerPersistentDataKey => _dalamudUtil.GetPlayerNameAsync().GetAwaiter().GetResult() + "_" + _dalamudUtil.GetHomeWorldIdAsync().GetAwaiter().GetResult(); - private ConcurrentDictionary>? _semiTransientResources = null; private ConcurrentDictionary> SemiTransientResources { get @@ -56,33 +64,16 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase if (_semiTransientResources == null) { _semiTransientResources = new(); - _semiTransientResources.TryAdd(ObjectKind.Player, new HashSet(StringComparer.Ordinal)); - if (_configurationService.Current.PlayerPersistentTransientCache.TryGetValue(PlayerPersistentDataKey, out var gamePaths)) - { - int restored = 0; - foreach (var gamePath in gamePaths) - { - if (string.IsNullOrEmpty(gamePath)) continue; - - try - { - Logger.LogDebug("Loaded persistent transient resource {path}", gamePath); - SemiTransientResources[ObjectKind.Player].Add(gamePath); - restored++; - } - catch (Exception ex) - { - Logger.LogWarning(ex, "Error during loading persistent transient resource {path}", gamePath); - } - } - Logger.LogDebug("Restored {restored}/{total} semi persistent resources", restored, gamePaths.Count); - } + PlayerConfig.JobSpecificCache.TryGetValue(_dalamudUtil.ClassJobId, out var jobSpecificData); + _semiTransientResources[ObjectKind.Player] = PlayerConfig.GlobalPersistentCache.Concat(jobSpecificData ?? []).ToHashSet(StringComparer.Ordinal); + PlayerConfig.JobSpecificPetCache.TryGetValue(_dalamudUtil.ClassJobId, out var petSpecificData); + _semiTransientResources[ObjectKind.Pet] = [.. petSpecificData ?? []]; } return _semiTransientResources; } } - private ConcurrentDictionary> TransientResources { get; } = new(); + private ConcurrentDictionary> TransientResources { get; } = new(); public void CleanUpSemiTransientResources(ObjectKind objectKind, List? fileReplacement = null) { @@ -96,73 +87,98 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase foreach (var replacement in fileReplacement.Where(p => !p.HasFileReplacement).SelectMany(p => p.GamePaths).ToList()) { - value.RemoveWhere(p => string.Equals(p, replacement, StringComparison.OrdinalIgnoreCase)); + PlayerConfig.RemovePath(replacement); } + + _configurationService.Save(); } } public HashSet GetSemiTransientResources(ObjectKind objectKind) { - if (SemiTransientResources.TryGetValue(objectKind, out var result)) - { - return result ?? new HashSet(StringComparer.Ordinal); - } + SemiTransientResources.TryGetValue(objectKind, out var result); - return new HashSet(StringComparer.Ordinal); + return result ?? new HashSet(StringComparer.Ordinal); } - public List GetTransientResources(IntPtr gameObject) + public void PersistTransientResources(ObjectKind objectKind) { - if (TransientResources.TryGetValue(gameObject, out var result)) + if (!SemiTransientResources.TryGetValue(objectKind, out HashSet? semiTransientResources)) { - return [.. result]; + SemiTransientResources[objectKind] = semiTransientResources = new(StringComparer.Ordinal); } - return []; - } - - public void PersistTransientResources(IntPtr gameObject, ObjectKind objectKind) - { - if (!SemiTransientResources.TryGetValue(objectKind, out HashSet? value)) - { - value = new HashSet(StringComparer.Ordinal); - SemiTransientResources[objectKind] = value; - } - - if (!TransientResources.TryGetValue(gameObject, out var resources)) + if (!TransientResources.TryGetValue(objectKind, out var resources)) { return; } var transientResources = resources.ToList(); Logger.LogDebug("Persisting {count} transient resources", transientResources.Count); + List newlyAddedGamePaths = resources.Except(semiTransientResources, StringComparer.Ordinal).ToList(); foreach (var gamePath in transientResources) { - value.Add(gamePath); + semiTransientResources.Add(gamePath); } - if (objectKind == ObjectKind.Player && SemiTransientResources.TryGetValue(ObjectKind.Player, out var fileReplacements)) + if (objectKind == ObjectKind.Player && newlyAddedGamePaths.Any()) { - _configurationService.Current.PlayerPersistentTransientCache[PlayerPersistentDataKey] = fileReplacements.Where(f => !string.IsNullOrEmpty(f)).ToHashSet(StringComparer.Ordinal); + foreach (var item in newlyAddedGamePaths.Where(f => !string.IsNullOrEmpty(f))) + { + PlayerConfig.AddOrElevate(_dalamudUtil.ClassJobId, item); + } + _configurationService.Save(); } - TransientResources[gameObject].Clear(); + else if (objectKind == ObjectKind.Pet && newlyAddedGamePaths.Any()) + { + foreach (var item in newlyAddedGamePaths.Where(f => !string.IsNullOrEmpty(f))) + { + if (!PlayerConfig.JobSpecificPetCache.TryGetValue(_dalamudUtil.ClassJobId, out var petPerma)) + { + PlayerConfig.JobSpecificPetCache[_dalamudUtil.ClassJobId] = petPerma = []; + } + + petPerma.Add(item); + } + + _configurationService.Save(); + } + + TransientResources[objectKind].Clear(); } - internal void AddSemiTransientResource(ObjectKind objectKind, string item) + public void RemoveTransientResource(ObjectKind objectKind, string path) { - if (!SemiTransientResources.TryGetValue(objectKind, out HashSet? value)) + if (SemiTransientResources.TryGetValue(objectKind, out var resources)) + { + resources.RemoveWhere(f => string.Equals(path, f, StringComparison.Ordinal)); + if (objectKind == ObjectKind.Player) + { + PlayerConfig.RemovePath(path); + _configurationService.Save(); + } + } + } + + internal bool AddTransientResource(ObjectKind objectKind, string item) + { + if (SemiTransientResources.TryGetValue(objectKind, out var semiTransient) && semiTransient != null && semiTransient.Contains(item)) + return false; + + if (!TransientResources.TryGetValue(objectKind, out HashSet? value)) { value = new HashSet(StringComparer.Ordinal); - SemiTransientResources[objectKind] = value; + TransientResources[objectKind] = value; } value.Add(item.ToLowerInvariant()); + return true; } - internal void ClearTransientPaths(IntPtr ptr, List list) + internal void ClearTransientPaths(ObjectKind objectKind, List list) { - if (TransientResources.TryGetValue(ptr, out var set)) + if (TransientResources.TryGetValue(objectKind, out var set)) { foreach (var file in set.Where(p => list.Contains(p, StringComparer.OrdinalIgnoreCase))) { @@ -172,44 +188,66 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase int removed = set.RemoveWhere(p => list.Contains(p, StringComparer.OrdinalIgnoreCase)); Logger.LogInformation("Removed {removed} previously existing transient paths", removed); } + + bool reloadSemiTransient = false; + if (objectKind == ObjectKind.Player && SemiTransientResources.TryGetValue(objectKind, out var semiset)) + { + foreach (var file in semiset.Where(p => list.Contains(p, StringComparer.OrdinalIgnoreCase))) + { + Logger.LogTrace("Removing From Transient: {file}", file); + PlayerConfig.RemovePath(file); + } + + int removed = semiset.RemoveWhere(p => list.Contains(p, StringComparer.OrdinalIgnoreCase)); + Logger.LogInformation("Removed {removed} previously existing semi transient paths", removed); + if (removed > 0) + { + reloadSemiTransient = true; + _configurationService.Save(); + } + } + + if (reloadSemiTransient) + _semiTransientResources = null; } protected override void Dispose(bool disposing) { base.Dispose(disposing); - try - { - TransientResources.Clear(); - SemiTransientResources.Clear(); - if (SemiTransientResources.TryGetValue(ObjectKind.Player, out HashSet? value)) - { - _configurationService.Current.PlayerPersistentTransientCache[PlayerPersistentDataKey] = value; - _configurationService.Save(); - } - } - catch { } - } - - private void DalamudUtil_ClassJobChanged() - { - if (SemiTransientResources.TryGetValue(ObjectKind.Pet, out HashSet? value)) - { - value?.Clear(); - } + TransientResources.Clear(); + SemiTransientResources.Clear(); } private void DalamudUtil_FrameworkUpdate() { - _cachedFrameAddresses = _cachedFrameAddresses = new ConcurrentDictionary(_playerRelatedPointers.Where(k => k.Address != nint.Zero).ToDictionary(c => c.CurrentAddress(), c => c.ObjectKind)); + _cachedFrameAddresses = new(_playerRelatedPointers.Where(k => k.Address != nint.Zero).ToDictionary(c => c.CurrentAddress(), c => c.ObjectKind)); lock (_cacheAdditionLock) { _cachedHandledPaths.Clear(); } - foreach (var item in TransientResources.Where(item => !_dalamudUtil.IsGameObjectPresent(item.Key)).Select(i => i.Key).ToList()) + + if (_lastClassJobId != _dalamudUtil.ClassJobId) { - Logger.LogDebug("Object not present anymore: {addr}", item.ToString("X")); - TransientResources.TryRemove(item, out _); + _lastClassJobId = _dalamudUtil.ClassJobId; + if (SemiTransientResources.TryGetValue(ObjectKind.Pet, out HashSet? value)) + { + value?.Clear(); + } + + // reload config for current new classjob + PlayerConfig.JobSpecificCache.TryGetValue(_dalamudUtil.ClassJobId, out var jobSpecificData); + SemiTransientResources[ObjectKind.Player] = PlayerConfig.GlobalPersistentCache.Concat(jobSpecificData ?? []).ToHashSet(StringComparer.OrdinalIgnoreCase); + PlayerConfig.JobSpecificPetCache.TryGetValue(_dalamudUtil.ClassJobId, out var petSpecificData); + SemiTransientResources[ObjectKind.Pet] = [.. petSpecificData ?? []]; + } + + foreach (var kind in Enum.GetValues(typeof(ObjectKind))) + { + if (!_cachedFrameAddresses.Any(k => k.Value == (ObjectKind)kind) && TransientResources.Remove((ObjectKind)kind, out _)) + { + Logger.LogDebug("Object not present anymore: {kind}", kind.ToString()); + } } } @@ -271,14 +309,16 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase return; } - if (!TransientResources.TryGetValue(gameObject, out HashSet? value)) + // ^ all of the code above is just to sanitize the data + + if (!TransientResources.TryGetValue(objectKind, out HashSet? value)) { value = new(StringComparer.OrdinalIgnoreCase); - TransientResources[gameObject] = value; + TransientResources[objectKind] = value; } - if (value.Contains(replacedGamePath) || - SemiTransientResources.SelectMany(k => k.Value).Any(f => string.Equals(f, gamePath, StringComparison.OrdinalIgnoreCase))) + if (value.Contains(replacedGamePath) + || SemiTransientResources.SelectMany(k => k.Value).Any(f => string.Equals(f, gamePath, StringComparison.OrdinalIgnoreCase))) { Logger.LogTrace("Not adding {replacedPath} : {filePath}", replacedGamePath, filePath); } @@ -299,15 +339,5 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase } } - internal void RemoveTransientResource(ObjectKind objectKind, string path) - { - if (SemiTransientResources.TryGetValue(objectKind, out var resources)) - { - resources.RemoveWhere(f => string.Equals(path, f, StringComparison.OrdinalIgnoreCase)); - _configurationService.Current.PlayerPersistentTransientCache[PlayerPersistentDataKey] = resources; - _configurationService.Save(); - } - } - private CancellationTokenSource _sendTransientCts = new(); } \ No newline at end of file diff --git a/MareSynchronos/MareConfiguration/Configurations/TransientConfig.cs b/MareSynchronos/MareConfiguration/Configurations/TransientConfig.cs index 668dc2b..32a4f7a 100644 --- a/MareSynchronos/MareConfiguration/Configurations/TransientConfig.cs +++ b/MareSynchronos/MareConfiguration/Configurations/TransientConfig.cs @@ -2,6 +2,69 @@ public class TransientConfig : IMareConfiguration { - public Dictionary> PlayerPersistentTransientCache { get; set; } = new(StringComparer.Ordinal); + public Dictionary TransientConfigs { get; set; } = []; public int Version { get; set; } = 0; + + public class TransientPlayerConfig + { + public List GlobalPersistentCache { get; set; } = []; + public Dictionary> JobSpecificCache { get; set; } = []; + public Dictionary> JobSpecificPetCache { get; set; } = []; + + public TransientPlayerConfig() + { + + } + + private bool ElevateIfNeeded(uint jobId, string gamePath) + { + // check if it's in the job cache of other jobs and elevate if needed + foreach (var kvp in JobSpecificCache) + { + if (kvp.Key == jobId) continue; + + // elevate if the gamepath is included somewhere else + if (kvp.Value.Contains(gamePath, StringComparer.Ordinal)) + { + JobSpecificCache[kvp.Key].Remove(gamePath); + GlobalPersistentCache.Add(gamePath); + return true; + } + } + + return false; + } + + public void RemovePath(string gamePath) + { + GlobalPersistentCache.Remove(gamePath); + foreach (var kvp in JobSpecificCache) + { + kvp.Value.Remove(gamePath); + } + } + + public void AddOrElevate(uint jobId, string gamePath) + { + // check if it's in the global cache, if yes, do nothing + if (GlobalPersistentCache.Contains(gamePath, StringComparer.Ordinal)) + { + return; + } + + if (ElevateIfNeeded(jobId, gamePath)) return; + + // check if the jobid is already in the cache to start + if (!JobSpecificCache.TryGetValue(jobId, out var jobCache)) + { + JobSpecificCache[jobId] = jobCache = new(); + } + + // check if the path is already in the job specific cache + if (!jobCache.Contains(gamePath, StringComparer.Ordinal)) + { + jobCache.Add(gamePath); + } + } + } } diff --git a/MareSynchronos/PlayerData/Factories/PlayerDataFactory.cs b/MareSynchronos/PlayerData/Factories/PlayerDataFactory.cs index 259779c..24a5371 100644 --- a/MareSynchronos/PlayerData/Factories/PlayerDataFactory.cs +++ b/MareSynchronos/PlayerData/Factories/PlayerDataFactory.cs @@ -166,20 +166,25 @@ public class PlayerDataFactory // or we get into redraw city for every change and nothing works properly if (objectKind == ObjectKind.Pet) { - foreach (var item in previousData.FileReplacements[objectKind].Where(i => i.HasFileReplacement).SelectMany(p => p.GamePaths)) + foreach (var item in previousData.FileReplacements[ObjectKind.Pet].Where(i => i.HasFileReplacement).SelectMany(p => p.GamePaths)) { - _logger.LogDebug("Persisting {item}", item); - _transientResourceManager.AddSemiTransientResource(objectKind, item); + if (_transientResourceManager.AddTransientResource(objectKind, item)) + { + _logger.LogDebug("Marking static {item} for Pet as transient", item); + } } + + _logger.LogTrace("Clearing {count} Static Replacements for Pet", previousData.FileReplacements[ObjectKind.Pet].Count); + previousData.FileReplacements[ObjectKind.Pet].Clear(); } _logger.LogDebug("Handling transient update for {obj}", playerRelatedObject); // remove all potentially gathered paths from the transient resource manager that are resolved through static resolving - _transientResourceManager.ClearTransientPaths(playerRelatedObject.Address, previousData.FileReplacements[objectKind].SelectMany(c => c.GamePaths).ToList()); + _transientResourceManager.ClearTransientPaths(objectKind, previousData.FileReplacements[objectKind].SelectMany(c => c.GamePaths).ToList()); // get all remaining paths and resolve them - var transientPaths = ManageSemiTransientData(objectKind, playerRelatedObject.Address); + var transientPaths = ManageSemiTransientData(objectKind); var resolvedTransientPaths = await GetFileReplacementsFromPaths(transientPaths, new HashSet(StringComparer.Ordinal)).ConfigureAwait(false); _logger.LogDebug("== Transient Replacements =="); @@ -349,9 +354,9 @@ public class PlayerDataFactory return resolvedPaths.ToDictionary(k => k.Key, k => k.Value.ToArray(), StringComparer.OrdinalIgnoreCase).AsReadOnly(); } - private HashSet ManageSemiTransientData(ObjectKind objectKind, IntPtr charaPointer) + private HashSet ManageSemiTransientData(ObjectKind objectKind) { - _transientResourceManager.PersistTransientResources(charaPointer, objectKind); + _transientResourceManager.PersistTransientResources(objectKind); HashSet pathsToResolve = new(StringComparer.Ordinal); foreach (var path in _transientResourceManager.GetSemiTransientResources(objectKind).Where(path => !string.IsNullOrEmpty(path))) diff --git a/MareSynchronos/Services/DalamudUtilService.cs b/MareSynchronos/Services/DalamudUtilService.cs index ce9acf0..7d60a15 100644 --- a/MareSynchronos/Services/DalamudUtilService.cs +++ b/MareSynchronos/Services/DalamudUtilService.cs @@ -171,7 +171,7 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber public bool IsZoning => _condition[ConditionFlag.BetweenAreas] || _condition[ConditionFlag.BetweenAreas51]; public bool IsInCombatOrPerforming { get; private set; } = false; public bool HasModifiedGameFiles => _gameData.HasModifiedGameDataFiles; - + public uint ClassJobId => _classJobId!.Value; public Lazy> WorldData { get; private set; } public Lazy> UiColors { get; private set; } public Lazy> TerritoryData { get; private set; } @@ -330,7 +330,7 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber public uint GetHomeWorldId() { EnsureIsOnFramework(); - return _clientState.LocalPlayer!.HomeWorld.RowId; + return _clientState.LocalPlayer?.HomeWorld.RowId ?? 0; } public uint GetWorldId()