From 99eecbdc097c6ec68faf83784b08ed9652d82f36 Mon Sep 17 00:00:00 2001 From: Loporrit <141286461+loporrit@users.noreply.github.com> Date: Sun, 23 Feb 2025 12:18:34 +0000 Subject: [PATCH] Add texture shrinking feature --- MareSynchronos/FileCache/CacheMonitor.cs | 161 ++++++++++++++++- MareSynchronos/FileCache/FileCacheEntity.cs | 1 + MareSynchronos/FileCache/FileCacheManager.cs | 57 +++++- .../Configurations/PlayerPerformanceConfig.cs | 5 +- .../Configurations/XivDataStorageConfig.cs | 1 + .../Models/TextureShrinkMode.cs | 10 ++ .../PlayerData/Handlers/PairHandler.cs | 14 +- MareSynchronos/Services/PairAnalyzer.cs | 2 +- .../Services/PlayerPerformanceService.cs | 162 ++++++++++++++++-- MareSynchronos/Services/XivDataAnalyzer.cs | 48 +++++- MareSynchronos/UI/SettingsUi.cs | 29 ++++ 11 files changed, 458 insertions(+), 32 deletions(-) create mode 100644 MareSynchronos/MareConfiguration/Models/TextureShrinkMode.cs diff --git a/MareSynchronos/FileCache/CacheMonitor.cs b/MareSynchronos/FileCache/CacheMonitor.cs index f8cb4e3..9984f14 100644 --- a/MareSynchronos/FileCache/CacheMonitor.cs +++ b/MareSynchronos/FileCache/CacheMonitor.cs @@ -36,6 +36,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase { StartPenumbraWatcher(_ipcManager.Penumbra.ModDirectory); StartMareWatcher(configService.Current.CacheFolder); + StartSubstWatcher(_fileDbManager.SubstFolder); InvokeScan(); }); Mediator.Subscribe(this, (msg) => HaltScan(msg.Source)); @@ -43,6 +44,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase Mediator.Subscribe(this, (_) => { StartMareWatcher(configService.Current.CacheFolder); + StartSubstWatcher(_fileDbManager.SubstFolder); StartPenumbraWatcher(_ipcManager.Penumbra.ModDirectory); InvokeScan(); }); @@ -58,6 +60,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase if (configService.Current.HasValidSetup()) { StartMareWatcher(configService.Current.CacheFolder); + StartSubstWatcher(_fileDbManager.SubstFolder); InvokeScan(); } @@ -90,6 +93,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase public long FileCacheSize { get; set; } public long FileCacheDriveFree { get; set; } public ConcurrentDictionary HaltScanLocks { get; set; } = new(StringComparer.Ordinal); + private int LockCount = 0; public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0; public long TotalFiles { get; private set; } public long TotalFilesStorage { get; private set; } @@ -97,18 +101,22 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase public void HaltScan(string source) { HaltScanLocks.AddOrUpdate(source, 1, (k, v) => v + 1); + Interlocked.Increment(ref LockCount); } record WatcherChange(WatcherChangeTypes ChangeType, string? OldPath = null); private readonly Dictionary _watcherChanges = new Dictionary(StringComparer.OrdinalIgnoreCase); private readonly Dictionary _mareChanges = new Dictionary(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _substChanges = new Dictionary(StringComparer.OrdinalIgnoreCase); public void StopMonitoring() { Logger.LogInformation("Stopping monitoring of Penumbra and Mare storage folders"); MareWatcher?.Dispose(); + SubstWatcher?.Dispose(); PenumbraWatcher?.Dispose(); MareWatcher = null; + SubstWatcher = null; PenumbraWatcher = null; } @@ -147,13 +155,53 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase MareWatcher.EnableRaisingEvents = true; } + public void StartSubstWatcher(string? substPath) + { + SubstWatcher?.Dispose(); + if (string.IsNullOrEmpty(substPath)) + { + SubstWatcher = null; + Logger.LogWarning("Mare file path is not set, cannot start the FSW for Mare."); + return; + } + + try + { + if (!Directory.Exists(substPath)) + Directory.CreateDirectory(substPath); + } + catch + { + Logger.LogWarning("Could not create subst directory at {path}.", substPath); + return; + } + + Logger.LogDebug("Initializing Subst FSW on {path}", substPath); + SubstWatcher = new() + { + Path = substPath, + InternalBufferSize = 8388608, + NotifyFilter = NotifyFilters.CreationTime + | NotifyFilters.LastWrite + | NotifyFilters.FileName + | NotifyFilters.DirectoryName + | NotifyFilters.Size, + Filter = "*.*", + IncludeSubdirectories = false, + }; + + SubstWatcher.Deleted += SubstWatcher_FileChanged; + SubstWatcher.Created += SubstWatcher_FileChanged; + SubstWatcher.EnableRaisingEvents = true; + } + private void MareWatcher_FileChanged(object sender, FileSystemEventArgs e) { Logger.LogTrace("Mare FSW: FileChanged: {change} => {path}", e.ChangeType, e.FullPath); if (!AllowedFileExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return; - lock (_watcherChanges) + lock (_mareChanges) { _mareChanges[e.FullPath] = new(e.ChangeType); } @@ -161,6 +209,20 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase _ = MareWatcherExecution(); } + private void SubstWatcher_FileChanged(object sender, FileSystemEventArgs e) + { + Logger.LogTrace("Subst FSW: FileChanged: {change} => {path}", e.ChangeType, e.FullPath); + + if (!AllowedFileExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return; + + lock (_substChanges) + { + _substChanges[e.FullPath] = new(e.ChangeType); + } + + _ = SubstWatcherExecution(); + } + public void StartPenumbraWatcher(string? penumbraPath) { PenumbraWatcher?.Dispose(); @@ -247,8 +309,10 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase private CancellationTokenSource _penumbraFswCts = new(); private CancellationTokenSource _mareFswCts = new(); + private CancellationTokenSource _substFswCts = new(); public FileSystemWatcher? PenumbraWatcher { get; private set; } public FileSystemWatcher? MareWatcher { get; private set; } + public FileSystemWatcher? SubstWatcher { get; private set; } private async Task MareWatcherExecution() { @@ -281,6 +345,67 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase HandleChanges(changes); } + private async Task SubstWatcherExecution() + { + _substFswCts = _substFswCts.CancelRecreate(); + var token = _substFswCts.Token; + var delay = TimeSpan.FromSeconds(5); + Dictionary changes; + lock (_substChanges) + changes = _substChanges.ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal); + try + { + do + { + await Task.Delay(delay, token).ConfigureAwait(false); + } while (HaltScanLocks.Any(f => f.Value > 0)); + } + catch (TaskCanceledException) + { + return; + } + + lock (_substChanges) + { + foreach (var key in changes.Keys) + { + _substChanges.Remove(key); + } + } + + HandleChanges(changes); + } + + public void ClearSubstStorage() + { + var substDir = _fileDbManager.SubstFolder; + var allSubstFiles = Directory.GetFiles(substDir, "*.*", SearchOption.TopDirectoryOnly) + .Where(f => + { + var val = f.Split('\\')[^1]; + return val.Length == 40 || (val.Split('.').FirstOrDefault()?.Length ?? 0) == 40 + || val.EndsWith(".tmp", StringComparison.OrdinalIgnoreCase); + }); + if (SubstWatcher != null) + SubstWatcher.EnableRaisingEvents = false; + + Dictionary changes = _substChanges.ToDictionary(t => t.Key, t => new WatcherChange(WatcherChangeTypes.Deleted, t.Key), StringComparer.Ordinal); + + foreach (var file in allSubstFiles) + { + try + { + File.Delete(file); + } + catch { } + } + + HandleChanges(changes); + + if (SubstWatcher != null) + SubstWatcher.EnableRaisingEvents = true; + } + private void HandleChanges(Dictionary changes) { lock (_fileDbManager) @@ -408,7 +533,9 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder); } - var files = Directory.EnumerateFiles(_configService.Current.CacheFolder).Select(f => new FileInfo(f)) + var files = Directory.EnumerateFiles(_configService.Current.CacheFolder) + .Concat(Directory.EnumerateFiles(_fileDbManager.SubstFolder)) + .Select(f => new FileInfo(f)) .OrderBy(f => f.LastAccessTime).ToList(); FileCacheSize = files .Sum(f => @@ -429,6 +556,8 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase if (FileCacheSize < maxCacheInBytes) return; + var substDir = _fileDbManager.SubstFolder; + var maxCacheBuffer = maxCacheInBytes * 0.05d; while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer) { @@ -442,11 +571,16 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase public void ResetLocks() { HaltScanLocks.Clear(); + LockCount = 0; } public void ResumeScan(string source) { HaltScanLocks.AddOrUpdate(source, 0, (k, v) => Math.Max(0, v - 1)); + int lockCount = Interlocked.Decrement(ref LockCount); + + if (lockCount == 0) + HaltScanLocks.Clear(); } protected override void Dispose(bool disposing) @@ -455,8 +589,10 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase _scanCancellationTokenSource?.Cancel(); PenumbraWatcher?.Dispose(); MareWatcher?.Dispose(); + SubstWatcher?.Dispose(); _penumbraFswCts?.CancelDispose(); _mareFswCts?.CancelDispose(); + _substFswCts?.CancelDispose(); _periodicCalculationTokenSource?.CancelDispose(); } @@ -466,6 +602,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase var penumbraDir = _ipcManager.Penumbra.ModDirectory; bool penDirExists = true; bool cacheDirExists = true; + var substDir = _fileDbManager.SubstFolder; if (string.IsNullOrEmpty(penumbraDir) || !Directory.Exists(penumbraDir)) { penDirExists = false; @@ -481,6 +618,16 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase return; } + try + { + if (!Directory.Exists(substDir)) + Directory.CreateDirectory(substDir); + } + catch + { + Logger.LogWarning("Could not create subst directory at {path}.", substDir); + } + var previousThreadPriority = Thread.CurrentThread.Priority; Thread.CurrentThread.Priority = ThreadPriority.Lowest; Logger.LogDebug("Getting files from {penumbra} and {storage}", penumbraDir, _configService.Current.CacheFolder); @@ -509,6 +656,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase } var allCacheFiles = Directory.GetFiles(_configService.Current.CacheFolder, "*.*", SearchOption.TopDirectoryOnly) + .Concat(Directory.GetFiles(substDir, "*.*", SearchOption.TopDirectoryOnly)) .AsParallel() .Where(f => { @@ -654,7 +802,13 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase try { var entry = _fileDbManager.CreateFileEntry(cachePath); - if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath); + if (entry == null) + { + if (cachePath.StartsWith(substDir, StringComparison.Ordinal)) + _ = _fileDbManager.CreateSubstEntry(cachePath); + else + _ = _fileDbManager.CreateCacheEntry(cachePath); + } } catch (Exception ex) { @@ -678,6 +832,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase _configService.Current.InitialScanComplete = true; _configService.Save(); StartMareWatcher(_configService.Current.CacheFolder); + StartSubstWatcher(_fileDbManager.SubstFolder); StartPenumbraWatcher(penumbraDir); } } diff --git a/MareSynchronos/FileCache/FileCacheEntity.cs b/MareSynchronos/FileCache/FileCacheEntity.cs index 2c07b87..e81353a 100644 --- a/MareSynchronos/FileCache/FileCacheEntity.cs +++ b/MareSynchronos/FileCache/FileCacheEntity.cs @@ -17,6 +17,7 @@ public class FileCacheEntity public string CsvEntry => $"{Hash}{FileCacheManager.CsvSplit}{PrefixedFilePath}{FileCacheManager.CsvSplit}{LastModifiedDateTicks}|{Size ?? -1}|{CompressedSize ?? -1}"; public string Hash { get; set; } public bool IsCacheEntry => PrefixedFilePath.StartsWith(FileCacheManager.CachePrefix, StringComparison.OrdinalIgnoreCase); + public bool IsSubstEntry => PrefixedFilePath.StartsWith(FileCacheManager.SubstPrefix, StringComparison.OrdinalIgnoreCase); public string LastModifiedDateTicks { get; set; } public string PrefixedFilePath { get; init; } public string ResolvedFilepath { get; private set; } = string.Empty; diff --git a/MareSynchronos/FileCache/FileCacheManager.cs b/MareSynchronos/FileCache/FileCacheManager.cs index e87dd62..4243633 100644 --- a/MareSynchronos/FileCache/FileCacheManager.cs +++ b/MareSynchronos/FileCache/FileCacheManager.cs @@ -1,4 +1,5 @@ -using K4os.Compression.LZ4.Streams; +using Dalamud.Utility; +using K4os.Compression.LZ4.Streams; using MareSynchronos.Interop.Ipc; using MareSynchronos.MareConfiguration; using MareSynchronos.Services.Mediator; @@ -16,6 +17,10 @@ public sealed class FileCacheManager : IHostedService public const string CachePrefix = "{cache}"; public const string CsvSplit = "|"; public const string PenumbraPrefix = "{penumbra}"; + public const string SubstPrefix = "{subst}"; + public const string SubstPath = "subst"; + public string CacheFolder => _configService.Current.CacheFolder; + public string SubstFolder => CacheFolder.IsNullOrEmpty() ? string.Empty : CacheFolder.ToLowerInvariant().TrimEnd('\\') + "\\" + SubstPath; private readonly MareConfigService _configService; private readonly MareMediator _mareMediator; private readonly string _csvPath; @@ -47,6 +52,19 @@ public sealed class FileCacheManager : IHostedService return CreateFileCacheEntity(fi, prefixedPath); } + public FileCacheEntity? CreateSubstEntry(string path) + { + FileInfo fi = new(path); + if (!fi.Exists) return null; + _logger.LogTrace("Creating substitute entry for {path}", path); + var fullName = fi.FullName.ToLowerInvariant(); + if (!fullName.Contains(SubstFolder, StringComparison.Ordinal)) return null; + string prefixedPath = fullName.Replace(SubstFolder, SubstPrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal); + var fakeHash = Path.GetFileNameWithoutExtension(fi.FullName).ToUpperInvariant(); + var result = CreateFileCacheEntity(fi, prefixedPath, fakeHash); + return result; + } + public FileCacheEntity? CreateFileEntry(string path) { FileInfo fi = new(path); @@ -65,7 +83,7 @@ public sealed class FileCacheManager : IHostedService List output = []; if (_fileCaches.TryGetValue(hash, out var fileCacheEntities)) { - foreach (var fileCache in fileCacheEntities.Where(c => ignoreCacheEntries ? !c.IsCacheEntry : true).ToList()) + foreach (var fileCache in fileCacheEntities.Where(c => ignoreCacheEntries ? (!c.IsCacheEntry && !c.IsSubstEntry) : true).ToList()) { if (!validate) output.Add(fileCache); else @@ -89,6 +107,7 @@ public sealed class FileCacheManager : IHostedService foreach (var fileCache in cacheEntries) { if (cancellationToken.IsCancellationRequested) break; + if (fileCache.IsSubstEntry) continue; _logger.LogInformation("Validating {file}", fileCache.ResolvedFilepath); @@ -139,6 +158,11 @@ public sealed class FileCacheManager : IHostedService return Path.Combine(_configService.Current.CacheFolder, hash + "." + extension); } + public string GetSubstFilePath(string hash, string extension) + { + return Path.Combine(SubstFolder, hash + "." + extension); + } + public async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken) { var fileCache = GetFileCacheByHash(fileHash)!; @@ -151,14 +175,24 @@ public sealed class FileCacheManager : IHostedService return (fileHash, ms.ToArray()); } - public FileCacheEntity? GetFileCacheByHash(string hash) + public FileCacheEntity? GetFileCacheByHash(string hash, bool preferSubst = false) { + var caches = GetFileCachesByHash(hash); + if (preferSubst && caches.Subst != null) + return caches.Subst; + return caches.Penumbra ?? caches.Cache; + } + + public (FileCacheEntity? Penumbra, FileCacheEntity? Cache, FileCacheEntity? Subst) GetFileCachesByHash(string hash) + { + (FileCacheEntity? Penumbra, FileCacheEntity? Cache, FileCacheEntity? Subst) result = (null, null, null); if (_fileCaches.TryGetValue(hash, out var hashes)) { - var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix) ? 0 : 1).FirstOrDefault(); - if (item != null) return GetValidatedFileCache(item); + result.Penumbra = hashes.Where(p => p.PrefixedFilePath.StartsWith(PenumbraPrefix, StringComparison.Ordinal)).Select(GetValidatedFileCache).FirstOrDefault(); + result.Cache = hashes.Where(p => p.PrefixedFilePath.StartsWith(CachePrefix, StringComparison.Ordinal)).Select(GetValidatedFileCache).FirstOrDefault(); + result.Subst = hashes.Where(p => p.PrefixedFilePath.StartsWith(SubstPrefix, StringComparison.Ordinal)).Select(GetValidatedFileCache).FirstOrDefault(); } - return null; + return result; } private FileCacheEntity? GetFileCacheByPath(string path) @@ -187,6 +221,7 @@ public sealed class FileCacheManager : IHostedService var cleanedPaths = paths.Distinct(StringComparer.OrdinalIgnoreCase).ToDictionary(p => p, p => p.Replace("/", "\\", StringComparison.OrdinalIgnoreCase) .Replace(_ipcManager.Penumbra.ModDirectory!, _ipcManager.Penumbra.ModDirectory!.EndsWith('\\') ? PenumbraPrefix + '\\' : PenumbraPrefix, StringComparison.OrdinalIgnoreCase) + .Replace(SubstFolder, SubstPrefix, StringComparison.OrdinalIgnoreCase) .Replace(_configService.Current.CacheFolder, _configService.Current.CacheFolder.EndsWith('\\') ? CachePrefix + '\\' : CachePrefix, StringComparison.OrdinalIgnoreCase) .Replace("\\\\", "\\", StringComparison.Ordinal), StringComparer.OrdinalIgnoreCase); @@ -207,9 +242,11 @@ public sealed class FileCacheManager : IHostedService } else { - if (!entry.Value.Contains(CachePrefix, StringComparison.Ordinal)) + if (entry.Value.StartsWith(PenumbraPrefix, StringComparison.Ordinal)) result.Add(entry.Key, CreateFileEntry(entry.Key)); - else + else if (entry.Value.StartsWith(SubstPrefix, StringComparison.Ordinal)) + result.Add(entry.Key, CreateSubstEntry(entry.Key)); + else if (entry.Value.StartsWith(CachePrefix, StringComparison.Ordinal)) result.Add(entry.Key, CreateCacheEntry(entry.Key)); } } @@ -360,6 +397,10 @@ public sealed class FileCacheManager : IHostedService { fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(PenumbraPrefix, _ipcManager.Penumbra.ModDirectory, StringComparison.Ordinal)); } + else if (fileCache.PrefixedFilePath.StartsWith(SubstPrefix, StringComparison.OrdinalIgnoreCase)) + { + fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(SubstPrefix, SubstFolder, StringComparison.Ordinal)); + } else if (fileCache.PrefixedFilePath.StartsWith(CachePrefix, StringComparison.OrdinalIgnoreCase)) { fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(CachePrefix, _configService.Current.CacheFolder, StringComparison.Ordinal)); diff --git a/MareSynchronos/MareConfiguration/Configurations/PlayerPerformanceConfig.cs b/MareSynchronos/MareConfiguration/Configurations/PlayerPerformanceConfig.cs index 2a8edb5..0c2a64e 100644 --- a/MareSynchronos/MareConfiguration/Configurations/PlayerPerformanceConfig.cs +++ b/MareSynchronos/MareConfiguration/Configurations/PlayerPerformanceConfig.cs @@ -1,4 +1,6 @@ -namespace MareSynchronos.MareConfiguration.Configurations; +using MareSynchronos.MareConfiguration.Models; + +namespace MareSynchronos.MareConfiguration.Configurations; public class PlayerPerformanceConfig : IMareConfiguration { @@ -9,4 +11,5 @@ public class PlayerPerformanceConfig : IMareConfiguration public int VRAMSizeAutoPauseThresholdMiB { get; set; } = 550; public int TrisAutoPauseThresholdThousands { get; set; } = 375; public bool IgnoreDirectPairs { get; set; } = true; + public TextureShrinkMode TextureShrinkMode { get; set; } = TextureShrinkMode.Default; } \ No newline at end of file diff --git a/MareSynchronos/MareConfiguration/Configurations/XivDataStorageConfig.cs b/MareSynchronos/MareConfiguration/Configurations/XivDataStorageConfig.cs index 4d56a9d..7237e82 100644 --- a/MareSynchronos/MareConfiguration/Configurations/XivDataStorageConfig.cs +++ b/MareSynchronos/MareConfiguration/Configurations/XivDataStorageConfig.cs @@ -5,6 +5,7 @@ namespace MareSynchronos.MareConfiguration.Configurations; public class XivDataStorageConfig : IMareConfiguration { public ConcurrentDictionary TriangleDictionary { get; set; } = new(StringComparer.OrdinalIgnoreCase); + public ConcurrentDictionary TexDictionary { get; set; } = new(StringComparer.OrdinalIgnoreCase); public ConcurrentDictionary>> BonesDictionary { get; set; } = new(StringComparer.OrdinalIgnoreCase); public int Version { get; set; } = 0; } \ No newline at end of file diff --git a/MareSynchronos/MareConfiguration/Models/TextureShrinkMode.cs b/MareSynchronos/MareConfiguration/Models/TextureShrinkMode.cs new file mode 100644 index 0000000..0991d91 --- /dev/null +++ b/MareSynchronos/MareConfiguration/Models/TextureShrinkMode.cs @@ -0,0 +1,10 @@ +namespace MareSynchronos.MareConfiguration.Models; + +public enum TextureShrinkMode +{ + Never, + Default, + DefaultHiRes, + Always, + AlwaysHiRes +} diff --git a/MareSynchronos/PlayerData/Handlers/PairHandler.cs b/MareSynchronos/PlayerData/Handlers/PairHandler.cs index 06b5acc..b3e2706 100644 --- a/MareSynchronos/PlayerData/Handlers/PairHandler.cs +++ b/MareSynchronos/PlayerData/Handlers/PairHandler.cs @@ -13,6 +13,7 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Diagnostics; +using System.Runtime.CompilerServices; using ObjectKind = MareSynchronos.API.Data.Enum.ObjectKind; namespace MareSynchronos.PlayerData.Handlers; @@ -489,6 +490,17 @@ public sealed class PairHandler : DisposableMediatorSubscriberBase await Task.Delay(TimeSpan.FromSeconds(2), downloadToken).ConfigureAwait(false); } + try + { + Mediator.Publish(new HaltScanMessage(nameof(PlayerPerformanceService.ShrinkTextures))); + if (await _playerPerformanceService.ShrinkTextures(this, charaData, downloadToken).ConfigureAwait(false)) + _ = TryCalculateModdedDictionary(applicationBase, charaData, out moddedPaths, downloadToken); + } + finally + { + Mediator.Publish(new ResumeScanMessage(nameof(PlayerPerformanceService.ShrinkTextures))); + } + bool exceedsThreshold = !await _playerPerformanceService.CheckBothThresholds(this, charaData).ConfigureAwait(false); if (exceedsThreshold) @@ -753,7 +765,7 @@ public sealed class PairHandler : DisposableMediatorSubscriberBase (item) => { token.ThrowIfCancellationRequested(); - var fileCache = _fileDbManager.GetFileCacheByHash(item.Hash); + var fileCache = _fileDbManager.GetFileCacheByHash(item.Hash, preferSubst: true); if (fileCache != null) { if (string.IsNullOrEmpty(new FileInfo(fileCache.ResolvedFilepath).Extension)) diff --git a/MareSynchronos/Services/PairAnalyzer.cs b/MareSynchronos/Services/PairAnalyzer.cs index 57a4bf7..c3cdbe0 100644 --- a/MareSynchronos/Services/PairAnalyzer.cs +++ b/MareSynchronos/Services/PairAnalyzer.cs @@ -131,7 +131,7 @@ public sealed class PairAnalyzer : DisposableMediatorSubscriberBase var fileCacheEntries = _fileCacheManager.GetAllFileCachesByHash(fileEntry.Hash, ignoreCacheEntries: false, validate: false).ToList(); if (fileCacheEntries.Count == 0) continue; - var filePath = fileCacheEntries[0].ResolvedFilepath; + var filePath = fileCacheEntries[^1].ResolvedFilepath; FileInfo fi = new(filePath); string ext = "unk?"; try diff --git a/MareSynchronos/Services/PlayerPerformanceService.cs b/MareSynchronos/Services/PlayerPerformanceService.cs index b454ee6..a4f1180 100644 --- a/MareSynchronos/Services/PlayerPerformanceService.cs +++ b/MareSynchronos/Services/PlayerPerformanceService.cs @@ -1,3 +1,4 @@ +using Lumina.Data; using MareSynchronos.API.Data; using MareSynchronos.FileCache; using MareSynchronos.MareConfiguration; @@ -8,6 +9,7 @@ using MareSynchronos.Services.ServerConfiguration; using MareSynchronos.UI; using MareSynchronos.WebAPI.Files.Models; using Microsoft.Extensions.Logging; +using System.ComponentModel; namespace MareSynchronos.Services; @@ -56,13 +58,8 @@ public class PlayerPerformanceService : DisposableMediatorSubscriberBase long triUsage = 0; - if (!charaData.FileReplacements.TryGetValue(API.Data.Enum.ObjectKind.Player, out List? playerReplacements)) - { - pair.LastAppliedDataTris = 0; - return true; - } - - var moddedModelHashes = playerReplacements.Where(p => string.IsNullOrEmpty(p.FileSwapPath) && p.GamePaths.Any(g => g.EndsWith("mdl", StringComparison.OrdinalIgnoreCase))) + var moddedModelHashes = charaData.FileReplacements.SelectMany(k => k.Value) + .Where(p => string.IsNullOrEmpty(p.FileSwapPath) && p.GamePaths.Any(g => g.EndsWith("mdl", StringComparison.OrdinalIgnoreCase))) .Select(p => p.Hash) .Distinct(StringComparer.OrdinalIgnoreCase) .ToList(); @@ -107,20 +104,15 @@ public class PlayerPerformanceService : DisposableMediatorSubscriberBase return true; } - public bool ComputeAndAutoPauseOnVRAMUsageThresholds(PairHandler pairHandler, CharacterData charaData, List toDownloadFiles) + public bool ComputeAndAutoPauseOnVRAMUsageThresholds(PairHandler pairHandler, CharacterData charaData, List toDownloadFiles, bool affect = false) { var config = _playerPerformanceConfigService.Current; var pair = pairHandler.Pair; long vramUsage = 0; - if (!charaData.FileReplacements.TryGetValue(API.Data.Enum.ObjectKind.Player, out List? playerReplacements)) - { - pair.LastAppliedApproximateVRAMBytes = 0; - return true; - } - - var moddedTextureHashes = playerReplacements.Where(p => string.IsNullOrEmpty(p.FileSwapPath) && p.GamePaths.Any(g => g.EndsWith(".tex", StringComparison.OrdinalIgnoreCase))) + var moddedTextureHashes = charaData.FileReplacements.SelectMany(k => k.Value) + .Where(p => string.IsNullOrEmpty(p.FileSwapPath) && p.GamePaths.Any(g => g.EndsWith(".tex", StringComparison.OrdinalIgnoreCase))) .Select(p => p.Hash) .Distinct(StringComparer.OrdinalIgnoreCase) .ToList(); @@ -136,7 +128,7 @@ public class PlayerPerformanceService : DisposableMediatorSubscriberBase } else { - var fileEntry = _fileCacheManager.GetFileCacheByHash(hash); + var fileEntry = _fileCacheManager.GetFileCacheByHash(hash, preferSubst: true); if (fileEntry == null) continue; if (fileEntry.Size == null) @@ -168,6 +160,9 @@ public class PlayerPerformanceService : DisposableMediatorSubscriberBase if (vramUsage > vramUsageThreshold * 1024 * 1024) { + if (!affect) + return false; + if (notify && !pair.IsApplicationBlocked) { _mediator.Publish(new NotificationMessage($"{pair.PlayerName} ({pair.UserData.AliasOrUID}) automatically blocked", @@ -185,4 +180,139 @@ public class PlayerPerformanceService : DisposableMediatorSubscriberBase return true; } + + public async Task ShrinkTextures(PairHandler pairHandler, CharacterData charaData, CancellationToken token) + { + var config = _playerPerformanceConfigService.Current; + var pair = pairHandler.Pair; + + if (config.TextureShrinkMode == MareConfiguration.Models.TextureShrinkMode.Never) + return false; + + // XXX: Temporary + if (config.TextureShrinkMode == MareConfiguration.Models.TextureShrinkMode.Default) + return false; + + var moddedTextureHashes = charaData.FileReplacements.SelectMany(k => k.Value) + .Where(p => string.IsNullOrEmpty(p.FileSwapPath) && p.GamePaths.Any(g => g.EndsWith(".tex", StringComparison.OrdinalIgnoreCase))) + .Select(p => p.Hash) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + + bool shrunken = false; + + await Parallel.ForEachAsync(moddedTextureHashes, + token, + async (hash, token) => { + var fileEntry = _fileCacheManager.GetFileCacheByHash(hash, preferSubst: true); + if (fileEntry == null) return; + if (fileEntry.IsSubstEntry) return; + + var texFormat = await _xivDataAnalyzer.GetTexFormatByHash(hash); + var filePath = fileEntry.ResolvedFilepath; + var tmpFilePath = _fileCacheManager.GetSubstFilePath(Guid.NewGuid().ToString(), "tmp"); + var newFilePath = _fileCacheManager.GetSubstFilePath(hash, "tex"); + var mipLevel = 0; + uint width = texFormat.Width; + uint height = texFormat.Height; + long offsetDelta = 0; + + uint bitsPerPixel = texFormat.Format switch + { + 0x1130 => 8, // L8 + 0x1131 => 8, // A8 + 0x1440 => 16, // A4R4G4B4 + 0x1441 => 16, // A1R5G5B5 + 0x1450 => 32, // A8R8G8B8 + 0x1451 => 32, // X8R8G8B8 + 0x2150 => 32, // R32F + 0x2250 => 32, // G16R16F + 0x2260 => 64, // R32G32F + 0x2460 => 64, // A16B16G16R16F + 0x2470 => 128, // A32B32G32R32F + 0x3420 => 4, // DXT1 + 0x3430 => 8, // DXT3 + 0x3431 => 8, // DXT5 + 0x4140 => 16, // D16 + 0x4250 => 32, // D24S8 + 0x6120 => 4, // BC4 + 0x6230 => 8, // BC5 + 0x6432 => 8, // BC7 + _ => 0 + }; + + uint maxSize = (bitsPerPixel <= 8) ? (2048U * 2048U) : (1024U * 1024U); + + while (width * height > maxSize && mipLevel < texFormat.MipCount - 1) + { + offsetDelta += width * height * bitsPerPixel / 8; + mipLevel++; + width /= 2; + height /= 2; + } + + if (offsetDelta == 0) + return; + + _logger.LogDebug("Shrinking {hash} from from {a}x{b} to {c}x{d}", + hash, texFormat.Width, texFormat.Height, width, height); + + try + { + var inFile = new FileStream(filePath, FileMode.Open, FileAccess.Read); + using var reader = new BinaryReader(inFile); + + var header = reader.ReadBytes(80); + reader.BaseStream.Position = 14; + byte mipByte = reader.ReadByte(); + byte mipCount = (byte)(mipByte & 0x7F); + + var outFile = new FileStream(tmpFilePath, FileMode.Create, FileAccess.Write, FileShare.None); + using var writer = new BinaryWriter(outFile); + writer.Write(header); + + // Update width/height + writer.BaseStream.Position = 8; + writer.Write((ushort)width); + writer.Write((ushort)height); + + // Update the mip count + writer.BaseStream.Position = 14; + writer.Write((ushort)((mipByte & 0x80) | (mipCount - mipLevel))); + + // Reset all of the LoD mips + writer.BaseStream.Position = 16; + for (int i = 0; i < 3; ++i) + writer.Write((uint)0); + + // Reset all of the mip offsets + // (This data is garbage in a lot of modded textures, so its hard to fix it up correctly) + writer.BaseStream.Position = 28; + for (int i = 0; i < 13; ++i) + writer.Write((uint)80); + + // Write the texture data shifted + outFile.Position = 80; + inFile.Position = 80 + offsetDelta; + + await inFile.CopyToAsync(outFile, 81920, token); + + reader.Dispose(); + writer.Dispose(); + + File.Move(tmpFilePath, newFilePath); + _fileCacheManager.CreateSubstEntry(newFilePath); + shrunken = true; + } + catch (Exception e) + { + _logger.LogWarning(e, "Failed to shrink texture {hash}", hash); + if (File.Exists(tmpFilePath)) + File.Delete(tmpFilePath); + } + } + ); + + return shrunken; + } } \ No newline at end of file diff --git a/MareSynchronos/Services/XivDataAnalyzer.cs b/MareSynchronos/Services/XivDataAnalyzer.cs index 60e0db8..4919852 100644 --- a/MareSynchronos/Services/XivDataAnalyzer.cs +++ b/MareSynchronos/Services/XivDataAnalyzer.cs @@ -3,12 +3,14 @@ using FFXIVClientStructs.FFXIV.Client.Graphics.Scene; using FFXIVClientStructs.Havok.Animation; using FFXIVClientStructs.Havok.Common.Base.Types; using FFXIVClientStructs.Havok.Common.Serialize.Util; +using Lumina.Data; using MareSynchronos.FileCache; using MareSynchronos.Interop.GameModel; using MareSynchronos.MareConfiguration; using MareSynchronos.PlayerData.Handlers; using Microsoft.Extensions.Logging; using System.Runtime.InteropServices; +using System.Text; namespace MareSynchronos.Services; @@ -18,6 +20,7 @@ public sealed class XivDataAnalyzer private readonly FileCacheManager _fileCacheManager; private readonly XivDataStorageService _configService; private readonly List _failedCalculatedTris = []; + private readonly List _failedCalculatedTex = []; public XivDataAnalyzer(ILogger logger, FileCacheManager fileCacheManager, XivDataStorageService configService) @@ -67,7 +70,7 @@ public sealed class XivDataAnalyzer { if (_configService.Current.BonesDictionary.TryGetValue(hash, out var bones)) return bones; - var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash); + var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash, preferSubst: true); if (cacheEntity == null) return null; using BinaryReader reader = new BinaryReader(File.Open(cacheEntity.ResolvedFilepath, FileMode.Open, FileAccess.Read, FileShare.Read)); @@ -158,7 +161,7 @@ public sealed class XivDataAnalyzer if (_failedCalculatedTris.Contains(hash, StringComparer.Ordinal)) return 0; - var path = _fileCacheManager.GetFileCacheByHash(hash); + var path = _fileCacheManager.GetFileCacheByHash(hash, preferSubst: true); if (path == null || !path.ResolvedFilepath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase)) return 0; @@ -211,4 +214,45 @@ public sealed class XivDataAnalyzer return 0; } } + + public async Task<(uint Format, int MipCount, ushort Width, ushort Height)> GetTexFormatByHash(string hash) + { + if (_configService.Current.TexDictionary.TryGetValue(hash, out var cachedTex) && cachedTex.Mip0Size > 0) + return cachedTex; + + if (_failedCalculatedTex.Contains(hash, StringComparer.Ordinal)) + return default; + + var path = _fileCacheManager.GetFileCacheByHash(hash); + if (path == null || !path.ResolvedFilepath.EndsWith(".tex", StringComparison.OrdinalIgnoreCase)) + return default; + + var filePath = path.ResolvedFilepath; + + try + { + _logger.LogDebug("Detected Texture File {path}, reading header", filePath); + using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read); + using var r = new LuminaBinaryReader(stream); + var texHeader = r.ReadStructure(); + + if (texHeader.Format == 0 || texHeader.MipCount == 0 || texHeader.ArraySize != 0 || texHeader.MipCount > 13) + { + _failedCalculatedTex.Add(hash); + _configService.Current.TexDictionary[hash] = default; + _configService.Save(); + return default; + } + + return ((uint)texHeader.Format, texHeader.MipCount, texHeader.Width, texHeader.Height); + } + catch (Exception e) + { + _failedCalculatedTex.Add(hash); + _configService.Current.TriangleDictionary[hash] = 0; + _configService.Save(); + _logger.LogWarning(e, "Could not parse file {file}", filePath); + return default; + } + } } diff --git a/MareSynchronos/UI/SettingsUi.cs b/MareSynchronos/UI/SettingsUi.cs index f3a2cd3..3c77eee 100644 --- a/MareSynchronos/UI/SettingsUi.cs +++ b/MareSynchronos/UI/SettingsUi.cs @@ -1242,6 +1242,35 @@ public class SettingsUi : WindowMediatorSubscriberBase bool recalculatePerformance = false; string? recalculatePerformanceUID = null; + _uiShared.BigText("Global Configuration"); + + bool alwaysShrinkTextures = _playerPerformanceConfigService.Current.TextureShrinkMode == TextureShrinkMode.Always; + if (ImGui.Checkbox("Shrink downloaded textures", ref alwaysShrinkTextures)) + { + if (alwaysShrinkTextures) + _playerPerformanceConfigService.Current.TextureShrinkMode = TextureShrinkMode.Always; + else + _playerPerformanceConfigService.Current.TextureShrinkMode = TextureShrinkMode.Never; + _playerPerformanceConfigService.Save(); + recalculatePerformance = true; + _cacheMonitor.ClearSubstStorage(); + } + _uiShared.DrawHelpText("Automatically shrinks texture resolution of synced players to reduce VRAM utilization." + UiSharedService.TooltipSeparator + + "Texture Size Limit (DXT/BC5/BC7 Compressed): 2048x2048" + Environment.NewLine + + "Texture Size Limit (A8R8G8B8 Uncompressed): 1024x1024" + UiSharedService.TooltipSeparator + + "Enable to reduce lag in large crowds." + Environment.NewLine + + "Disable this for higher quality during GPose."); + + var totalVramBytes = _pairManager.GetOnlineUserPairs().Where(p => p.IsVisible && p.LastAppliedApproximateVRAMBytes > 0).Sum(p => p.LastAppliedApproximateVRAMBytes); + + ImGui.TextUnformatted("Current VRAM utilization by all nearby players:"); + ImGui.SameLine(); + using (ImRaii.PushColor(ImGuiCol.Text, ImGuiColors.HealerGreen, totalVramBytes < 2.0 * 1024.0 * 1024.0 * 1024.0)) + using (ImRaii.PushColor(ImGuiCol.Text, ImGuiColors.DalamudYellow, totalVramBytes >= 4.0 * 1024.0 * 1024.0 * 1024.0)) + using (ImRaii.PushColor(ImGuiCol.Text, ImGuiColors.DalamudRed, totalVramBytes >= 6.0 * 1024.0 * 1024.0 * 1024.0)) + ImGui.TextUnformatted($"{totalVramBytes / 1024.0 / 1024.0 / 1024.0:0.00} GiB"); + + ImGui.Separator(); _uiShared.BigText("Individual Limits"); bool autoPause = _playerPerformanceConfigService.Current.AutoPausePlayersExceedingThresholds; if (ImGui.Checkbox("Automatically block players exceeding thresholds", ref autoPause))