Add texture shrinking feature

MareSynchronos/FileCache/CacheMonitor.cs

@@ -36,6 +36,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         {
             StartPenumbraWatcher(_ipcManager.Penumbra.ModDirectory);
             StartMareWatcher(configService.Current.CacheFolder);
+            StartSubstWatcher(_fileDbManager.SubstFolder);
             InvokeScan();
         });
         Mediator.Subscribe<HaltScanMessage>(this, (msg) => HaltScan(msg.Source));
@@ -43,6 +44,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         Mediator.Subscribe<DalamudLoginMessage>(this, (_) =>
         {
             StartMareWatcher(configService.Current.CacheFolder);
+            StartSubstWatcher(_fileDbManager.SubstFolder);
             StartPenumbraWatcher(_ipcManager.Penumbra.ModDirectory);
             InvokeScan();
         });
@@ -58,6 +60,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         if (configService.Current.HasValidSetup())
         {
             StartMareWatcher(configService.Current.CacheFolder);
+            StartSubstWatcher(_fileDbManager.SubstFolder);
             InvokeScan();
         }
 
@@ -90,6 +93,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
     public long FileCacheSize { get; set; }
     public long FileCacheDriveFree { get; set; }
     public ConcurrentDictionary<string, int> HaltScanLocks { get; set; } = new(StringComparer.Ordinal);
+    private int LockCount = 0;
     public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
     public long TotalFiles { get; private set; }
     public long TotalFilesStorage { get; private set; }
@@ -97,18 +101,22 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
     public void HaltScan(string source)
     {
         HaltScanLocks.AddOrUpdate(source, 1, (k, v) => v + 1);
+        Interlocked.Increment(ref LockCount);
     }
 
     record WatcherChange(WatcherChangeTypes ChangeType, string? OldPath = null);
     private readonly Dictionary<string, WatcherChange> _watcherChanges = new Dictionary<string, WatcherChange>(StringComparer.OrdinalIgnoreCase);
     private readonly Dictionary<string, WatcherChange> _mareChanges = new Dictionary<string, WatcherChange>(StringComparer.OrdinalIgnoreCase);
+    private readonly Dictionary<string, WatcherChange> _substChanges = new Dictionary<string, WatcherChange>(StringComparer.OrdinalIgnoreCase);
 
     public void StopMonitoring()
     {
         Logger.LogInformation("Stopping monitoring of Penumbra and Mare storage folders");
         MareWatcher?.Dispose();
+        SubstWatcher?.Dispose();
         PenumbraWatcher?.Dispose();
         MareWatcher = null;
+        SubstWatcher = null;
         PenumbraWatcher = null;
     }
 
@@ -147,13 +155,53 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         MareWatcher.EnableRaisingEvents = true;
     }
 
+    public void StartSubstWatcher(string? substPath)
+    {
+        SubstWatcher?.Dispose();
+        if (string.IsNullOrEmpty(substPath))
+        {
+            SubstWatcher = null;
+            Logger.LogWarning("Mare file path is not set, cannot start the FSW for Mare.");
+            return;
+        }
+
+        try
+        {
+            if (!Directory.Exists(substPath))
+                Directory.CreateDirectory(substPath);
+        }
+        catch
+        {
+            Logger.LogWarning("Could not create subst directory at {path}.", substPath);
+            return;
+        }
+
+        Logger.LogDebug("Initializing Subst FSW on {path}", substPath);
+        SubstWatcher = new()
+        {
+            Path = substPath,
+            InternalBufferSize = 8388608,
+            NotifyFilter = NotifyFilters.CreationTime
+                | NotifyFilters.LastWrite
+                | NotifyFilters.FileName
+                | NotifyFilters.DirectoryName
+                | NotifyFilters.Size,
+            Filter = "*.*",
+            IncludeSubdirectories = false,
+        };
+
+        SubstWatcher.Deleted += SubstWatcher_FileChanged;
+        SubstWatcher.Created += SubstWatcher_FileChanged;
+        SubstWatcher.EnableRaisingEvents = true;
+    }
+
     private void MareWatcher_FileChanged(object sender, FileSystemEventArgs e)
     {
         Logger.LogTrace("Mare FSW: FileChanged: {change} => {path}", e.ChangeType, e.FullPath);
 
         if (!AllowedFileExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return;
 
-        lock (_watcherChanges)
+        lock (_mareChanges)
         {
             _mareChanges[e.FullPath] = new(e.ChangeType);
         }
@@ -161,6 +209,20 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         _ = MareWatcherExecution();
     }
 
+    private void SubstWatcher_FileChanged(object sender, FileSystemEventArgs e)
+    {
+        Logger.LogTrace("Subst FSW: FileChanged: {change} => {path}", e.ChangeType, e.FullPath);
+
+        if (!AllowedFileExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return;
+
+        lock (_substChanges)
+        {
+            _substChanges[e.FullPath] = new(e.ChangeType);
+        }
+
+        _ = SubstWatcherExecution();
+    }
+
     public void StartPenumbraWatcher(string? penumbraPath)
     {
         PenumbraWatcher?.Dispose();
@@ -247,8 +309,10 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
 
     private CancellationTokenSource _penumbraFswCts = new();
     private CancellationTokenSource _mareFswCts = new();
+    private CancellationTokenSource _substFswCts = new();
     public FileSystemWatcher? PenumbraWatcher { get; private set; }
     public FileSystemWatcher? MareWatcher { get; private set; }
+    public FileSystemWatcher? SubstWatcher { get; private set; }
 
     private async Task MareWatcherExecution()
     {
@@ -281,6 +345,67 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         HandleChanges(changes);
     }
 
+    private async Task SubstWatcherExecution()
+    {
+        _substFswCts = _substFswCts.CancelRecreate();
+        var token = _substFswCts.Token;
+        var delay = TimeSpan.FromSeconds(5);
+        Dictionary<string, WatcherChange> changes;
+        lock (_substChanges)
+            changes = _substChanges.ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal);
+        try
+        {
+            do
+            {
+                await Task.Delay(delay, token).ConfigureAwait(false);
+            } while (HaltScanLocks.Any(f => f.Value > 0));
+        }
+        catch (TaskCanceledException)
+        {
+            return;
+        }
+
+        lock (_substChanges)
+        {
+            foreach (var key in changes.Keys)
+            {
+                _substChanges.Remove(key);
+            }
+        }
+
+        HandleChanges(changes);
+    }
+
+    public void ClearSubstStorage()
+    {
+        var substDir = _fileDbManager.SubstFolder;
+        var allSubstFiles = Directory.GetFiles(substDir, "*.*", SearchOption.TopDirectoryOnly)
+                                .Where(f =>
+                                {
+                                    var val = f.Split('\\')[^1];
+                                    return val.Length == 40 || (val.Split('.').FirstOrDefault()?.Length ?? 0) == 40
+                                        || val.EndsWith(".tmp", StringComparison.OrdinalIgnoreCase);
+                                });
+        if (SubstWatcher != null)
+            SubstWatcher.EnableRaisingEvents = false;
+
+        Dictionary<string, WatcherChange> changes = _substChanges.ToDictionary(t => t.Key, t => new WatcherChange(WatcherChangeTypes.Deleted, t.Key), StringComparer.Ordinal);
+
+        foreach (var file in allSubstFiles)
+        {
+            try
+            {
+                File.Delete(file);
+            }
+            catch { }
+        }
+
+        HandleChanges(changes);
+
+        if (SubstWatcher != null)
+            SubstWatcher.EnableRaisingEvents = true;
+    }
+
     private void HandleChanges(Dictionary<string, WatcherChange> changes)
     {
         lock (_fileDbManager)
@@ -408,7 +533,9 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
             Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder);
         }
 
-        var files = Directory.EnumerateFiles(_configService.Current.CacheFolder).Select(f => new FileInfo(f))
+        var files = Directory.EnumerateFiles(_configService.Current.CacheFolder)
+            .Concat(Directory.EnumerateFiles(_fileDbManager.SubstFolder))
+            .Select(f => new FileInfo(f))
             .OrderBy(f => f.LastAccessTime).ToList();
         FileCacheSize = files
             .Sum(f =>
@@ -429,6 +556,8 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
 
         if (FileCacheSize < maxCacheInBytes) return;
 
+        var substDir = _fileDbManager.SubstFolder;
+
         var maxCacheBuffer = maxCacheInBytes * 0.05d;
         while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer)
         {
@@ -442,11 +571,16 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
     public void ResetLocks()
     {
         HaltScanLocks.Clear();
+        LockCount = 0;
     }
 
     public void ResumeScan(string source)
     {
         HaltScanLocks.AddOrUpdate(source, 0, (k, v) => Math.Max(0, v - 1));
+        int lockCount = Interlocked.Decrement(ref LockCount);
+
+        if (lockCount == 0)
+            HaltScanLocks.Clear();
     }
 
     protected override void Dispose(bool disposing)
@@ -455,8 +589,10 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         _scanCancellationTokenSource?.Cancel();
         PenumbraWatcher?.Dispose();
         MareWatcher?.Dispose();
+        SubstWatcher?.Dispose();
         _penumbraFswCts?.CancelDispose();
         _mareFswCts?.CancelDispose();
+        _substFswCts?.CancelDispose();
         _periodicCalculationTokenSource?.CancelDispose();
     }
 
@@ -466,6 +602,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         var penumbraDir = _ipcManager.Penumbra.ModDirectory;
         bool penDirExists = true;
         bool cacheDirExists = true;
+        var substDir = _fileDbManager.SubstFolder;
         if (string.IsNullOrEmpty(penumbraDir) || !Directory.Exists(penumbraDir))
         {
             penDirExists = false;
@@ -481,6 +618,16 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
             return;
         }
 
+        try
+        {
+            if (!Directory.Exists(substDir))
+                Directory.CreateDirectory(substDir);
+        }
+        catch
+        {
+            Logger.LogWarning("Could not create subst directory at {path}.", substDir);
+        }
+
         var previousThreadPriority = Thread.CurrentThread.Priority;
         Thread.CurrentThread.Priority = ThreadPriority.Lowest;
         Logger.LogDebug("Getting files from {penumbra} and {storage}", penumbraDir, _configService.Current.CacheFolder);
@@ -509,6 +656,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         }
 
         var allCacheFiles = Directory.GetFiles(_configService.Current.CacheFolder, "*.*", SearchOption.TopDirectoryOnly)
+                                .Concat(Directory.GetFiles(substDir, "*.*", SearchOption.TopDirectoryOnly))
                                 .AsParallel()
                                 .Where(f =>
                                 {
@@ -654,7 +802,13 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
                     try
                     {
                         var entry = _fileDbManager.CreateFileEntry(cachePath);
-                        if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath);
+                        if (entry == null)
+                        {
+                            if (cachePath.StartsWith(substDir, StringComparison.Ordinal))
+                                _ = _fileDbManager.CreateSubstEntry(cachePath);
+                            else
+                                _ = _fileDbManager.CreateCacheEntry(cachePath);
+                        }
                     }
                     catch (Exception ex)
                     {
@@ -678,6 +832,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
             _configService.Current.InitialScanComplete = true;
             _configService.Save();
             StartMareWatcher(_configService.Current.CacheFolder);
+            StartSubstWatcher(_fileDbManager.SubstFolder);
             StartPenumbraWatcher(penumbraDir);
         }
     }

MareSynchronos/FileCache/FileCacheEntity.cs

@@ -17,6 +17,7 @@ public class FileCacheEntity
     public string CsvEntry => $"{Hash}{FileCacheManager.CsvSplit}{PrefixedFilePath}{FileCacheManager.CsvSplit}{LastModifiedDateTicks}|{Size ?? -1}|{CompressedSize ?? -1}";
     public string Hash { get; set; }
     public bool IsCacheEntry => PrefixedFilePath.StartsWith(FileCacheManager.CachePrefix, StringComparison.OrdinalIgnoreCase);
+    public bool IsSubstEntry => PrefixedFilePath.StartsWith(FileCacheManager.SubstPrefix, StringComparison.OrdinalIgnoreCase);
     public string LastModifiedDateTicks { get; set; }
     public string PrefixedFilePath { get; init; }
     public string ResolvedFilepath { get; private set; } = string.Empty;

MareSynchronos/FileCache/FileCacheManager.cs

@@ -1,4 +1,5 @@
-using K4os.Compression.LZ4.Streams;
+using Dalamud.Utility;
+using K4os.Compression.LZ4.Streams;
 using MareSynchronos.Interop.Ipc;
 using MareSynchronos.MareConfiguration;
 using MareSynchronos.Services.Mediator;
@@ -16,6 +17,10 @@ public sealed class FileCacheManager : IHostedService
     public const string CachePrefix = "{cache}";
     public const string CsvSplit = "|";
     public const string PenumbraPrefix = "{penumbra}";
+    public const string SubstPrefix = "{subst}";
+    public const string SubstPath = "subst";
+    public string CacheFolder => _configService.Current.CacheFolder;
+    public string SubstFolder => CacheFolder.IsNullOrEmpty() ? string.Empty : CacheFolder.ToLowerInvariant().TrimEnd('\\') + "\\" + SubstPath;
     private readonly MareConfigService _configService;
     private readonly MareMediator _mareMediator;
     private readonly string _csvPath;
@@ -47,6 +52,19 @@ public sealed class FileCacheManager : IHostedService
         return CreateFileCacheEntity(fi, prefixedPath);
     }
 
+    public FileCacheEntity? CreateSubstEntry(string path)
+    {
+        FileInfo fi = new(path);
+        if (!fi.Exists) return null;
+        _logger.LogTrace("Creating substitute entry for {path}", path);
+        var fullName = fi.FullName.ToLowerInvariant();
+        if (!fullName.Contains(SubstFolder, StringComparison.Ordinal)) return null;
+        string prefixedPath = fullName.Replace(SubstFolder, SubstPrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
+        var fakeHash = Path.GetFileNameWithoutExtension(fi.FullName).ToUpperInvariant();
+        var result = CreateFileCacheEntity(fi, prefixedPath, fakeHash);
+        return result;
+    }
+
     public FileCacheEntity? CreateFileEntry(string path)
     {
         FileInfo fi = new(path);
@@ -65,7 +83,7 @@ public sealed class FileCacheManager : IHostedService
         List<FileCacheEntity> output = [];
         if (_fileCaches.TryGetValue(hash, out var fileCacheEntities))
         {
-            foreach (var fileCache in fileCacheEntities.Where(c => ignoreCacheEntries ? !c.IsCacheEntry : true).ToList())
+            foreach (var fileCache in fileCacheEntities.Where(c => ignoreCacheEntries ? (!c.IsCacheEntry && !c.IsSubstEntry) : true).ToList())
             {
                 if (!validate) output.Add(fileCache);
                 else
@@ -89,6 +107,7 @@ public sealed class FileCacheManager : IHostedService
         foreach (var fileCache in cacheEntries)
         {
             if (cancellationToken.IsCancellationRequested) break;
+            if (fileCache.IsSubstEntry) continue;
 
             _logger.LogInformation("Validating {file}", fileCache.ResolvedFilepath);
 
@@ -139,6 +158,11 @@ public sealed class FileCacheManager : IHostedService
         return Path.Combine(_configService.Current.CacheFolder, hash + "." + extension);
     }
 
+    public string GetSubstFilePath(string hash, string extension)
+    {
+        return Path.Combine(SubstFolder, hash + "." + extension);
+    }
+
     public async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken)
     {
         var fileCache = GetFileCacheByHash(fileHash)!;
@@ -151,14 +175,24 @@ public sealed class FileCacheManager : IHostedService
         return (fileHash, ms.ToArray());
     }
 
-    public FileCacheEntity? GetFileCacheByHash(string hash)
+    public FileCacheEntity? GetFileCacheByHash(string hash, bool preferSubst = false)
     {
+        var caches = GetFileCachesByHash(hash);
+        if (preferSubst && caches.Subst != null)
+            return caches.Subst;
+        return caches.Penumbra ?? caches.Cache;
+    }
+
+    public (FileCacheEntity? Penumbra, FileCacheEntity? Cache, FileCacheEntity? Subst) GetFileCachesByHash(string hash)
+    {
+        (FileCacheEntity? Penumbra, FileCacheEntity? Cache, FileCacheEntity? Subst) result = (null, null, null);
         if (_fileCaches.TryGetValue(hash, out var hashes))
         {
-            var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix) ? 0 : 1).FirstOrDefault();
+            result.Penumbra = hashes.Where(p => p.PrefixedFilePath.StartsWith(PenumbraPrefix, StringComparison.Ordinal)).Select(GetValidatedFileCache).FirstOrDefault();
-            if (item != null) return GetValidatedFileCache(item);
+            result.Cache = hashes.Where(p => p.PrefixedFilePath.StartsWith(CachePrefix, StringComparison.Ordinal)).Select(GetValidatedFileCache).FirstOrDefault();
+            result.Subst = hashes.Where(p => p.PrefixedFilePath.StartsWith(SubstPrefix, StringComparison.Ordinal)).Select(GetValidatedFileCache).FirstOrDefault();
         }
-        return null;
+        return result;
     }
 
     private FileCacheEntity? GetFileCacheByPath(string path)
@@ -187,6 +221,7 @@ public sealed class FileCacheManager : IHostedService
             var cleanedPaths = paths.Distinct(StringComparer.OrdinalIgnoreCase).ToDictionary(p => p,
                 p => p.Replace("/", "\\", StringComparison.OrdinalIgnoreCase)
                     .Replace(_ipcManager.Penumbra.ModDirectory!, _ipcManager.Penumbra.ModDirectory!.EndsWith('\\') ? PenumbraPrefix + '\\' : PenumbraPrefix, StringComparison.OrdinalIgnoreCase)
+                    .Replace(SubstFolder, SubstPrefix, StringComparison.OrdinalIgnoreCase)
                     .Replace(_configService.Current.CacheFolder, _configService.Current.CacheFolder.EndsWith('\\') ? CachePrefix + '\\' : CachePrefix, StringComparison.OrdinalIgnoreCase)
                     .Replace("\\\\", "\\", StringComparison.Ordinal),
                 StringComparer.OrdinalIgnoreCase);
@@ -207,9 +242,11 @@ public sealed class FileCacheManager : IHostedService
                 }
                 else
                 {
-                    if (!entry.Value.Contains(CachePrefix, StringComparison.Ordinal))
+                    if (entry.Value.StartsWith(PenumbraPrefix, StringComparison.Ordinal))
                         result.Add(entry.Key, CreateFileEntry(entry.Key));
-                    else
+                    else if (entry.Value.StartsWith(SubstPrefix, StringComparison.Ordinal))
+                        result.Add(entry.Key, CreateSubstEntry(entry.Key));
+                    else if (entry.Value.StartsWith(CachePrefix, StringComparison.Ordinal))
                         result.Add(entry.Key, CreateCacheEntry(entry.Key));
                 }
             }
@@ -360,6 +397,10 @@ public sealed class FileCacheManager : IHostedService
         {
             fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(PenumbraPrefix, _ipcManager.Penumbra.ModDirectory, StringComparison.Ordinal));
         }
+        else if (fileCache.PrefixedFilePath.StartsWith(SubstPrefix, StringComparison.OrdinalIgnoreCase))
+        {
+            fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(SubstPrefix, SubstFolder, StringComparison.Ordinal));
+        }
         else if (fileCache.PrefixedFilePath.StartsWith(CachePrefix, StringComparison.OrdinalIgnoreCase))
         {
             fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(CachePrefix, _configService.Current.CacheFolder, StringComparison.Ordinal));

MareSynchronos/MareConfiguration/Configurations/PlayerPerformanceConfig.cs

@@ -1,4 +1,6 @@
-namespace MareSynchronos.MareConfiguration.Configurations;
+using MareSynchronos.MareConfiguration.Models;
+
+namespace MareSynchronos.MareConfiguration.Configurations;
 
 public class PlayerPerformanceConfig : IMareConfiguration
 {
@@ -9,4 +11,5 @@ public class PlayerPerformanceConfig : IMareConfiguration
     public int VRAMSizeAutoPauseThresholdMiB { get; set; } = 550;
     public int TrisAutoPauseThresholdThousands { get; set; } = 375;
     public bool IgnoreDirectPairs { get; set; } = true;
+    public TextureShrinkMode TextureShrinkMode { get; set; } = TextureShrinkMode.Default;
 }

MareSynchronos/MareConfiguration/Configurations/XivDataStorageConfig.cs

@@ -5,6 +5,7 @@ namespace MareSynchronos.MareConfiguration.Configurations;
 public class XivDataStorageConfig : IMareConfiguration
 {
     public ConcurrentDictionary<string, long> TriangleDictionary { get; set; } = new(StringComparer.OrdinalIgnoreCase);
+    public ConcurrentDictionary<string, (uint Mip0Size, int MipCount, ushort Width, ushort Height)> TexDictionary { get; set; } = new(StringComparer.OrdinalIgnoreCase);
     public ConcurrentDictionary<string, Dictionary<string, List<ushort>>> BonesDictionary { get; set; } = new(StringComparer.OrdinalIgnoreCase);
     public int Version { get; set; } = 0;
 }

MareSynchronos/MareConfiguration/Models/TextureShrinkMode.cs

@@ -0,0 +1,10 @@
+namespace MareSynchronos.MareConfiguration.Models;
+
+public enum TextureShrinkMode
+{
+	Never,
+	Default,
+	DefaultHiRes,
+	Always,
+	AlwaysHiRes
+}

MareSynchronos/PlayerData/Handlers/PairHandler.cs

@@ -13,6 +13,7 @@ using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using System.Collections.Concurrent;
 using System.Diagnostics;
+using System.Runtime.CompilerServices;
 using ObjectKind = MareSynchronos.API.Data.Enum.ObjectKind;
 
 namespace MareSynchronos.PlayerData.Handlers;
@@ -489,6 +490,17 @@ public sealed class PairHandler : DisposableMediatorSubscriberBase
                 await Task.Delay(TimeSpan.FromSeconds(2), downloadToken).ConfigureAwait(false);
             }
 
+            try
+            {
+                Mediator.Publish(new HaltScanMessage(nameof(PlayerPerformanceService.ShrinkTextures)));
+                if (await _playerPerformanceService.ShrinkTextures(this, charaData, downloadToken).ConfigureAwait(false))
+                    _ = TryCalculateModdedDictionary(applicationBase, charaData, out moddedPaths, downloadToken);
+            }
+            finally
+            {
+                Mediator.Publish(new ResumeScanMessage(nameof(PlayerPerformanceService.ShrinkTextures)));
+            }
+
             bool exceedsThreshold = !await _playerPerformanceService.CheckBothThresholds(this, charaData).ConfigureAwait(false);
 
             if (exceedsThreshold)
@@ -753,7 +765,7 @@ public sealed class PairHandler : DisposableMediatorSubscriberBase
             (item) =>
             {
                 token.ThrowIfCancellationRequested();
-                var fileCache = _fileDbManager.GetFileCacheByHash(item.Hash);
+                var fileCache = _fileDbManager.GetFileCacheByHash(item.Hash, preferSubst: true);
                 if (fileCache != null)
                 {
                     if (string.IsNullOrEmpty(new FileInfo(fileCache.ResolvedFilepath).Extension))

MareSynchronos/Services/PairAnalyzer.cs

@@ -131,7 +131,7 @@ public sealed class PairAnalyzer : DisposableMediatorSubscriberBase
                 var fileCacheEntries = _fileCacheManager.GetAllFileCachesByHash(fileEntry.Hash, ignoreCacheEntries: false, validate: false).ToList();
                 if (fileCacheEntries.Count == 0) continue;
 
-                var filePath = fileCacheEntries[0].ResolvedFilepath;
+                var filePath = fileCacheEntries[^1].ResolvedFilepath;
                 FileInfo fi = new(filePath);
                 string ext = "unk?";
                 try

MareSynchronos/Services/PlayerPerformanceService.cs

@@ -1,3 +1,4 @@
+using Lumina.Data;
 using MareSynchronos.API.Data;
 using MareSynchronos.FileCache;
 using MareSynchronos.MareConfiguration;
@@ -8,6 +9,7 @@ using MareSynchronos.Services.ServerConfiguration;
 using MareSynchronos.UI;
 using MareSynchronos.WebAPI.Files.Models;
 using Microsoft.Extensions.Logging;
+using System.ComponentModel;
 
 namespace MareSynchronos.Services;
 
@@ -56,13 +58,8 @@ public class PlayerPerformanceService : DisposableMediatorSubscriberBase
 
         long triUsage = 0;
 
-        if (!charaData.FileReplacements.TryGetValue(API.Data.Enum.ObjectKind.Player, out List<FileReplacementData>? playerReplacements))
+        var moddedModelHashes = charaData.FileReplacements.SelectMany(k => k.Value)
-        {
+            .Where(p => string.IsNullOrEmpty(p.FileSwapPath) && p.GamePaths.Any(g => g.EndsWith("mdl", StringComparison.OrdinalIgnoreCase)))
-            pair.LastAppliedDataTris = 0;
-            return true;
-        }
-
-        var moddedModelHashes = playerReplacements.Where(p => string.IsNullOrEmpty(p.FileSwapPath) && p.GamePaths.Any(g => g.EndsWith("mdl", StringComparison.OrdinalIgnoreCase)))
             .Select(p => p.Hash)
             .Distinct(StringComparer.OrdinalIgnoreCase)
             .ToList();
@@ -107,20 +104,15 @@ public class PlayerPerformanceService : DisposableMediatorSubscriberBase
         return true;
     }
 
-    public bool ComputeAndAutoPauseOnVRAMUsageThresholds(PairHandler pairHandler, CharacterData charaData, List<DownloadFileTransfer> toDownloadFiles)
+    public bool ComputeAndAutoPauseOnVRAMUsageThresholds(PairHandler pairHandler, CharacterData charaData, List<DownloadFileTransfer> toDownloadFiles, bool affect = false)
     {
         var config = _playerPerformanceConfigService.Current;
         var pair = pairHandler.Pair;
 
         long vramUsage = 0;
 
-        if (!charaData.FileReplacements.TryGetValue(API.Data.Enum.ObjectKind.Player, out List<FileReplacementData>? playerReplacements))
+        var moddedTextureHashes = charaData.FileReplacements.SelectMany(k => k.Value)
-        {
+            .Where(p => string.IsNullOrEmpty(p.FileSwapPath) && p.GamePaths.Any(g => g.EndsWith(".tex", StringComparison.OrdinalIgnoreCase)))
-            pair.LastAppliedApproximateVRAMBytes = 0;
-            return true;
-        }
-
-        var moddedTextureHashes = playerReplacements.Where(p => string.IsNullOrEmpty(p.FileSwapPath) && p.GamePaths.Any(g => g.EndsWith(".tex", StringComparison.OrdinalIgnoreCase)))
             .Select(p => p.Hash)
             .Distinct(StringComparer.OrdinalIgnoreCase)
             .ToList();
@@ -136,7 +128,7 @@ public class PlayerPerformanceService : DisposableMediatorSubscriberBase
             }
             else
             {
-                var fileEntry = _fileCacheManager.GetFileCacheByHash(hash);
+                var fileEntry = _fileCacheManager.GetFileCacheByHash(hash, preferSubst: true);
                 if (fileEntry == null) continue;
 
                 if (fileEntry.Size == null)
@@ -168,6 +160,9 @@ public class PlayerPerformanceService : DisposableMediatorSubscriberBase
 
         if (vramUsage > vramUsageThreshold * 1024 * 1024)
         {
+            if (!affect)
+                return false;
+
             if (notify && !pair.IsApplicationBlocked)
             {
                 _mediator.Publish(new NotificationMessage($"{pair.PlayerName} ({pair.UserData.AliasOrUID}) automatically blocked",
@@ -185,4 +180,139 @@ public class PlayerPerformanceService : DisposableMediatorSubscriberBase
 
         return true;
     }
+
+    public async Task<bool> ShrinkTextures(PairHandler pairHandler, CharacterData charaData, CancellationToken token)
+    {
+        var config = _playerPerformanceConfigService.Current;
+        var pair = pairHandler.Pair;
+
+        if (config.TextureShrinkMode == MareConfiguration.Models.TextureShrinkMode.Never)
+            return false;
+
+        // XXX: Temporary
+        if (config.TextureShrinkMode == MareConfiguration.Models.TextureShrinkMode.Default)
+            return false;
+
+        var moddedTextureHashes = charaData.FileReplacements.SelectMany(k => k.Value)
+            .Where(p => string.IsNullOrEmpty(p.FileSwapPath) && p.GamePaths.Any(g => g.EndsWith(".tex", StringComparison.OrdinalIgnoreCase)))
+            .Select(p => p.Hash)
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToList();
+
+        bool shrunken = false;
+
+        await Parallel.ForEachAsync(moddedTextureHashes,
+            token,
+            async (hash, token) => {
+                var fileEntry = _fileCacheManager.GetFileCacheByHash(hash, preferSubst: true);
+                if (fileEntry == null) return;
+                if (fileEntry.IsSubstEntry) return;
+
+                var texFormat = await _xivDataAnalyzer.GetTexFormatByHash(hash);
+                var filePath = fileEntry.ResolvedFilepath;
+                var tmpFilePath = _fileCacheManager.GetSubstFilePath(Guid.NewGuid().ToString(), "tmp");
+                var newFilePath = _fileCacheManager.GetSubstFilePath(hash, "tex");
+                var mipLevel = 0;
+                uint width = texFormat.Width;
+                uint height = texFormat.Height;
+                long offsetDelta = 0;
+
+                uint bitsPerPixel = texFormat.Format switch
+                {
+                    0x1130 => 8, // L8
+                    0x1131 => 8, // A8
+                    0x1440 => 16, // A4R4G4B4
+                    0x1441 => 16, // A1R5G5B5
+                    0x1450 => 32, // A8R8G8B8
+                    0x1451 => 32, // X8R8G8B8
+                    0x2150 => 32, // R32F
+                    0x2250 => 32, // G16R16F
+                    0x2260 => 64, // R32G32F
+                    0x2460 => 64, // A16B16G16R16F
+                    0x2470 => 128, // A32B32G32R32F
+                    0x3420 => 4, // DXT1
+                    0x3430 => 8, // DXT3
+                    0x3431 => 8, // DXT5
+                    0x4140 => 16, // D16
+                    0x4250 => 32, // D24S8
+                    0x6120 => 4, // BC4
+                    0x6230 => 8, // BC5
+                    0x6432 => 8, // BC7
+                    _ => 0
+                };
+
+                uint maxSize = (bitsPerPixel <= 8) ? (2048U * 2048U) : (1024U * 1024U);
+
+                while (width * height > maxSize && mipLevel < texFormat.MipCount - 1)
+                {
+                    offsetDelta += width * height * bitsPerPixel / 8;
+                    mipLevel++;
+                    width /= 2;
+                    height /= 2;
+                }
+
+                if (offsetDelta == 0)
+                    return;
+
+                _logger.LogDebug("Shrinking {hash} from from {a}x{b} to {c}x{d}",
+                    hash, texFormat.Width, texFormat.Height, width, height);
+
+                try
+                {
+                    var inFile = new FileStream(filePath, FileMode.Open, FileAccess.Read);
+                    using var reader = new BinaryReader(inFile);
+
+                    var header = reader.ReadBytes(80);
+                    reader.BaseStream.Position = 14;
+                    byte mipByte = reader.ReadByte();
+                    byte mipCount = (byte)(mipByte & 0x7F);
+
+                    var outFile = new FileStream(tmpFilePath, FileMode.Create, FileAccess.Write, FileShare.None);
+                    using var writer = new BinaryWriter(outFile);
+                    writer.Write(header);
+
+                    // Update width/height
+                    writer.BaseStream.Position = 8;
+                    writer.Write((ushort)width);
+                    writer.Write((ushort)height);
+
+                    // Update the mip count
+                    writer.BaseStream.Position = 14;
+                    writer.Write((ushort)((mipByte & 0x80) | (mipCount - mipLevel)));
+
+                    // Reset all of the LoD mips
+                    writer.BaseStream.Position = 16;
+                    for (int i = 0; i < 3; ++i)
+                        writer.Write((uint)0);
+
+                    // Reset all of the mip offsets
+                    // (This data is garbage in a lot of modded textures, so its hard to fix it up correctly)
+                    writer.BaseStream.Position = 28;
+                    for (int i = 0; i < 13; ++i)
+                        writer.Write((uint)80);
+
+                    // Write the texture data shifted
+                    outFile.Position = 80;
+                    inFile.Position = 80 + offsetDelta;
+
+                    await inFile.CopyToAsync(outFile, 81920, token);
+
+                    reader.Dispose();
+                    writer.Dispose();
+
+                    File.Move(tmpFilePath, newFilePath);
+                    _fileCacheManager.CreateSubstEntry(newFilePath);
+                    shrunken = true;
+                }
+                catch (Exception e)
+                {
+                    _logger.LogWarning(e, "Failed to shrink texture {hash}", hash);
+                    if (File.Exists(tmpFilePath))
+                        File.Delete(tmpFilePath);
+                }
+            }
+        );
+
+        return shrunken;
+    }
 }

MareSynchronos/Services/XivDataAnalyzer.cs

@@ -3,12 +3,14 @@ using FFXIVClientStructs.FFXIV.Client.Graphics.Scene;
 using FFXIVClientStructs.Havok.Animation;
 using FFXIVClientStructs.Havok.Common.Base.Types;
 using FFXIVClientStructs.Havok.Common.Serialize.Util;
+using Lumina.Data;
 using MareSynchronos.FileCache;
 using MareSynchronos.Interop.GameModel;
 using MareSynchronos.MareConfiguration;
 using MareSynchronos.PlayerData.Handlers;
 using Microsoft.Extensions.Logging;
 using System.Runtime.InteropServices;
+using System.Text;
 
 namespace MareSynchronos.Services;
 
@@ -18,6 +20,7 @@ public sealed class XivDataAnalyzer
     private readonly FileCacheManager _fileCacheManager;
     private readonly XivDataStorageService _configService;
     private readonly List<string> _failedCalculatedTris = [];
+    private readonly List<string> _failedCalculatedTex = [];
 
     public XivDataAnalyzer(ILogger<XivDataAnalyzer> logger, FileCacheManager fileCacheManager,
         XivDataStorageService configService)
@@ -67,7 +70,7 @@ public sealed class XivDataAnalyzer
     {
         if (_configService.Current.BonesDictionary.TryGetValue(hash, out var bones)) return bones;
 
-        var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash);
+        var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash, preferSubst: true);
         if (cacheEntity == null) return null;
 
         using BinaryReader reader = new BinaryReader(File.Open(cacheEntity.ResolvedFilepath, FileMode.Open, FileAccess.Read, FileShare.Read));
@@ -158,7 +161,7 @@ public sealed class XivDataAnalyzer
         if (_failedCalculatedTris.Contains(hash, StringComparer.Ordinal))
             return 0;
 
-        var path = _fileCacheManager.GetFileCacheByHash(hash);
+        var path = _fileCacheManager.GetFileCacheByHash(hash, preferSubst: true);
         if (path == null || !path.ResolvedFilepath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase))
             return 0;
 
@@ -211,4 +214,45 @@ public sealed class XivDataAnalyzer
             return 0;
         }
     }
+
+    public async Task<(uint Format, int MipCount, ushort Width, ushort Height)> GetTexFormatByHash(string hash)
+    {
+        if (_configService.Current.TexDictionary.TryGetValue(hash, out var cachedTex) && cachedTex.Mip0Size > 0)
+            return cachedTex;
+
+        if (_failedCalculatedTex.Contains(hash, StringComparer.Ordinal))
+            return default;
+
+        var path = _fileCacheManager.GetFileCacheByHash(hash);
+        if (path == null || !path.ResolvedFilepath.EndsWith(".tex", StringComparison.OrdinalIgnoreCase))
+            return default;
+
+        var filePath = path.ResolvedFilepath;
+
+        try
+        {
+            _logger.LogDebug("Detected Texture File {path}, reading header", filePath);
+            using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read);
+            using var r = new LuminaBinaryReader(stream);
+            var texHeader = r.ReadStructure<Lumina.Data.Files.TexFile.TexHeader>();
+
+            if (texHeader.Format == 0 || texHeader.MipCount == 0 || texHeader.ArraySize != 0 || texHeader.MipCount > 13)
+            {
+                _failedCalculatedTex.Add(hash);
+                _configService.Current.TexDictionary[hash] = default;
+                _configService.Save();
+                return default;
+            }
+
+            return ((uint)texHeader.Format, texHeader.MipCount, texHeader.Width, texHeader.Height);
+        }
+        catch (Exception e)
+        {
+            _failedCalculatedTex.Add(hash);
+            _configService.Current.TriangleDictionary[hash] = 0;
+            _configService.Save();
+            _logger.LogWarning(e, "Could not parse file {file}", filePath);
+            return default;
+        }
+    }
 }

MareSynchronos/UI/SettingsUi.cs

@@ -1242,6 +1242,35 @@ public class SettingsUi : WindowMediatorSubscriberBase
         bool recalculatePerformance = false;
         string? recalculatePerformanceUID = null;
 
+        _uiShared.BigText("Global Configuration");
+
+        bool alwaysShrinkTextures = _playerPerformanceConfigService.Current.TextureShrinkMode == TextureShrinkMode.Always;
+        if (ImGui.Checkbox("Shrink downloaded textures", ref alwaysShrinkTextures))
+        {
+            if (alwaysShrinkTextures)
+                _playerPerformanceConfigService.Current.TextureShrinkMode = TextureShrinkMode.Always;
+            else
+                _playerPerformanceConfigService.Current.TextureShrinkMode = TextureShrinkMode.Never;
+            _playerPerformanceConfigService.Save();
+            recalculatePerformance = true;
+            _cacheMonitor.ClearSubstStorage();
+        }
+        _uiShared.DrawHelpText("Automatically shrinks texture resolution of synced players to reduce VRAM utilization." + UiSharedService.TooltipSeparator
+            + "Texture Size Limit (DXT/BC5/BC7 Compressed): 2048x2048" + Environment.NewLine
+            + "Texture Size Limit (A8R8G8B8 Uncompressed): 1024x1024" + UiSharedService.TooltipSeparator
+            + "Enable to reduce lag in large crowds." + Environment.NewLine
+            + "Disable this for higher quality during GPose.");
+
+        var totalVramBytes = _pairManager.GetOnlineUserPairs().Where(p => p.IsVisible && p.LastAppliedApproximateVRAMBytes > 0).Sum(p => p.LastAppliedApproximateVRAMBytes);
+
+        ImGui.TextUnformatted("Current VRAM utilization by all nearby players:");
+        ImGui.SameLine();
+        using (ImRaii.PushColor(ImGuiCol.Text, ImGuiColors.HealerGreen, totalVramBytes < 2.0 * 1024.0 * 1024.0 * 1024.0))
+            using (ImRaii.PushColor(ImGuiCol.Text, ImGuiColors.DalamudYellow, totalVramBytes >= 4.0 * 1024.0 * 1024.0 * 1024.0))
+                using (ImRaii.PushColor(ImGuiCol.Text, ImGuiColors.DalamudRed, totalVramBytes >= 6.0 * 1024.0 * 1024.0 * 1024.0))
+                    ImGui.TextUnformatted($"{totalVramBytes / 1024.0 / 1024.0 / 1024.0:0.00} GiB");
+
+        ImGui.Separator();
         _uiShared.BigText("Individual Limits");
         bool autoPause = _playerPerformanceConfigService.Current.AutoPausePlayersExceedingThresholds;
         if (ImGui.Checkbox("Automatically block players exceeding thresholds", ref autoPause))