t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add texture shrinking feature

Browse Source
This commit is contained in:
Loporrit
2025-02-23 12:18:34 +00:00
parent eaaded1ed5
commit 99eecbdc09
11 changed files with 458 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

161
MareSynchronos/FileCache/CacheMonitor.cs
View File

@@ -36,6 +36,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
{
StartPenumbraWatcher(_ipcManager.Penumbra.ModDirectory);
StartMareWatcher(configService.Current.CacheFolder);
StartSubstWatcher(_fileDbManager.SubstFolder);
InvokeScan();
});
Mediator.Subscribe<HaltScanMessage>(this, (msg) => HaltScan(msg.Source));
@@ -43,6 +44,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
Mediator.Subscribe<DalamudLoginMessage>(this, (_) =>
{
StartMareWatcher(configService.Current.CacheFolder);
StartSubstWatcher(_fileDbManager.SubstFolder);
StartPenumbraWatcher(_ipcManager.Penumbra.ModDirectory);
InvokeScan();
});
@@ -58,6 +60,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
if (configService.Current.HasValidSetup())
{
StartMareWatcher(configService.Current.CacheFolder);
StartSubstWatcher(_fileDbManager.SubstFolder);
InvokeScan();
}
@@ -90,6 +93,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
public long FileCacheSize { get; set; }
public long FileCacheDriveFree { get; set; }
public ConcurrentDictionary<string, int> HaltScanLocks { get; set; } = new(StringComparer.Ordinal);
private int LockCount = 0;
public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
public long TotalFiles { get; private set; }
public long TotalFilesStorage { get; private set; }
@@ -97,18 +101,22 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
public void HaltScan(string source)
{
HaltScanLocks.AddOrUpdate(source, 1, (k, v) => v + 1);
Interlocked.Increment(ref LockCount);
}
record WatcherChange(WatcherChangeTypes ChangeType, string? OldPath = null);
private readonly Dictionary<string, WatcherChange> _watcherChanges = new Dictionary<string, WatcherChange>(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, WatcherChange> _mareChanges = new Dictionary<string, WatcherChange>(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, WatcherChange> _substChanges = new Dictionary<string, WatcherChange>(StringComparer.OrdinalIgnoreCase);
public void StopMonitoring()
{
Logger.LogInformation("Stopping monitoring of Penumbra and Mare storage folders");
MareWatcher?.Dispose();
SubstWatcher?.Dispose();
PenumbraWatcher?.Dispose();
MareWatcher = null;
SubstWatcher = null;
PenumbraWatcher = null;
}
@@ -147,13 +155,53 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
MareWatcher.EnableRaisingEvents = true;
}
public void StartSubstWatcher(string? substPath)
{
SubstWatcher?.Dispose();
if (string.IsNullOrEmpty(substPath))
{
SubstWatcher = null;
Logger.LogWarning("Mare file path is not set, cannot start the FSW for Mare.");
return;
}
try
{
if (!Directory.Exists(substPath))
Directory.CreateDirectory(substPath);
}
catch
{
Logger.LogWarning("Could not create subst directory at {path}.", substPath);
return;
}
Logger.LogDebug("Initializing Subst FSW on {path}", substPath);
SubstWatcher = new()
{
Path = substPath,
InternalBufferSize = 8388608,
NotifyFilter = NotifyFilters.CreationTime
| NotifyFilters.LastWrite
| NotifyFilters.FileName
| NotifyFilters.DirectoryName
| NotifyFilters.Size,
Filter = "*.*",
IncludeSubdirectories = false,
};
SubstWatcher.Deleted += SubstWatcher_FileChanged;
SubstWatcher.Created += SubstWatcher_FileChanged;
SubstWatcher.EnableRaisingEvents = true;
}
private void MareWatcher_FileChanged(object sender, FileSystemEventArgs e)
{
Logger.LogTrace("Mare FSW: FileChanged: {change} => {path}", e.ChangeType, e.FullPath);
if (!AllowedFileExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return;
lock (_watcherChanges)
lock (_mareChanges)
{
_mareChanges[e.FullPath] = new(e.ChangeType);
}
@@ -161,6 +209,20 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
_ = MareWatcherExecution();
}
private void SubstWatcher_FileChanged(object sender, FileSystemEventArgs e)
{
Logger.LogTrace("Subst FSW: FileChanged: {change} => {path}", e.ChangeType, e.FullPath);
if (!AllowedFileExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return;
lock (_substChanges)
{
_substChanges[e.FullPath] = new(e.ChangeType);
}
_ = SubstWatcherExecution();
}
public void StartPenumbraWatcher(string? penumbraPath)
{
PenumbraWatcher?.Dispose();
@@ -247,8 +309,10 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
private CancellationTokenSource _penumbraFswCts = new();
private CancellationTokenSource _mareFswCts = new();
private CancellationTokenSource _substFswCts = new();
public FileSystemWatcher? PenumbraWatcher { get; private set; }
public FileSystemWatcher? MareWatcher { get; private set; }
public FileSystemWatcher? SubstWatcher { get; private set; }
private async Task MareWatcherExecution()
{
@@ -281,6 +345,67 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
HandleChanges(changes);
}
private async Task SubstWatcherExecution()
{
_substFswCts = _substFswCts.CancelRecreate();
var token = _substFswCts.Token;
var delay = TimeSpan.FromSeconds(5);
Dictionary<string, WatcherChange> changes;
lock (_substChanges)
changes = _substChanges.ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal);
try
{
do
{
await Task.Delay(delay, token).ConfigureAwait(false);
} while (HaltScanLocks.Any(f => f.Value > 0));
}
catch (TaskCanceledException)
{
return;
}
lock (_substChanges)
{
foreach (var key in changes.Keys)
{
_substChanges.Remove(key);
}
}
HandleChanges(changes);
}
public void ClearSubstStorage()
{
var substDir = _fileDbManager.SubstFolder;
var allSubstFiles = Directory.GetFiles(substDir, "*.*", SearchOption.TopDirectoryOnly)
.Where(f =>
{
var val = f.Split('\\')[^1];
return val.Length == 40 || (val.Split('.').FirstOrDefault()?.Length ?? 0) == 40
|| val.EndsWith(".tmp", StringComparison.OrdinalIgnoreCase);
});
if (SubstWatcher != null)
SubstWatcher.EnableRaisingEvents = false;
Dictionary<string, WatcherChange> changes = _substChanges.ToDictionary(t => t.Key, t => new WatcherChange(WatcherChangeTypes.Deleted, t.Key), StringComparer.Ordinal);
foreach (var file in allSubstFiles)
{
try
{
File.Delete(file);
}
catch { }
}
HandleChanges(changes);
if (SubstWatcher != null)
SubstWatcher.EnableRaisingEvents = true;
}
private void HandleChanges(Dictionary<string, WatcherChange> changes)
{
lock (_fileDbManager)
@@ -408,7 +533,9 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder);
}
var files = Directory.EnumerateFiles(_configService.Current.CacheFolder).Select(f => new FileInfo(f))
var files = Directory.EnumerateFiles(_configService.Current.CacheFolder)
.Concat(Directory.EnumerateFiles(_fileDbManager.SubstFolder))
.Select(f => new FileInfo(f))
.OrderBy(f => f.LastAccessTime).ToList();
FileCacheSize = files
.Sum(f =>
@@ -429,6 +556,8 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
if (FileCacheSize < maxCacheInBytes) return;
var substDir = _fileDbManager.SubstFolder;
var maxCacheBuffer = maxCacheInBytes * 0.05d;
while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer)
{
@@ -442,11 +571,16 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
public void ResetLocks()
{
HaltScanLocks.Clear();
LockCount = 0;
}
public void ResumeScan(string source)
{
HaltScanLocks.AddOrUpdate(source, 0, (k, v) => Math.Max(0, v - 1));
int lockCount = Interlocked.Decrement(ref LockCount);
if (lockCount == 0)
HaltScanLocks.Clear();
}
protected override void Dispose(bool disposing)
@@ -455,8 +589,10 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
_scanCancellationTokenSource?.Cancel();
PenumbraWatcher?.Dispose();
MareWatcher?.Dispose();
SubstWatcher?.Dispose();
_penumbraFswCts?.CancelDispose();
_mareFswCts?.CancelDispose();
_substFswCts?.CancelDispose();
_periodicCalculationTokenSource?.CancelDispose();
}
@@ -466,6 +602,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
var penumbraDir = _ipcManager.Penumbra.ModDirectory;
bool penDirExists = true;
bool cacheDirExists = true;
var substDir = _fileDbManager.SubstFolder;
if (string.IsNullOrEmpty(penumbraDir) || !Directory.Exists(penumbraDir))
{
penDirExists = false;
@@ -481,6 +618,16 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
return;
}
try
{
if (!Directory.Exists(substDir))
Directory.CreateDirectory(substDir);
}
catch
{
Logger.LogWarning("Could not create subst directory at {path}.", substDir);
}
var previousThreadPriority = Thread.CurrentThread.Priority;
Thread.CurrentThread.Priority = ThreadPriority.Lowest;
Logger.LogDebug("Getting files from {penumbra} and {storage}", penumbraDir, _configService.Current.CacheFolder);
@@ -509,6 +656,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
}
var allCacheFiles = Directory.GetFiles(_configService.Current.CacheFolder, "*.*", SearchOption.TopDirectoryOnly)
.Concat(Directory.GetFiles(substDir, "*.*", SearchOption.TopDirectoryOnly))
.AsParallel()
.Where(f =>
{
@@ -654,7 +802,13 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
try
{
var entry = _fileDbManager.CreateFileEntry(cachePath);
if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath);
if (entry == null)
{
if (cachePath.StartsWith(substDir, StringComparison.Ordinal))
_ = _fileDbManager.CreateSubstEntry(cachePath);
else
_ = _fileDbManager.CreateCacheEntry(cachePath);
}
}
catch (Exception ex)
{
@@ -678,6 +832,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
_configService.Current.InitialScanComplete = true;
_configService.Save();
StartMareWatcher(_configService.Current.CacheFolder);
StartSubstWatcher(_fileDbManager.SubstFolder);
StartPenumbraWatcher(penumbraDir);
}
}

1
MareSynchronos/FileCache/FileCacheEntity.cs
View File

@@ -17,6 +17,7 @@ public class FileCacheEntity
public string CsvEntry => $"{Hash}{FileCacheManager.CsvSplit}{PrefixedFilePath}{FileCacheManager.CsvSplit}{LastModifiedDateTicks}|{Size ?? -1}|{CompressedSize ?? -1}";
public string Hash { get; set; }
public bool IsCacheEntry => PrefixedFilePath.StartsWith(FileCacheManager.CachePrefix, StringComparison.OrdinalIgnoreCase);
public bool IsSubstEntry => PrefixedFilePath.StartsWith(FileCacheManager.SubstPrefix, StringComparison.OrdinalIgnoreCase);
public string LastModifiedDateTicks { get; set; }
public string PrefixedFilePath { get; init; }
public string ResolvedFilepath { get; private set; } = string.Empty;

57
MareSynchronos/FileCache/FileCacheManager.cs
View File

@@ -1,4 +1,5 @@
using K4os.Compression.LZ4.Streams;
using Dalamud.Utility;
using K4os.Compression.LZ4.Streams;
using MareSynchronos.Interop.Ipc;
using MareSynchronos.MareConfiguration;
using MareSynchronos.Services.Mediator;
@@ -16,6 +17,10 @@ public sealed class FileCacheManager : IHostedService
public const string CachePrefix = "{cache}";
public const string CsvSplit = "|";
public const string PenumbraPrefix = "{penumbra}";
public const string SubstPrefix = "{subst}";
public const string SubstPath = "subst";
public string CacheFolder => _configService.Current.CacheFolder;
public string SubstFolder => CacheFolder.IsNullOrEmpty() ? string.Empty : CacheFolder.ToLowerInvariant().TrimEnd('\\') + "\\" + SubstPath;
private readonly MareConfigService _configService;
private readonly MareMediator _mareMediator;
private readonly string _csvPath;
@@ -47,6 +52,19 @@ public sealed class FileCacheManager : IHostedService
return CreateFileCacheEntity(fi, prefixedPath);
}
public FileCacheEntity? CreateSubstEntry(string path)
{
FileInfo fi = new(path);
if (!fi.Exists) return null;
_logger.LogTrace("Creating substitute entry for {path}", path);
var fullName = fi.FullName.ToLowerInvariant();
if (!fullName.Contains(SubstFolder, StringComparison.Ordinal)) return null;
string prefixedPath = fullName.Replace(SubstFolder, SubstPrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
var fakeHash = Path.GetFileNameWithoutExtension(fi.FullName).ToUpperInvariant();
var result = CreateFileCacheEntity(fi, prefixedPath, fakeHash);
return result;
}
public FileCacheEntity? CreateFileEntry(string path)
{
FileInfo fi = new(path);
@@ -65,7 +83,7 @@ public sealed class FileCacheManager : IHostedService
List<FileCacheEntity> output = [];
if (_fileCaches.TryGetValue(hash, out var fileCacheEntities))
{
foreach (var fileCache in fileCacheEntities.Where(c => ignoreCacheEntries ? !c.IsCacheEntry : true).ToList())
foreach (var fileCache in fileCacheEntities.Where(c => ignoreCacheEntries ? (!c.IsCacheEntry && !c.IsSubstEntry) : true).ToList())
{
if (!validate) output.Add(fileCache);
else
@@ -89,6 +107,7 @@ public sealed class FileCacheManager : IHostedService
foreach (var fileCache in cacheEntries)
{
if (cancellationToken.IsCancellationRequested) break;
if (fileCache.IsSubstEntry) continue;
_logger.LogInformation("Validating {file}", fileCache.ResolvedFilepath);
@@ -139,6 +158,11 @@ public sealed class FileCacheManager : IHostedService
return Path.Combine(_configService.Current.CacheFolder, hash + "." + extension);
}
public string GetSubstFilePath(string hash, string extension)
{
return Path.Combine(SubstFolder, hash + "." + extension);
}
public async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken)
{
var fileCache = GetFileCacheByHash(fileHash)!;
@@ -151,14 +175,24 @@ public sealed class FileCacheManager : IHostedService
return (fileHash, ms.ToArray());
}
public FileCacheEntity? GetFileCacheByHash(string hash)
public FileCacheEntity? GetFileCacheByHash(string hash, bool preferSubst = false)
{
var caches = GetFileCachesByHash(hash);
if (preferSubst && caches.Subst != null)
return caches.Subst;
return caches.Penumbra ?? caches.Cache;
}
public (FileCacheEntity? Penumbra, FileCacheEntity? Cache, FileCacheEntity? Subst) GetFileCachesByHash(string hash)
{
(FileCacheEntity? Penumbra, FileCacheEntity? Cache, FileCacheEntity? Subst) result = (null, null, null);
if (_fileCaches.TryGetValue(hash, out var hashes))
{
var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix) ? 0 : 1).FirstOrDefault();
if (item != null) return GetValidatedFileCache(item);
result.Penumbra = hashes.Where(p => p.PrefixedFilePath.StartsWith(PenumbraPrefix, StringComparison.Ordinal)).Select(GetValidatedFileCache).FirstOrDefault();
result.Cache = hashes.Where(p => p.PrefixedFilePath.StartsWith(CachePrefix, StringComparison.Ordinal)).Select(GetValidatedFileCache).FirstOrDefault();
result.Subst = hashes.Where(p => p.PrefixedFilePath.StartsWith(SubstPrefix, StringComparison.Ordinal)).Select(GetValidatedFileCache).FirstOrDefault();
}
return null;
return result;
}
private FileCacheEntity? GetFileCacheByPath(string path)
@@ -187,6 +221,7 @@ public sealed class FileCacheManager : IHostedService
var cleanedPaths = paths.Distinct(StringComparer.OrdinalIgnoreCase).ToDictionary(p => p,
p => p.Replace("/", "\\", StringComparison.OrdinalIgnoreCase)
.Replace(_ipcManager.Penumbra.ModDirectory!, _ipcManager.Penumbra.ModDirectory!.EndsWith('\\') ? PenumbraPrefix + '\\' : PenumbraPrefix, StringComparison.OrdinalIgnoreCase)
.Replace(SubstFolder, SubstPrefix, StringComparison.OrdinalIgnoreCase)
.Replace(_configService.Current.CacheFolder, _configService.Current.CacheFolder.EndsWith('\\') ? CachePrefix + '\\' : CachePrefix, StringComparison.OrdinalIgnoreCase)
.Replace("\\\\", "\\", StringComparison.Ordinal),
StringComparer.OrdinalIgnoreCase);
@@ -207,9 +242,11 @@ public sealed class FileCacheManager : IHostedService
}
else
{
if (!entry.Value.Contains(CachePrefix, StringComparison.Ordinal))
if (entry.Value.StartsWith(PenumbraPrefix, StringComparison.Ordinal))
result.Add(entry.Key, CreateFileEntry(entry.Key));
else
else if (entry.Value.StartsWith(SubstPrefix, StringComparison.Ordinal))
result.Add(entry.Key, CreateSubstEntry(entry.Key));
else if (entry.Value.StartsWith(CachePrefix, StringComparison.Ordinal))
result.Add(entry.Key, CreateCacheEntry(entry.Key));
}
}
@@ -360,6 +397,10 @@ public sealed class FileCacheManager : IHostedService
{
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(PenumbraPrefix, _ipcManager.Penumbra.ModDirectory, StringComparison.Ordinal));
}
else if (fileCache.PrefixedFilePath.StartsWith(SubstPrefix, StringComparison.OrdinalIgnoreCase))
{
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(SubstPrefix, SubstFolder, StringComparison.Ordinal));
}
else if (fileCache.PrefixedFilePath.StartsWith(CachePrefix, StringComparison.OrdinalIgnoreCase))
{
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(CachePrefix, _configService.Current.CacheFolder, StringComparison.Ordinal));

5
MareSynchronos/MareConfiguration/Configurations/PlayerPerformanceConfig.cs
View File

@@ -1,4 +1,6 @@
namespace MareSynchronos.MareConfiguration.Configurations;
using MareSynchronos.MareConfiguration.Models;
namespace MareSynchronos.MareConfiguration.Configurations;
public class PlayerPerformanceConfig : IMareConfiguration
{
@@ -9,4 +11,5 @@ public class PlayerPerformanceConfig : IMareConfiguration
public int VRAMSizeAutoPauseThresholdMiB { get; set; } = 550;
public int TrisAutoPauseThresholdThousands { get; set; } = 375;
public bool IgnoreDirectPairs { get; set; } = true;
public TextureShrinkMode TextureShrinkMode { get; set; } = TextureShrinkMode.Default;
}

1
MareSynchronos/MareConfiguration/Configurations/XivDataStorageConfig.cs
View File

@@ -5,6 +5,7 @@ namespace MareSynchronos.MareConfiguration.Configurations;
public class XivDataStorageConfig : IMareConfiguration
{
public ConcurrentDictionary<string, long> TriangleDictionary { get; set; } = new(StringComparer.OrdinalIgnoreCase);
public ConcurrentDictionary<string, (uint Mip0Size, int MipCount, ushort Width, ushort Height)> TexDictionary { get; set; } = new(StringComparer.OrdinalIgnoreCase);
public ConcurrentDictionary<string, Dictionary<string, List<ushort>>> BonesDictionary { get; set; } = new(StringComparer.OrdinalIgnoreCase);
public int Version { get; set; } = 0;
}

10
MareSynchronos/MareConfiguration/Models/TextureShrinkMode.cs Normal file
View File

@@ -0,0 +1,10 @@
namespace MareSynchronos.MareConfiguration.Models;
public enum TextureShrinkMode
{
Never,
Default,
DefaultHiRes,
Always,
AlwaysHiRes
}

14
MareSynchronos/PlayerData/Handlers/PairHandler.cs
View File

@@ -13,6 +13,7 @@ using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using ObjectKind = MareSynchronos.API.Data.Enum.ObjectKind;
namespace MareSynchronos.PlayerData.Handlers;
@@ -489,6 +490,17 @@ public sealed class PairHandler : DisposableMediatorSubscriberBase
await Task.Delay(TimeSpan.FromSeconds(2), downloadToken).ConfigureAwait(false);
}
try
{
Mediator.Publish(new HaltScanMessage(nameof(PlayerPerformanceService.ShrinkTextures)));
if (await _playerPerformanceService.ShrinkTextures(this, charaData, downloadToken).ConfigureAwait(false))
_ = TryCalculateModdedDictionary(applicationBase, charaData, out moddedPaths, downloadToken);
}
finally
{
Mediator.Publish(new ResumeScanMessage(nameof(PlayerPerformanceService.ShrinkTextures)));
}
bool exceedsThreshold = !await _playerPerformanceService.CheckBothThresholds(this, charaData).ConfigureAwait(false);
if (exceedsThreshold)
@@ -753,7 +765,7 @@ public sealed class PairHandler : DisposableMediatorSubscriberBase
(item) =>
{
token.ThrowIfCancellationRequested();
var fileCache = _fileDbManager.GetFileCacheByHash(item.Hash);
var fileCache = _fileDbManager.GetFileCacheByHash(item.Hash, preferSubst: true);
if (fileCache != null)
{
if (string.IsNullOrEmpty(new FileInfo(fileCache.ResolvedFilepath).Extension))

2
MareSynchronos/Services/PairAnalyzer.cs
View File

@@ -131,7 +131,7 @@ public sealed class PairAnalyzer : DisposableMediatorSubscriberBase
var fileCacheEntries = _fileCacheManager.GetAllFileCachesByHash(fileEntry.Hash, ignoreCacheEntries: false, validate: false).ToList();
if (fileCacheEntries.Count == 0) continue;
var filePath = fileCacheEntries[0].ResolvedFilepath;
var filePath = fileCacheEntries[^1].ResolvedFilepath;
FileInfo fi = new(filePath);
string ext = "unk?";
try

162
MareSynchronos/Services/PlayerPerformanceService.cs
View File

@@ -1,3 +1,4 @@
using Lumina.Data;
using MareSynchronos.API.Data;
using MareSynchronos.FileCache;
using MareSynchronos.MareConfiguration;
@@ -8,6 +9,7 @@ using MareSynchronos.Services.ServerConfiguration;
using MareSynchronos.UI;
using MareSynchronos.WebAPI.Files.Models;
using Microsoft.Extensions.Logging;
using System.ComponentModel;
namespace MareSynchronos.Services;
@@ -56,13 +58,8 @@ public class PlayerPerformanceService : DisposableMediatorSubscriberBase
long triUsage = 0;
if (!charaData.FileReplacements.TryGetValue(API.Data.Enum.ObjectKind.Player, out List<FileReplacementData>? playerReplacements))
{
pair.LastAppliedDataTris = 0;
return true;
}
var moddedModelHashes = playerReplacements.Where(p => string.IsNullOrEmpty(p.FileSwapPath) && p.GamePaths.Any(g => g.EndsWith("mdl", StringComparison.OrdinalIgnoreCase)))
var moddedModelHashes = charaData.FileReplacements.SelectMany(k => k.Value)
.Where(p => string.IsNullOrEmpty(p.FileSwapPath) && p.GamePaths.Any(g => g.EndsWith("mdl", StringComparison.OrdinalIgnoreCase)))
.Select(p => p.Hash)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
@@ -107,20 +104,15 @@ public class PlayerPerformanceService : DisposableMediatorSubscriberBase
return true;
}
public bool ComputeAndAutoPauseOnVRAMUsageThresholds(PairHandler pairHandler, CharacterData charaData, List<DownloadFileTransfer> toDownloadFiles)
public bool ComputeAndAutoPauseOnVRAMUsageThresholds(PairHandler pairHandler, CharacterData charaData, List<DownloadFileTransfer> toDownloadFiles, bool affect = false)
{
var config = _playerPerformanceConfigService.Current;
var pair = pairHandler.Pair;
long vramUsage = 0;
if (!charaData.FileReplacements.TryGetValue(API.Data.Enum.ObjectKind.Player, out List<FileReplacementData>? playerReplacements))
{
pair.LastAppliedApproximateVRAMBytes = 0;
return true;
}
var moddedTextureHashes = playerReplacements.Where(p => string.IsNullOrEmpty(p.FileSwapPath) && p.GamePaths.Any(g => g.EndsWith(".tex", StringComparison.OrdinalIgnoreCase)))
var moddedTextureHashes = charaData.FileReplacements.SelectMany(k => k.Value)
.Where(p => string.IsNullOrEmpty(p.FileSwapPath) && p.GamePaths.Any(g => g.EndsWith(".tex", StringComparison.OrdinalIgnoreCase)))
.Select(p => p.Hash)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
@@ -136,7 +128,7 @@ public class PlayerPerformanceService : DisposableMediatorSubscriberBase
}
else
{
var fileEntry = _fileCacheManager.GetFileCacheByHash(hash);
var fileEntry = _fileCacheManager.GetFileCacheByHash(hash, preferSubst: true);
if (fileEntry == null) continue;
if (fileEntry.Size == null)
@@ -168,6 +160,9 @@ public class PlayerPerformanceService : DisposableMediatorSubscriberBase
if (vramUsage > vramUsageThreshold * 1024 * 1024)
{
if (!affect)
return false;
if (notify && !pair.IsApplicationBlocked)
{
_mediator.Publish(new NotificationMessage($"{pair.PlayerName} ({pair.UserData.AliasOrUID}) automatically blocked",
@@ -185,4 +180,139 @@ public class PlayerPerformanceService : DisposableMediatorSubscriberBase
return true;
}
public async Task<bool> ShrinkTextures(PairHandler pairHandler, CharacterData charaData, CancellationToken token)
{
var config = _playerPerformanceConfigService.Current;
var pair = pairHandler.Pair;
if (config.TextureShrinkMode == MareConfiguration.Models.TextureShrinkMode.Never)
return false;
// XXX: Temporary
if (config.TextureShrinkMode == MareConfiguration.Models.TextureShrinkMode.Default)
return false;
var moddedTextureHashes = charaData.FileReplacements.SelectMany(k => k.Value)
.Where(p => string.IsNullOrEmpty(p.FileSwapPath) && p.GamePaths.Any(g => g.EndsWith(".tex", StringComparison.OrdinalIgnoreCase)))
.Select(p => p.Hash)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
bool shrunken = false;
await Parallel.ForEachAsync(moddedTextureHashes,
token,
async (hash, token) => {
var fileEntry = _fileCacheManager.GetFileCacheByHash(hash, preferSubst: true);
if (fileEntry == null) return;
if (fileEntry.IsSubstEntry) return;
var texFormat = await _xivDataAnalyzer.GetTexFormatByHash(hash);
var filePath = fileEntry.ResolvedFilepath;
var tmpFilePath = _fileCacheManager.GetSubstFilePath(Guid.NewGuid().ToString(), "tmp");
var newFilePath = _fileCacheManager.GetSubstFilePath(hash, "tex");
var mipLevel = 0;
uint width = texFormat.Width;
uint height = texFormat.Height;
long offsetDelta = 0;
uint bitsPerPixel = texFormat.Format switch
{
0x1130 => 8, // L8
0x1131 => 8, // A8
0x1440 => 16, // A4R4G4B4
0x1441 => 16, // A1R5G5B5
0x1450 => 32, // A8R8G8B8
0x1451 => 32, // X8R8G8B8
0x2150 => 32, // R32F
0x2250 => 32, // G16R16F
0x2260 => 64, // R32G32F
0x2460 => 64, // A16B16G16R16F
0x2470 => 128, // A32B32G32R32F
0x3420 => 4, // DXT1
0x3430 => 8, // DXT3
0x3431 => 8, // DXT5
0x4140 => 16, // D16
0x4250 => 32, // D24S8
0x6120 => 4, // BC4
0x6230 => 8, // BC5
0x6432 => 8, // BC7
_ => 0
};
uint maxSize = (bitsPerPixel <= 8) ? (2048U * 2048U) : (1024U * 1024U);
while (width * height > maxSize && mipLevel < texFormat.MipCount - 1)
{
offsetDelta += width * height * bitsPerPixel / 8;
mipLevel++;
width /= 2;
height /= 2;
}
if (offsetDelta == 0)
return;
_logger.LogDebug("Shrinking {hash} from from {a}x{b} to {c}x{d}",
hash, texFormat.Width, texFormat.Height, width, height);
try
{
var inFile = new FileStream(filePath, FileMode.Open, FileAccess.Read);
using var reader = new BinaryReader(inFile);
var header = reader.ReadBytes(80);
reader.BaseStream.Position = 14;
byte mipByte = reader.ReadByte();
byte mipCount = (byte)(mipByte & 0x7F);
var outFile = new FileStream(tmpFilePath, FileMode.Create, FileAccess.Write, FileShare.None);
using var writer = new BinaryWriter(outFile);
writer.Write(header);
// Update width/height
writer.BaseStream.Position = 8;
writer.Write((ushort)width);
writer.Write((ushort)height);
// Update the mip count
writer.BaseStream.Position = 14;
writer.Write((ushort)((mipByte & 0x80) | (mipCount - mipLevel)));
// Reset all of the LoD mips
writer.BaseStream.Position = 16;
for (int i = 0; i < 3; ++i)
writer.Write((uint)0);
// Reset all of the mip offsets
// (This data is garbage in a lot of modded textures, so its hard to fix it up correctly)
writer.BaseStream.Position = 28;
for (int i = 0; i < 13; ++i)
writer.Write((uint)80);
// Write the texture data shifted
outFile.Position = 80;
inFile.Position = 80 + offsetDelta;
await inFile.CopyToAsync(outFile, 81920, token);
reader.Dispose();
writer.Dispose();
File.Move(tmpFilePath, newFilePath);
_fileCacheManager.CreateSubstEntry(newFilePath);
shrunken = true;
}
catch (Exception e)
{
_logger.LogWarning(e, "Failed to shrink texture {hash}", hash);
if (File.Exists(tmpFilePath))
File.Delete(tmpFilePath);
}
}
);
return shrunken;
}
}

48
MareSynchronos/Services/XivDataAnalyzer.cs
View File

@@ -3,12 +3,14 @@ using FFXIVClientStructs.FFXIV.Client.Graphics.Scene;
using FFXIVClientStructs.Havok.Animation;
using FFXIVClientStructs.Havok.Common.Base.Types;
using FFXIVClientStructs.Havok.Common.Serialize.Util;
using Lumina.Data;
using MareSynchronos.FileCache;
using MareSynchronos.Interop.GameModel;
using MareSynchronos.MareConfiguration;
using MareSynchronos.PlayerData.Handlers;
using Microsoft.Extensions.Logging;
using System.Runtime.InteropServices;
using System.Text;
namespace MareSynchronos.Services;
@@ -18,6 +20,7 @@ public sealed class XivDataAnalyzer
private readonly FileCacheManager _fileCacheManager;
private readonly XivDataStorageService _configService;
private readonly List<string> _failedCalculatedTris = [];
private readonly List<string> _failedCalculatedTex = [];
public XivDataAnalyzer(ILogger<XivDataAnalyzer> logger, FileCacheManager fileCacheManager,
XivDataStorageService configService)
@@ -67,7 +70,7 @@ public sealed class XivDataAnalyzer
{
if (_configService.Current.BonesDictionary.TryGetValue(hash, out var bones)) return bones;
var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash);
var cacheEntity = _fileCacheManager.GetFileCacheByHash(hash, preferSubst: true);
if (cacheEntity == null) return null;
using BinaryReader reader = new BinaryReader(File.Open(cacheEntity.ResolvedFilepath, FileMode.Open, FileAccess.Read, FileShare.Read));
@@ -158,7 +161,7 @@ public sealed class XivDataAnalyzer
if (_failedCalculatedTris.Contains(hash, StringComparer.Ordinal))
return 0;
var path = _fileCacheManager.GetFileCacheByHash(hash);
var path = _fileCacheManager.GetFileCacheByHash(hash, preferSubst: true);
if (path == null || !path.ResolvedFilepath.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase))
return 0;
@@ -211,4 +214,45 @@ public sealed class XivDataAnalyzer
return 0;
}
}
public async Task<(uint Format, int MipCount, ushort Width, ushort Height)> GetTexFormatByHash(string hash)
{
if (_configService.Current.TexDictionary.TryGetValue(hash, out var cachedTex) && cachedTex.Mip0Size > 0)
return cachedTex;
if (_failedCalculatedTex.Contains(hash, StringComparer.Ordinal))
return default;
var path = _fileCacheManager.GetFileCacheByHash(hash);
if (path == null || !path.ResolvedFilepath.EndsWith(".tex", StringComparison.OrdinalIgnoreCase))
return default;
var filePath = path.ResolvedFilepath;
try
{
_logger.LogDebug("Detected Texture File {path}, reading header", filePath);
using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read);
using var r = new LuminaBinaryReader(stream);
var texHeader = r.ReadStructure<Lumina.Data.Files.TexFile.TexHeader>();
if (texHeader.Format == 0 || texHeader.MipCount == 0 || texHeader.ArraySize != 0 || texHeader.MipCount > 13)
{
_failedCalculatedTex.Add(hash);
_configService.Current.TexDictionary[hash] = default;
_configService.Save();
return default;
}
return ((uint)texHeader.Format, texHeader.MipCount, texHeader.Width, texHeader.Height);
}
catch (Exception e)
{
_failedCalculatedTex.Add(hash);
_configService.Current.TriangleDictionary[hash] = 0;
_configService.Save();
_logger.LogWarning(e, "Could not parse file {file}", filePath);
return default;
}
}
}

29
MareSynchronos/UI/SettingsUi.cs
View File

@@ -1242,6 +1242,35 @@ public class SettingsUi : WindowMediatorSubscriberBase
bool recalculatePerformance = false;
string? recalculatePerformanceUID = null;
_uiShared.BigText("Global Configuration");
bool alwaysShrinkTextures = _playerPerformanceConfigService.Current.TextureShrinkMode == TextureShrinkMode.Always;
if (ImGui.Checkbox("Shrink downloaded textures", ref alwaysShrinkTextures))
{
if (alwaysShrinkTextures)
_playerPerformanceConfigService.Current.TextureShrinkMode = TextureShrinkMode.Always;
else
_playerPerformanceConfigService.Current.TextureShrinkMode = TextureShrinkMode.Never;
_playerPerformanceConfigService.Save();
recalculatePerformance = true;
_cacheMonitor.ClearSubstStorage();
}
_uiShared.DrawHelpText("Automatically shrinks texture resolution of synced players to reduce VRAM utilization." + UiSharedService.TooltipSeparator
+ "Texture Size Limit (DXT/BC5/BC7 Compressed): 2048x2048" + Environment.NewLine
+ "Texture Size Limit (A8R8G8B8 Uncompressed): 1024x1024" + UiSharedService.TooltipSeparator
+ "Enable to reduce lag in large crowds." + Environment.NewLine
+ "Disable this for higher quality during GPose.");
var totalVramBytes = _pairManager.GetOnlineUserPairs().Where(p => p.IsVisible && p.LastAppliedApproximateVRAMBytes > 0).Sum(p => p.LastAppliedApproximateVRAMBytes);
ImGui.TextUnformatted("Current VRAM utilization by all nearby players:");
ImGui.SameLine();
using (ImRaii.PushColor(ImGuiCol.Text, ImGuiColors.HealerGreen, totalVramBytes < 2.0 * 1024.0 * 1024.0 * 1024.0))
using (ImRaii.PushColor(ImGuiCol.Text, ImGuiColors.DalamudYellow, totalVramBytes >= 4.0 * 1024.0 * 1024.0 * 1024.0))
using (ImRaii.PushColor(ImGuiCol.Text, ImGuiColors.DalamudRed, totalVramBytes >= 6.0 * 1024.0 * 1024.0 * 1024.0))
ImGui.TextUnformatted($"{totalVramBytes / 1024.0 / 1024.0 / 1024.0:0.00} GiB");
ImGui.Separator();
_uiShared.BigText("Individual Limits");
bool autoPause = _playerPerformanceConfigService.Current.AutoPausePlayersExceedingThresholds;
if (ImGui.Checkbox("Automatically block players exceeding thresholds", ref autoPause))