Add texture shrinking feature

2025-02-23 12:18:34 +00:00
parent eaaded1ed5
commit 99eecbdc09
11 changed files with 458 additions and 32 deletions
--- a/MareSynchronos/FileCache/CacheMonitor.cs
+++ b/MareSynchronos/FileCache/CacheMonitor.cs
@@ -36,6 +36,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
        {
            StartPenumbraWatcher(_ipcManager.Penumbra.ModDirectory);
            StartMareWatcher(configService.Current.CacheFolder);
+            StartSubstWatcher(_fileDbManager.SubstFolder);
            InvokeScan();
        });
        Mediator.Subscribe<HaltScanMessage>(this, (msg) => HaltScan(msg.Source));
@@ -43,6 +44,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
        Mediator.Subscribe<DalamudLoginMessage>(this, (_) =>
        {
            StartMareWatcher(configService.Current.CacheFolder);
+            StartSubstWatcher(_fileDbManager.SubstFolder);
            StartPenumbraWatcher(_ipcManager.Penumbra.ModDirectory);
            InvokeScan();
        });
@@ -58,6 +60,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
        if (configService.Current.HasValidSetup())
        {
            StartMareWatcher(configService.Current.CacheFolder);
+            StartSubstWatcher(_fileDbManager.SubstFolder);
            InvokeScan();
        }

@@ -90,6 +93,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
    public long FileCacheSize { get; set; }
    public long FileCacheDriveFree { get; set; }
    public ConcurrentDictionary<string, int> HaltScanLocks { get; set; } = new(StringComparer.Ordinal);
+    private int LockCount = 0;
    public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
    public long TotalFiles { get; private set; }
    public long TotalFilesStorage { get; private set; }
@@ -97,18 +101,22 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
    public void HaltScan(string source)
    {
        HaltScanLocks.AddOrUpdate(source, 1, (k, v) => v + 1);
+        Interlocked.Increment(ref LockCount);
    }

    record WatcherChange(WatcherChangeTypes ChangeType, string? OldPath = null);
    private readonly Dictionary<string, WatcherChange> _watcherChanges = new Dictionary<string, WatcherChange>(StringComparer.OrdinalIgnoreCase);
    private readonly Dictionary<string, WatcherChange> _mareChanges = new Dictionary<string, WatcherChange>(StringComparer.OrdinalIgnoreCase);
+    private readonly Dictionary<string, WatcherChange> _substChanges = new Dictionary<string, WatcherChange>(StringComparer.OrdinalIgnoreCase);

    public void StopMonitoring()
    {
        Logger.LogInformation("Stopping monitoring of Penumbra and Mare storage folders");
        MareWatcher?.Dispose();
+        SubstWatcher?.Dispose();
        PenumbraWatcher?.Dispose();
        MareWatcher = null;
+        SubstWatcher = null;
        PenumbraWatcher = null;
    }

@@ -147,13 +155,53 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
        MareWatcher.EnableRaisingEvents = true;
    }

+    public void StartSubstWatcher(string? substPath)
+    {
+        SubstWatcher?.Dispose();
+        if (string.IsNullOrEmpty(substPath))
+        {
+            SubstWatcher = null;
+            Logger.LogWarning("Mare file path is not set, cannot start the FSW for Mare.");
+            return;
+        }
+
+        try
+        {
+            if (!Directory.Exists(substPath))
+                Directory.CreateDirectory(substPath);
+        }
+        catch
+        {
+            Logger.LogWarning("Could not create subst directory at {path}.", substPath);
+            return;
+        }
+
+        Logger.LogDebug("Initializing Subst FSW on {path}", substPath);
+        SubstWatcher = new()
+        {
+            Path = substPath,
+            InternalBufferSize = 8388608,
+            NotifyFilter = NotifyFilters.CreationTime
+                | NotifyFilters.LastWrite
+                | NotifyFilters.FileName
+                | NotifyFilters.DirectoryName
+                | NotifyFilters.Size,
+            Filter = "*.*",
+            IncludeSubdirectories = false,
+        };
+
+        SubstWatcher.Deleted += SubstWatcher_FileChanged;
+        SubstWatcher.Created += SubstWatcher_FileChanged;
+        SubstWatcher.EnableRaisingEvents = true;
+    }
+
    private void MareWatcher_FileChanged(object sender, FileSystemEventArgs e)
    {
        Logger.LogTrace("Mare FSW: FileChanged: {change} => {path}", e.ChangeType, e.FullPath);

        if (!AllowedFileExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return;

-        lock (_watcherChanges)
+        lock (_mareChanges)
        {
            _mareChanges[e.FullPath] = new(e.ChangeType);
        }
@@ -161,6 +209,20 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
        _ = MareWatcherExecution();
    }

+    private void SubstWatcher_FileChanged(object sender, FileSystemEventArgs e)
+    {
+        Logger.LogTrace("Subst FSW: FileChanged: {change} => {path}", e.ChangeType, e.FullPath);
+
+        if (!AllowedFileExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return;
+
+        lock (_substChanges)
+        {
+            _substChanges[e.FullPath] = new(e.ChangeType);
+        }
+
+        _ = SubstWatcherExecution();
+    }
+
    public void StartPenumbraWatcher(string? penumbraPath)
    {
        PenumbraWatcher?.Dispose();
@@ -247,8 +309,10 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase

    private CancellationTokenSource _penumbraFswCts = new();
    private CancellationTokenSource _mareFswCts = new();
+    private CancellationTokenSource _substFswCts = new();
    public FileSystemWatcher? PenumbraWatcher { get; private set; }
    public FileSystemWatcher? MareWatcher { get; private set; }
+    public FileSystemWatcher? SubstWatcher { get; private set; }

    private async Task MareWatcherExecution()
    {
@@ -281,6 +345,67 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
        HandleChanges(changes);
    }

+    private async Task SubstWatcherExecution()
+    {
+        _substFswCts = _substFswCts.CancelRecreate();
+        var token = _substFswCts.Token;
+        var delay = TimeSpan.FromSeconds(5);
+        Dictionary<string, WatcherChange> changes;
+        lock (_substChanges)
+            changes = _substChanges.ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal);
+        try
+        {
+            do
+            {
+                await Task.Delay(delay, token).ConfigureAwait(false);
+            } while (HaltScanLocks.Any(f => f.Value > 0));
+        }
+        catch (TaskCanceledException)
+        {
+            return;
+        }
+
+        lock (_substChanges)
+        {
+            foreach (var key in changes.Keys)
+            {
+                _substChanges.Remove(key);
+            }
+        }
+
+        HandleChanges(changes);
+    }
+
+    public void ClearSubstStorage()
+    {
+        var substDir = _fileDbManager.SubstFolder;
+        var allSubstFiles = Directory.GetFiles(substDir, "*.*", SearchOption.TopDirectoryOnly)
+                                .Where(f =>
+                                {
+                                    var val = f.Split('\\')[^1];
+                                    return val.Length == 40 || (val.Split('.').FirstOrDefault()?.Length ?? 0) == 40
+                                        || val.EndsWith(".tmp", StringComparison.OrdinalIgnoreCase);
+                                });
+        if (SubstWatcher != null)
+            SubstWatcher.EnableRaisingEvents = false;
+
+        Dictionary<string, WatcherChange> changes = _substChanges.ToDictionary(t => t.Key, t => new WatcherChange(WatcherChangeTypes.Deleted, t.Key), StringComparer.Ordinal);
+
+        foreach (var file in allSubstFiles)
+        {
+            try
+            {
+                File.Delete(file);
+            }
+            catch { }
+        }
+
+        HandleChanges(changes);
+
+        if (SubstWatcher != null)
+            SubstWatcher.EnableRaisingEvents = true;
+    }
+
    private void HandleChanges(Dictionary<string, WatcherChange> changes)
    {
        lock (_fileDbManager)
@@ -408,7 +533,9 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
            Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder);
        }

-        var files = Directory.EnumerateFiles(_configService.Current.CacheFolder).Select(f => new FileInfo(f))
+        var files = Directory.EnumerateFiles(_configService.Current.CacheFolder)
+            .Concat(Directory.EnumerateFiles(_fileDbManager.SubstFolder))
+            .Select(f => new FileInfo(f))
            .OrderBy(f => f.LastAccessTime).ToList();
        FileCacheSize = files
            .Sum(f =>
@@ -429,6 +556,8 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase

        if (FileCacheSize < maxCacheInBytes) return;

+        var substDir = _fileDbManager.SubstFolder;
+
        var maxCacheBuffer = maxCacheInBytes * 0.05d;
        while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer)
        {
@@ -442,11 +571,16 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
    public void ResetLocks()
    {
        HaltScanLocks.Clear();
+        LockCount = 0;
    }

    public void ResumeScan(string source)
    {
        HaltScanLocks.AddOrUpdate(source, 0, (k, v) => Math.Max(0, v - 1));
+        int lockCount = Interlocked.Decrement(ref LockCount);
+
+        if (lockCount == 0)
+            HaltScanLocks.Clear();
    }

    protected override void Dispose(bool disposing)
@@ -455,8 +589,10 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
        _scanCancellationTokenSource?.Cancel();
        PenumbraWatcher?.Dispose();
        MareWatcher?.Dispose();
+        SubstWatcher?.Dispose();
        _penumbraFswCts?.CancelDispose();
        _mareFswCts?.CancelDispose();
+        _substFswCts?.CancelDispose();
        _periodicCalculationTokenSource?.CancelDispose();
    }

@@ -466,6 +602,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
        var penumbraDir = _ipcManager.Penumbra.ModDirectory;
        bool penDirExists = true;
        bool cacheDirExists = true;
+        var substDir = _fileDbManager.SubstFolder;
        if (string.IsNullOrEmpty(penumbraDir) || !Directory.Exists(penumbraDir))
        {
            penDirExists = false;
@@ -481,6 +618,16 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
            return;
        }

+        try
+        {
+            if (!Directory.Exists(substDir))
+                Directory.CreateDirectory(substDir);
+        }
+        catch
+        {
+            Logger.LogWarning("Could not create subst directory at {path}.", substDir);
+        }
+
        var previousThreadPriority = Thread.CurrentThread.Priority;
        Thread.CurrentThread.Priority = ThreadPriority.Lowest;
        Logger.LogDebug("Getting files from {penumbra} and {storage}", penumbraDir, _configService.Current.CacheFolder);
@@ -509,6 +656,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
        }

        var allCacheFiles = Directory.GetFiles(_configService.Current.CacheFolder, "*.*", SearchOption.TopDirectoryOnly)
+                                .Concat(Directory.GetFiles(substDir, "*.*", SearchOption.TopDirectoryOnly))
                                .AsParallel()
                                .Where(f =>
                                {
@@ -654,7 +802,13 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
                    try
                    {
                        var entry = _fileDbManager.CreateFileEntry(cachePath);
-                        if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath);
+                        if (entry == null)
+                        {
+                            if (cachePath.StartsWith(substDir, StringComparison.Ordinal))
+                                _ = _fileDbManager.CreateSubstEntry(cachePath);
+                            else
+                                _ = _fileDbManager.CreateCacheEntry(cachePath);
+                        }
                    }
                    catch (Exception ex)
                    {
@@ -678,6 +832,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
            _configService.Current.InitialScanComplete = true;
            _configService.Save();
            StartMareWatcher(_configService.Current.CacheFolder);
+            StartSubstWatcher(_fileDbManager.SubstFolder);
            StartPenumbraWatcher(penumbraDir);
        }
    }
--- a/MareSynchronos/FileCache/FileCacheEntity.cs
+++ b/MareSynchronos/FileCache/FileCacheEntity.cs
@@ -17,6 +17,7 @@ public class FileCacheEntity
    public string CsvEntry => $"{Hash}{FileCacheManager.CsvSplit}{PrefixedFilePath}{FileCacheManager.CsvSplit}{LastModifiedDateTicks}|{Size ?? -1}|{CompressedSize ?? -1}";
    public string Hash { get; set; }
    public bool IsCacheEntry => PrefixedFilePath.StartsWith(FileCacheManager.CachePrefix, StringComparison.OrdinalIgnoreCase);
+    public bool IsSubstEntry => PrefixedFilePath.StartsWith(FileCacheManager.SubstPrefix, StringComparison.OrdinalIgnoreCase);
    public string LastModifiedDateTicks { get; set; }
    public string PrefixedFilePath { get; init; }
    public string ResolvedFilepath { get; private set; } = string.Empty;
--- a/MareSynchronos/FileCache/FileCacheManager.cs
+++ b/MareSynchronos/FileCache/FileCacheManager.cs
@@ -1,4 +1,5 @@
-using K4os.Compression.LZ4.Streams;
+using Dalamud.Utility;
+using K4os.Compression.LZ4.Streams;
 using MareSynchronos.Interop.Ipc;
 using MareSynchronos.MareConfiguration;
 using MareSynchronos.Services.Mediator;
@@ -16,6 +17,10 @@ public sealed class FileCacheManager : IHostedService
    public const string CachePrefix = "{cache}";
    public const string CsvSplit = "|";
    public const string PenumbraPrefix = "{penumbra}";
+    public const string SubstPrefix = "{subst}";
+    public const string SubstPath = "subst";
+    public string CacheFolder => _configService.Current.CacheFolder;
+    public string SubstFolder => CacheFolder.IsNullOrEmpty() ? string.Empty : CacheFolder.ToLowerInvariant().TrimEnd('\\') + "\\" + SubstPath;
    private readonly MareConfigService _configService;
    private readonly MareMediator _mareMediator;
    private readonly string _csvPath;
@@ -47,6 +52,19 @@ public sealed class FileCacheManager : IHostedService
        return CreateFileCacheEntity(fi, prefixedPath);
    }

+    public FileCacheEntity? CreateSubstEntry(string path)
+    {
+        FileInfo fi = new(path);
+        if (!fi.Exists) return null;
+        _logger.LogTrace("Creating substitute entry for {path}", path);
+        var fullName = fi.FullName.ToLowerInvariant();
+        if (!fullName.Contains(SubstFolder, StringComparison.Ordinal)) return null;
+        string prefixedPath = fullName.Replace(SubstFolder, SubstPrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
+        var fakeHash = Path.GetFileNameWithoutExtension(fi.FullName).ToUpperInvariant();
+        var result = CreateFileCacheEntity(fi, prefixedPath, fakeHash);
+        return result;
+    }
+
    public FileCacheEntity? CreateFileEntry(string path)
    {
        FileInfo fi = new(path);
@@ -65,7 +83,7 @@ public sealed class FileCacheManager : IHostedService
        List<FileCacheEntity> output = [];
        if (_fileCaches.TryGetValue(hash, out var fileCacheEntities))
        {
-            foreach (var fileCache in fileCacheEntities.Where(c => ignoreCacheEntries ? !c.IsCacheEntry : true).ToList())
+            foreach (var fileCache in fileCacheEntities.Where(c => ignoreCacheEntries ? (!c.IsCacheEntry && !c.IsSubstEntry) : true).ToList())
            {
                if (!validate) output.Add(fileCache);
                else
@@ -89,6 +107,7 @@ public sealed class FileCacheManager : IHostedService
        foreach (var fileCache in cacheEntries)
        {
            if (cancellationToken.IsCancellationRequested) break;
+            if (fileCache.IsSubstEntry) continue;

            _logger.LogInformation("Validating {file}", fileCache.ResolvedFilepath);

@@ -139,6 +158,11 @@ public sealed class FileCacheManager : IHostedService
        return Path.Combine(_configService.Current.CacheFolder, hash + "." + extension);
    }

+    public string GetSubstFilePath(string hash, string extension)
+    {
+        return Path.Combine(SubstFolder, hash + "." + extension);
+    }
+
    public async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken)
    {
        var fileCache = GetFileCacheByHash(fileHash)!;
@@ -151,14 +175,24 @@ public sealed class FileCacheManager : IHostedService
        return (fileHash, ms.ToArray());
    }

-    public FileCacheEntity? GetFileCacheByHash(string hash)
+    public FileCacheEntity? GetFileCacheByHash(string hash, bool preferSubst = false)
    {
+        var caches = GetFileCachesByHash(hash);
+        if (preferSubst && caches.Subst != null)
+            return caches.Subst;
+        return caches.Penumbra ?? caches.Cache;
+    }
+
+    public (FileCacheEntity? Penumbra, FileCacheEntity? Cache, FileCacheEntity? Subst) GetFileCachesByHash(string hash)
+    {
+        (FileCacheEntity? Penumbra, FileCacheEntity? Cache, FileCacheEntity? Subst) result = (null, null, null);
        if (_fileCaches.TryGetValue(hash, out var hashes))
        {
-            var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix) ? 0 : 1).FirstOrDefault();
-            if (item != null) return GetValidatedFileCache(item);
+            result.Penumbra = hashes.Where(p => p.PrefixedFilePath.StartsWith(PenumbraPrefix, StringComparison.Ordinal)).Select(GetValidatedFileCache).FirstOrDefault();
+            result.Cache = hashes.Where(p => p.PrefixedFilePath.StartsWith(CachePrefix, StringComparison.Ordinal)).Select(GetValidatedFileCache).FirstOrDefault();
+            result.Subst = hashes.Where(p => p.PrefixedFilePath.StartsWith(SubstPrefix, StringComparison.Ordinal)).Select(GetValidatedFileCache).FirstOrDefault();
        }
-        return null;
+        return result;
    }

    private FileCacheEntity? GetFileCacheByPath(string path)
@@ -187,6 +221,7 @@ public sealed class FileCacheManager : IHostedService
            var cleanedPaths = paths.Distinct(StringComparer.OrdinalIgnoreCase).ToDictionary(p => p,
                p => p.Replace("/", "\\", StringComparison.OrdinalIgnoreCase)
                    .Replace(_ipcManager.Penumbra.ModDirectory!, _ipcManager.Penumbra.ModDirectory!.EndsWith('\\') ? PenumbraPrefix + '\\' : PenumbraPrefix, StringComparison.OrdinalIgnoreCase)
+                    .Replace(SubstFolder, SubstPrefix, StringComparison.OrdinalIgnoreCase)
                    .Replace(_configService.Current.CacheFolder, _configService.Current.CacheFolder.EndsWith('\\') ? CachePrefix + '\\' : CachePrefix, StringComparison.OrdinalIgnoreCase)
                    .Replace("\\\\", "\\", StringComparison.Ordinal),
                StringComparer.OrdinalIgnoreCase);
@@ -207,9 +242,11 @@ public sealed class FileCacheManager : IHostedService
                }
                else
                {
-                    if (!entry.Value.Contains(CachePrefix, StringComparison.Ordinal))
+                    if (entry.Value.StartsWith(PenumbraPrefix, StringComparison.Ordinal))
                        result.Add(entry.Key, CreateFileEntry(entry.Key));
-                    else
+                    else if (entry.Value.StartsWith(SubstPrefix, StringComparison.Ordinal))
+                        result.Add(entry.Key, CreateSubstEntry(entry.Key));
+                    else if (entry.Value.StartsWith(CachePrefix, StringComparison.Ordinal))
                        result.Add(entry.Key, CreateCacheEntry(entry.Key));
                }
            }
@@ -360,6 +397,10 @@ public sealed class FileCacheManager : IHostedService
        {
            fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(PenumbraPrefix, _ipcManager.Penumbra.ModDirectory, StringComparison.Ordinal));
        }
+        else if (fileCache.PrefixedFilePath.StartsWith(SubstPrefix, StringComparison.OrdinalIgnoreCase))
+        {
+            fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(SubstPrefix, SubstFolder, StringComparison.Ordinal));
+        }
        else if (fileCache.PrefixedFilePath.StartsWith(CachePrefix, StringComparison.OrdinalIgnoreCase))
        {
            fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(CachePrefix, _configService.Current.CacheFolder, StringComparison.Ordinal));