Mare 0.9 (#65)

* add jwt expiry * start of 0.9 api impl * some stuff idk * some more impl * some cleanup * remove grouppair, add configuration, rework some pair drawing stuff * do some stuff * rework some ui * I don't even know anymore * add cancellationtoken * token bla * ui fixes etc * probably individual adding/removing now working fully as expected * add working report popup * I guess it's more syncshell shit or so * popup shit idk * work out most of the syncshell bullshit I guess * delete some old crap * are we actually getting closer to the end * update pair info stuff * more fixes/adjustments, idk * refactor some things * some rework * some more cleanup * cleanup * make menu buttons w i d e * better icon text buttons * add all syncshell folder and ordering fixes --------- Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>
2023-10-17 21:36:44 +02:00
parent f15b8f6bbd
commit 14575a4a6b
111 changed files with 3456 additions and 3174 deletions
--- a/MareSynchronos/FileCache/FileCacheEntity.cs
+++ b/MareSynchronos/FileCache/FileCacheEntity.cs
@@ -13,14 +13,14 @@ public class FileCacheEntity
        LastModifiedDateTicks = lastModifiedDateTicks;
    }

-    public bool IsCacheEntry => PrefixedFilePath.StartsWith(FileCacheManager.CachePrefix, StringComparison.OrdinalIgnoreCase);
+    public long? CompressedSize { get; set; }
    public string CsvEntry => $"{Hash}{FileCacheManager.CsvSplit}{PrefixedFilePath}{FileCacheManager.CsvSplit}{LastModifiedDateTicks}|{Size ?? -1}|{CompressedSize ?? -1}";
    public string Hash { get; set; }
+    public bool IsCacheEntry => PrefixedFilePath.StartsWith(FileCacheManager.CachePrefix, StringComparison.OrdinalIgnoreCase);
    public string LastModifiedDateTicks { get; set; }
    public string PrefixedFilePath { get; init; }
    public string ResolvedFilepath { get; private set; } = string.Empty;
    public long? Size { get; set; }
-    public long? CompressedSize { get; set; }

    public void SetResolvedFilePath(string filePath)
    {
--- a/MareSynchronos/FileCache/FileCacheManager.cs
+++ b/MareSynchronos/FileCache/FileCacheManager.cs
@@ -11,8 +11,8 @@ namespace MareSynchronos.FileCache;

 public sealed class FileCacheManager : IDisposable
 {
-    public const string CsvSplit = "|";
    public const string CachePrefix = "{cache}";
+    public const string CsvSplit = "|";
    public const string PenumbraPrefix = "{penumbra}";
    private readonly MareConfigService _configService;
    private readonly string _csvPath;
@@ -55,7 +55,7 @@ public sealed class FileCacheManager : IDisposable
        if (File.Exists(_csvPath))
        {
            bool success = false;
-            string[] entries = Array.Empty<string>();
+            string[] entries = [];
            int attempts = 0;
            while (!success && attempts < 10)
            {
@@ -94,7 +94,7 @@ public sealed class FileCacheManager : IDisposable
                        continue;
                    }

-                    processedFiles.Add(path, true);
+                    processedFiles.Add(path, value: true);

                    long size = -1;
                    long compressed = -1;
@@ -157,11 +157,33 @@ public sealed class FileCacheManager : IDisposable

    public List<FileCacheEntity> GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v).ToList();

+    public List<FileCacheEntity> GetAllFileCachesByHash(string hash)
+    {
+        List<FileCacheEntity> output = [];
+        if (_fileCaches.TryGetValue(hash, out var fileCacheEntities))
+        {
+            foreach (var filecache in fileCacheEntities.ToList())
+            {
+                var validated = GetValidatedFileCache(filecache);
+                if (validated != null) output.Add(validated);
+            }
+        }
+
+        return output;
+    }
+
    public string GetCacheFilePath(string hash, string extension)
    {
        return Path.Combine(_configService.Current.CacheFolder, hash + "." + extension);
    }

+    public async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken)
+    {
+        var fileCache = GetFileCacheByHash(fileHash)!.ResolvedFilepath;
+        return (fileHash, LZ4Codec.WrapHC(await File.ReadAllBytesAsync(fileCache, uploadToken).ConfigureAwait(false), 0,
+            (int)new FileInfo(fileCache).Length));
+    }
+
    public FileCacheEntity? GetFileCacheByHash(string hash)
    {
        if (_fileCaches.TryGetValue(hash, out var hashes))
@@ -172,19 +194,20 @@ public sealed class FileCacheManager : IDisposable
        return null;
    }

-    public List<FileCacheEntity> GetAllFileCachesByHash(string hash)
+    public FileCacheEntity? GetFileCacheByPath(string path)
    {
-        List<FileCacheEntity> output = new();
-        if (_fileCaches.TryGetValue(hash, out var fileCacheEntities))
+        var cleanedPath = path.Replace("/", "\\", StringComparison.OrdinalIgnoreCase).ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), "", StringComparison.OrdinalIgnoreCase);
+        var entry = _fileCaches.SelectMany(v => v.Value).FirstOrDefault(f => f.ResolvedFilepath.EndsWith(cleanedPath, StringComparison.OrdinalIgnoreCase));
+
+        if (entry == null)
        {
-            foreach (var filecache in fileCacheEntities.ToList())
-            {
-                var validated = GetValidatedFileCache(filecache);
-                if (validated != null) output.Add(validated);
-            }
+            _logger.LogDebug("Found no entries for {path}", cleanedPath);
+            return CreateFileEntry(path);
        }

-        return output;
+        var validatedCacheEntry = GetValidatedFileCache(entry);
+
+        return validatedCacheEntry;
    }

    public Dictionary<string, FileCacheEntity?> GetFileCachesByPaths(string[] paths)
@@ -217,29 +240,6 @@ public sealed class FileCacheManager : IDisposable
        return result;
    }

-    public FileCacheEntity? GetFileCacheByPath(string path)
-    {
-        var cleanedPath = path.Replace("/", "\\", StringComparison.OrdinalIgnoreCase).ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), "", StringComparison.OrdinalIgnoreCase);
-        var entry = _fileCaches.SelectMany(v => v.Value).FirstOrDefault(f => f.ResolvedFilepath.EndsWith(cleanedPath, StringComparison.OrdinalIgnoreCase));
-
-        if (entry == null)
-        {
-            _logger.LogDebug("Found no entries for {path}", cleanedPath);
-            return CreateFileEntry(path);
-        }
-
-        var validatedCacheEntry = GetValidatedFileCache(entry);
-
-        return validatedCacheEntry;
-    }
-
-    public async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken)
-    {
-        var fileCache = GetFileCacheByHash(fileHash)!.ResolvedFilepath;
-        return (fileHash, LZ4Codec.WrapHC(await File.ReadAllBytesAsync(fileCache, uploadToken).ConfigureAwait(false), 0,
-            (int)new FileInfo(fileCache).Length));
-    }
-
    public void RemoveHashedFile(string hash, string prefixedFilePath)
    {
        if (_fileCaches.TryGetValue(hash, out var caches))
@@ -316,14 +316,12 @@ public sealed class FileCacheManager : IDisposable
        try
        {
            RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath);
-            FileInfo oldCache = new(fileCache.ResolvedFilepath);
-            var extensionPath = fileCache.ResolvedFilepath.ToUpper() + "." + ext;
-            File.Move(fileCache.ResolvedFilepath, extensionPath, true);
-            var newHashedEntity = new FileCacheEntity(fileCache.Hash, fileCache.PrefixedFilePath + "." + ext, DateTime.UtcNow.Ticks.ToString());
+            FileInfo fileInfo = new(fileCache.ResolvedFilepath);
+            FileInfo oldCache = fileInfo;
+            var extensionPath = fileCache.ResolvedFilepath.ToUpper(CultureInfo.InvariantCulture) + "." + ext;
+            File.Move(fileCache.ResolvedFilepath, extensionPath, overwrite: true);
+            var newHashedEntity = new FileCacheEntity(fileCache.Hash, fileCache.PrefixedFilePath + "." + ext, DateTime.UtcNow.Ticks.ToString(CultureInfo.InvariantCulture));
            newHashedEntity.SetResolvedFilePath(extensionPath);
-            FileInfo newCache = new FileInfo(extensionPath);
-            newCache.LastAccessTime = oldCache.LastAccessTime;
-            newCache.LastWriteTime = oldCache.LastWriteTime;
            AddHashedFile(newHashedEntity);
            _logger.LogDebug("Migrated from {oldPath} to {newPath}", fileCache.ResolvedFilepath, newHashedEntity.ResolvedFilepath);
            return newHashedEntity;
@@ -340,7 +338,7 @@ public sealed class FileCacheManager : IDisposable
    {
        if (!_fileCaches.TryGetValue(fileCache.Hash, out var entries))
        {
-            _fileCaches[fileCache.Hash] = entries = new();
+            _fileCaches[fileCache.Hash] = entries = [];
        }

        if (!entries.Exists(u => string.Equals(u.PrefixedFilePath, fileCache.PrefixedFilePath, StringComparison.OrdinalIgnoreCase)))
@@ -394,7 +392,7 @@ public sealed class FileCacheManager : IDisposable
            return null;
        }

-        if (!string.Equals(file.LastWriteTimeUtc.Ticks.ToString(), fileCache.LastModifiedDateTicks, StringComparison.Ordinal))
+        if (!string.Equals(file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), fileCache.LastModifiedDateTicks, StringComparison.Ordinal))
        {
            UpdateHashedFile(fileCache);
        }
--- a/MareSynchronos/FileCache/FileCompactor.cs
+++ b/MareSynchronos/FileCache/FileCompactor.cs
@@ -137,8 +137,10 @@ public sealed class FileCompactor
        {
            using (var fs = new FileStream(path, FileMode.Open))
            {
+#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called
                var hDevice = fs.SafeFileHandle.DangerousGetHandle();
-                var ret = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _);
+#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called
+                _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _);
            }
        }
        catch (Exception ex)
@@ -153,7 +155,7 @@ public sealed class FileCompactor
        if (!fi.Exists) return -1;
        var root = fi.Directory?.Root.FullName.ToLower() ?? string.Empty;
        if (string.IsNullOrEmpty(root)) return -1;
-        if (_clusterSizes.ContainsKey(root)) return _clusterSizes[root];
+        if (_clusterSizes.TryGetValue(root, out int value)) return value;
        _logger.LogDebug("Getting Cluster Size for {path}, root {root}", filePath, root);
        int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _);
        if (result == 0) return -1;
@@ -162,7 +164,7 @@ public sealed class FileCompactor
        return _clusterSizes[root];
    }

-    private bool IsCompactedFile(string filePath)
+    private static bool IsCompactedFile(string filePath)
    {
        uint buf = 8;
        _ = WofIsExternalFile(filePath, out int isExtFile, out uint _, out var info, ref buf);
@@ -173,13 +175,15 @@ public sealed class FileCompactor
    private void WOFCompressFile(string path)
    {
        var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo));
-        Marshal.StructureToPtr(_efInfo, efInfoPtr, true);
+        Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: true);
        ulong length = (ulong)Marshal.SizeOf(_efInfo);
        try
        {
            using (var fs = new FileStream(path, FileMode.Open))
            {
+#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called
                var hFile = fs.SafeFileHandle.DangerousGetHandle();
+#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called
                if (fs.SafeFileHandle.IsInvalid)
                {
                    _logger.LogWarning("Invalid file handle to {file}", path);
--- a/MareSynchronos/FileCache/FileState.cs
+++ b/MareSynchronos/FileCache/FileState.cs
@@ -1,9 +1,8 @@
 namespace MareSynchronos.FileCache;

-
 public enum FileState
 {
    Valid,
    RequireUpdate,
    RequireDeletion,
-}
+}
--- a/MareSynchronos/FileCache/PeriodicFileScanner.cs
+++ b/MareSynchronos/FileCache/PeriodicFileScanner.cs
@@ -10,11 +10,11 @@ namespace MareSynchronos.FileCache;
 public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
 {
    private readonly MareConfigService _configService;
+    private readonly DalamudUtilService _dalamudUtil;
+    private readonly FileCompactor _fileCompactor;
    private readonly FileCacheManager _fileDbManager;
    private readonly IpcManager _ipcManager;
    private readonly PerformanceCollectorService _performanceCollector;
-    private readonly DalamudUtilService _dalamudUtil;
-    private readonly FileCompactor _fileCompactor;
    private long _currentFileProgress = 0;
    private bool _fileScanWasRunning = false;
    private CancellationTokenSource _scanCancellationTokenSource = new();
@@ -38,15 +38,12 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
    }

    public long CurrentFileProgress => _currentFileProgress;
-    public long TotalFilesStorage { get; private set; }
    public long FileCacheSize { get; set; }
    public ConcurrentDictionary<string, int> HaltScanLocks { get; set; } = new(StringComparer.Ordinal);
    public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
-
    public string TimeUntilNextScan => _timeUntilNextScan.ToString(@"mm\:ss");
-
    public long TotalFiles { get; private set; }
-
+    public long TotalFilesStorage { get; private set; }
    private int TimeBetweenScans => _configService.Current.TimeSpanBetweenScansInSeconds;

    public void HaltScan(string source)
@@ -213,19 +210,22 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
        var previousThreadPriority = Thread.CurrentThread.Priority;
        Thread.CurrentThread.Priority = ThreadPriority.Lowest;
        Logger.LogDebug("Getting files from {penumbra} and {storage}", penumbraDir, _configService.Current.CacheFolder);
-        string[] ext = { ".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp", ".shpk" };
+        string[] ext = [".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp", ".shpk"];

        Dictionary<string, string[]> penumbraFiles = new(StringComparer.Ordinal);
        foreach (var folder in Directory.EnumerateDirectories(penumbraDir!))
        {
            try
            {
-                penumbraFiles[folder] = Directory.GetFiles(folder, "*.*", SearchOption.AllDirectories)
-                        .AsParallel()
-                        .Where(f => ext.Any(e => f.EndsWith(e, StringComparison.OrdinalIgnoreCase))
-                            && !f.Contains(@"\bg\", StringComparison.OrdinalIgnoreCase)
-                            && !f.Contains(@"\bgcommon\", StringComparison.OrdinalIgnoreCase)
-                            && !f.Contains(@"\ui\", StringComparison.OrdinalIgnoreCase)).ToArray();
+                penumbraFiles[folder] =
+                [
+                    .. Directory.GetFiles(folder, "*.*", SearchOption.AllDirectories)
+                                            .AsParallel()
+                                            .Where(f => ext.Any(e => f.EndsWith(e, StringComparison.OrdinalIgnoreCase))
+                                                && !f.Contains(@"\bg\", StringComparison.OrdinalIgnoreCase)
+                                                && !f.Contains(@"\bgcommon\", StringComparison.OrdinalIgnoreCase)
+                                                && !f.Contains(@"\ui\", StringComparison.OrdinalIgnoreCase)),
+                ];
            }
            catch (Exception ex)
            {
@@ -239,7 +239,7 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
                                .AsParallel()
                                .Where(f =>
                                {
-                                    var val = f.Split('\\').Last();
+                                    var val = f.Split('\\')[^1];
                                    return val.Length == 40 || (val.Split('.').FirstOrDefault()?.Length ?? 0) == 40;
                                });

@@ -260,8 +260,8 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
        // scan files from database
        var threadCount = Math.Clamp((int)(Environment.ProcessorCount / 2.0f), 2, 8);

-        List<FileCacheEntity> entitiesToRemove = new();
-        List<FileCacheEntity> entitiesToUpdate = new();
+        List<FileCacheEntity> entitiesToRemove = [];
+        List<FileCacheEntity> entitiesToUpdate = [];
        object sync = new();
        Thread[] workerThreads = new Thread[threadCount];

--- a/MareSynchronos/FileCache/TransientResourceManager.cs
+++ b/MareSynchronos/FileCache/TransientResourceManager.cs
@@ -14,9 +14,9 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
    private readonly HashSet<string> _cachedHandledPaths = new(StringComparer.Ordinal);
    private readonly TransientConfigService _configurationService;
    private readonly DalamudUtilService _dalamudUtil;
-    private readonly string[] _fileTypesToHandle = new[] { "tmb", "pap", "avfx", "atex", "sklb", "eid", "phyb", "scd", "skp", "shpk" };
-    private readonly HashSet<GameObjectHandler> _playerRelatedPointers = new();
-    private HashSet<IntPtr> _cachedFrameAddresses = new();
+    private readonly string[] _fileTypesToHandle = ["tmb", "pap", "avfx", "atex", "sklb", "eid", "phyb", "scd", "skp", "shpk"];
+    private readonly HashSet<GameObjectHandler> _playerRelatedPointers = [];
+    private HashSet<IntPtr> _cachedFrameAddresses = [];

    public TransientResourceManager(ILogger<TransientResourceManager> logger, TransientConfigService configurationService,
        DalamudUtilService dalamudUtil, MareMediator mediator) : base(logger, mediator)
@@ -68,17 +68,17 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase

    public void CleanUpSemiTransientResources(ObjectKind objectKind, List<FileReplacement>? fileReplacement = null)
    {
-        if (SemiTransientResources.ContainsKey(objectKind))
+        if (SemiTransientResources.TryGetValue(objectKind, out HashSet<string>? value))
        {
            if (fileReplacement == null)
            {
-                SemiTransientResources[objectKind].Clear();
+                value.Clear();
                return;
            }

            foreach (var replacement in fileReplacement.Where(p => !p.HasFileReplacement).SelectMany(p => p.GamePaths).ToList())
            {
-                SemiTransientResources[objectKind].RemoveWhere(p => string.Equals(p, replacement, StringComparison.OrdinalIgnoreCase));
+                value.RemoveWhere(p => string.Equals(p, replacement, StringComparison.OrdinalIgnoreCase));
            }
        }
    }
@@ -97,17 +97,18 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
    {
        if (TransientResources.TryGetValue(gameObject, out var result))
        {
-            return result.ToList();
+            return [.. result];
        }

-        return new List<string>();
+        return [];
    }

    public void PersistTransientResources(IntPtr gameObject, ObjectKind objectKind)
    {
-        if (!SemiTransientResources.ContainsKey(objectKind))
+        if (!SemiTransientResources.TryGetValue(objectKind, out HashSet<string>? value))
        {
-            SemiTransientResources[objectKind] = new HashSet<string>(StringComparer.Ordinal);
+            value = new HashSet<string>(StringComparer.Ordinal);
+            SemiTransientResources[objectKind] = value;
        }

        if (!TransientResources.TryGetValue(gameObject, out var resources))
@@ -119,7 +120,7 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
        Logger.LogDebug("Persisting {count} transient resources", transientResources.Count);
        foreach (var gamePath in transientResources)
        {
-            SemiTransientResources[objectKind].Add(gamePath);
+            value.Add(gamePath);
        }

        if (objectKind == ObjectKind.Player && SemiTransientResources.TryGetValue(ObjectKind.Player, out var fileReplacements))
@@ -132,12 +133,13 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase

    internal void AddSemiTransientResource(ObjectKind objectKind, string item)
    {
-        if (!SemiTransientResources.ContainsKey(objectKind))
+        if (!SemiTransientResources.TryGetValue(objectKind, out HashSet<string>? value))
        {
-            SemiTransientResources[objectKind] = new HashSet<string>(StringComparer.Ordinal);
+            value = new HashSet<string>(StringComparer.Ordinal);
+            SemiTransientResources[objectKind] = value;
        }

-        SemiTransientResources[objectKind].Add(item.ToLowerInvariant());
+        value.Add(item.ToLowerInvariant());
    }

    internal void ClearTransientPaths(IntPtr ptr, List<string> list)
@@ -154,9 +156,9 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase

        TransientResources.Clear();
        SemiTransientResources.Clear();
-        if (SemiTransientResources.ContainsKey(ObjectKind.Player))
+        if (SemiTransientResources.TryGetValue(ObjectKind.Player, out HashSet<string>? value))
        {
-            _configurationService.Current.PlayerPersistentTransientCache[PlayerPersistentDataKey] = SemiTransientResources[ObjectKind.Player];
+            _configurationService.Current.PlayerPersistentTransientCache[PlayerPersistentDataKey] = value;
            _configurationService.Save();
        }
    }
@@ -182,7 +184,7 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase

    private void Manager_PenumbraModSettingChanged()
    {
-        Task.Run(() =>
+        _ = Task.Run(() =>
        {
            Logger.LogDebug("Penumbra Mod Settings changed, verifying SemiTransientResources");
            foreach (var item in _playerRelatedPointers)
@@ -229,19 +231,20 @@ public sealed class TransientResourceManager : DisposableMediatorSubscriberBase
            return;
        }

-        if (!TransientResources.ContainsKey(gameObject))
+        if (!TransientResources.TryGetValue(gameObject, out HashSet<string>? value))
        {
-            TransientResources[gameObject] = new(StringComparer.OrdinalIgnoreCase);
+            value = new(StringComparer.OrdinalIgnoreCase);
+            TransientResources[gameObject] = value;
        }

-        if (TransientResources[gameObject].Contains(replacedGamePath) ||
+        if (value.Contains(replacedGamePath) ||
            SemiTransientResources.SelectMany(k => k.Value).Any(f => string.Equals(f, gamePath, StringComparison.OrdinalIgnoreCase)))
        {
            Logger.LogTrace("Not adding {replacedPath} : {filePath}", replacedGamePath, filePath);
        }
        else
        {
-            TransientResources[gameObject].Add(replacedGamePath);
+            value.Add(replacedGamePath);
            Logger.LogDebug("Adding {replacedGamePath} for {gameObject} ({filePath})", replacedGamePath, gameObject.ToString("X"), filePath);
            Mediator.Publish(new TransientResourceChangedMessage(gameObject));
        }