remove database, use in-memory

MareSynchronos/Factories/CharacterDataFactory.cs

@@ -8,7 +8,7 @@ using FFXIVClientStructs.FFXIV.Client.Game.Character;
 using FFXIVClientStructs.FFXIV.Client.Graphics.Scene;
 using FFXIVClientStructs.FFXIV.Client.System.Resource;
 using MareSynchronos.API;
-using MareSynchronos.FileCacheDB;
+using MareSynchronos.FileCache;
 using MareSynchronos.Interop;
 using MareSynchronos.Managers;
 using MareSynchronos.Models;
@@ -24,9 +24,9 @@ public class CharacterDataFactory
     private readonly DalamudUtil _dalamudUtil;
     private readonly IpcManager _ipcManager;
     private readonly TransientResourceManager transientResourceManager;
-    private readonly FileDbManager fileDbManager;
+    private readonly FileCacheManager fileDbManager;
 
-    public CharacterDataFactory(DalamudUtil dalamudUtil, IpcManager ipcManager, TransientResourceManager transientResourceManager, FileDbManager fileDbManager)
+    public CharacterDataFactory(DalamudUtil dalamudUtil, IpcManager ipcManager, TransientResourceManager transientResourceManager, FileCacheManager fileDbManager)
     {
         Logger.Verbose("Creating " + nameof(CharacterDataFactory));
         this.fileDbManager = fileDbManager;

MareSynchronos/FileCache/FileCache.cs

@@ -0,0 +1,29 @@
+#nullable disable
+
+
+using MareSynchronos;
+using System.Globalization;
+
+namespace MareSynchronos.FileCache;
+
+public class FileCache
+{
+    public string ResolvedFilepath { get; private set; }
+    public string Hash { get; set; }
+    public string PrefixedFilePath { get; init; }
+    public string LastModifiedDateTicks { get; init; }
+
+    public FileCache(string hash, string path, string lastModifiedDateTicks)
+    {
+        Hash = hash;
+        PrefixedFilePath = path;
+        LastModifiedDateTicks = lastModifiedDateTicks;
+    }
+
+    public void SetResolvedFilePath(string filePath)
+    {
+        ResolvedFilepath = filePath.ToLowerInvariant().Replace("\\\\", "\\");
+    }
+
+    public string CsvEntry => $"{Hash}{FileCacheManager.CsvSplit}{PrefixedFilePath}{FileCacheManager.CsvSplit}{LastModifiedDateTicks.ToString(CultureInfo.InvariantCulture)}";
+}

MareSynchronos/FileCache/FileDbManager.cs

@@ -0,0 +1,210 @@
+using MareSynchronos.Managers;
+using MareSynchronos.Utils;
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace MareSynchronos.FileCache;
+
+
+public enum FileState
+{
+    Valid,
+    RequireUpdate,
+    RequireDeletion
+}
+
+public class FileCacheManager : IDisposable
+{
+    private const string PenumbraPrefix = "{penumbra}";
+    private const string CachePrefix = "{cache}";
+    private readonly IpcManager _ipcManager;
+    private readonly Configuration _configuration;
+    private readonly string CsvPath;
+    private string CsvBakPath => CsvPath + ".bak";
+    private readonly ConcurrentDictionary<string, FileCache> FileCaches = new();
+    public const string CsvSplit = "|";
+    private object _fileWriteLock = new object();
+
+    public FileCacheManager(IpcManager ipcManager, Configuration configuration, string configDirectoryName)
+    {
+        _ipcManager = ipcManager;
+        _configuration = configuration;
+        CsvPath = Path.Combine(configDirectoryName, "FileCache.csv");
+
+        if (File.Exists(CsvBakPath))
+        {
+            File.Move(CsvBakPath, CsvPath, true);
+        }
+
+        if (File.Exists(CsvPath))
+        {
+            var entries = File.ReadAllLines(CsvPath);
+            foreach (var entry in entries)
+            {
+                var splittedEntry = entry.Split(CsvSplit, StringSplitOptions.None);
+                var hash = splittedEntry[0];
+                var path = splittedEntry[1];
+                var time = splittedEntry[2];
+                FileCaches[hash] = new FileCache(hash, path, time);
+            }
+        }
+    }
+
+    public void WriteOutFullCsv()
+    {
+        StringBuilder sb = new StringBuilder();
+        foreach (var entry in FileCaches)
+        {
+            sb.AppendLine(entry.Value.CsvEntry);
+        }
+        if (File.Exists(CsvPath))
+        {
+            File.Copy(CsvPath, CsvBakPath, true);
+        }
+        lock (_fileWriteLock)
+        {
+            File.WriteAllText(CsvPath, sb.ToString());
+        }
+    }
+
+    public List<FileCache> GetAllFileCaches() => FileCaches.Values.ToList();
+
+    public FileCache? GetFileCacheByHash(string hash)
+    {
+        if (FileCaches.ContainsKey(hash))
+        {
+            return GetValidatedFileCache(FileCaches[hash]);
+        }
+
+        return null;
+    }
+
+    public (FileState, FileCache) ValidateFileCacheEntity(FileCache fileCache)
+    {
+        fileCache = ReplacePathPrefixes(fileCache);
+        FileInfo fi = new(fileCache.ResolvedFilepath);
+        if (!fi.Exists)
+        {
+            return (FileState.RequireDeletion, fileCache);
+        }
+        if (fi.LastWriteTimeUtc.Ticks.ToString() != fileCache.LastModifiedDateTicks)
+        {
+            return (FileState.RequireUpdate, fileCache);
+        }
+
+        return (FileState.Valid, fileCache);
+    }
+
+    public FileCache? GetFileCacheByPath(string path)
+    {
+        var cleanedPath = path.Replace("/", "\\").ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), "");
+        var entry = FileCaches.FirstOrDefault(f => f.Value.ResolvedFilepath.EndsWith(cleanedPath)).Value;
+
+        if (entry == null)
+        {
+            Logger.Debug("Found no entries for " + cleanedPath);
+            return CreateFileEntry(path);
+        }
+
+        var validatedCacheEntry = GetValidatedFileCache(entry);
+
+        return validatedCacheEntry;
+    }
+
+    public FileCache? CreateCacheEntry(string path)
+    {
+        Logger.Debug("Creating cache entry for " + path);
+        FileInfo fi = new(path);
+        if (!fi.Exists) return null;
+        var fullName = fi.FullName.ToLowerInvariant();
+        if (!fullName.Contains(_configuration.CacheFolder.ToLowerInvariant())) return null;
+        string prefixedPath = fullName.Replace(_configuration.CacheFolder.ToLowerInvariant(), CachePrefix + "\\").Replace("\\\\", "\\");
+        return CreateFileCacheEntity(fi, prefixedPath);
+    }
+
+    public FileCache? CreateFileEntry(string path)
+    {
+        Logger.Debug("Creating file entry for " + path);
+        FileInfo fi = new(path);
+        if (!fi.Exists) return null;
+        var fullName = fi.FullName.ToLowerInvariant();
+        if (!fullName.Contains(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant())) return null;
+        string prefixedPath = fullName.Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), PenumbraPrefix + "\\").Replace("\\\\", "\\");
+        return CreateFileCacheEntity(fi, prefixedPath);
+    }
+
+    private FileCache? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath)
+    {
+        var hash = Crypto.GetFileHash(fileInfo.FullName);
+        var entity = new FileCache(hash, prefixedPath, fileInfo.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture));
+        FileCaches[hash] = entity;
+        lock (_fileWriteLock)
+        {
+            File.AppendAllLines(CsvPath, new[] { entity.CsvEntry });
+        }
+        var result = GetFileCacheByPath(prefixedPath);
+        Logger.Debug("Creating file cache for " + fileInfo.FullName + " success: " + (result != null));
+        return result;
+    }
+
+    private FileCache? GetValidatedFileCache(FileCache fileCache)
+    {
+        var resulingFileCache = ReplacePathPrefixes(fileCache);
+        resulingFileCache = Validate(resulingFileCache);
+        return resulingFileCache;
+    }
+
+    private FileCache? Validate(FileCache fileCache)
+    {
+        var file = new FileInfo(fileCache.ResolvedFilepath);
+        if (!file.Exists)
+        {
+            FileCaches.Remove(fileCache.Hash, out _);
+            return null;
+        }
+
+        if (file.LastWriteTimeUtc.Ticks.ToString() != fileCache.LastModifiedDateTicks)
+        {
+            UpdateHash(fileCache);
+        }
+
+        return fileCache;
+    }
+
+    public void RemoveHash(FileCache entity)
+    {
+        FileCaches.Remove(entity.Hash, out _);
+    }
+
+    public void UpdateHash(FileCache fileCache)
+    {
+        var prevHash = fileCache.Hash;
+        fileCache.Hash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
+        FileCaches.Remove(prevHash, out _);
+        FileCaches[fileCache.Hash] = fileCache;
+    }
+
+    private FileCache ReplacePathPrefixes(FileCache fileCache)
+    {
+        if (fileCache.PrefixedFilePath.StartsWith(PenumbraPrefix))
+        {
+            fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory()));
+        }
+        else if (fileCache.PrefixedFilePath.StartsWith(CachePrefix))
+        {
+            fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(CachePrefix, _configuration.CacheFolder));
+        }
+
+        return fileCache;
+    }
+
+    public void Dispose()
+    {
+        WriteOutFullCsv();
+    }
+}

MareSynchronos/FileCache/PeriodicFileScanner.cs

@@ -7,21 +7,20 @@ using System.Threading.Tasks;
 using MareSynchronos.Managers;
 using MareSynchronos.Utils;
 using MareSynchronos.WebAPI;
-using Microsoft.EntityFrameworkCore;
 
-namespace MareSynchronos.FileCacheDB;
+namespace MareSynchronos.FileCache;
 
 public class PeriodicFileScanner : IDisposable
 {
     private readonly IpcManager _ipcManager;
     private readonly Configuration _pluginConfiguration;
-    private readonly FileDbManager _fileDbManager;
+    private readonly FileCacheManager _fileDbManager;
     private readonly ApiController _apiController;
     private readonly DalamudUtil _dalamudUtil;
     private int haltScanRequests = 0;
     private CancellationTokenSource? _scanCancellationTokenSource;
     private Task? _fileScannerTask = null;
-    public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileDbManager fileDbManager, ApiController apiController, DalamudUtil dalamudUtil)
+    public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileCacheManager fileDbManager, ApiController apiController, DalamudUtil dalamudUtil)
     {
         Logger.Verbose("Creating " + nameof(PeriodicFileScanner));
 
@@ -198,47 +197,33 @@ public class PeriodicFileScanner : IDisposable
         var cpuCount = (int)(Environment.ProcessorCount / 2.0f);
         Task[] dbTasks = Enumerable.Range(0, cpuCount).Select(c => Task.CompletedTask).ToArray();
 
-        ConcurrentBag<Tuple<string, string>> entitiesToRemove = new();
+        ConcurrentBag<FileCache> entitiesToRemove = new();
-        ConcurrentBag<string> entitiesToUpdate = new();
+        ConcurrentBag<FileCache> entitiesToUpdate = new();
         try
         {
-            using var ctx = new FileCacheContext();
+            foreach (var value in _fileDbManager.GetAllFileCaches())
-            Logger.Debug("Database contains " + ctx.FileCaches.Count() + " files, local system contains " + TotalFiles);
-            using var cmd = ctx.Database.GetDbConnection().CreateCommand();
-            cmd.CommandText = "SELECT Hash, FilePath, LastModifiedDate FROM FileCache";
-            ctx.Database.OpenConnection();
-            using var reader = cmd.ExecuteReader();
-            while (reader.Read())
             {
-                var hash = reader["Hash"].ToString();
-                var filePath = reader["FilePath"].ToString();
-                var date = reader["LastModifiedDate"].ToString();
                 var idx = Task.WaitAny(dbTasks, ct);
                 dbTasks[idx] = Task.Run(() =>
                 {
                     try
                     {
-                        var fileState = _fileDbManager.ValidateFileCacheEntity(hash, filePath, date);
+                        var result = _fileDbManager.ValidateFileCacheEntity(value);
-                        switch (fileState.Item1)
+                        scannedFiles[result.Item2.ResolvedFilepath] = true;
+                        if (result.Item1 == FileState.RequireUpdate)
                         {
-                            case FileState.Valid:
+                            entitiesToUpdate.Add(result.Item2);
-                                scannedFiles[fileState.Item2] = true;
+                        }
-                                break;
+                        else if (result.Item1 == FileState.RequireDeletion)
-                            case FileState.RequireDeletion:
+                        {
-                                entitiesToRemove.Add(new Tuple<string, string>(hash, filePath));
+                            entitiesToRemove.Add(result.Item2);
-                                break;
-                            case FileState.RequireUpdate:
-                                scannedFiles[fileState.Item2] = true;
-                                entitiesToUpdate.Add(fileState.Item2);
-                                break;
                         }
                     }
                     catch (Exception ex)
                     {
-                        Logger.Warn("Failed validating " + filePath);
+                        Logger.Warn("Failed validating " + value.ResolvedFilepath);
                         Logger.Warn(ex.Message);
                         Logger.Warn(ex.StackTrace);
-                        entitiesToRemove.Add(new Tuple<string, string>(hash, filePath));
                     }
 
                     Interlocked.Increment(ref currentFileProgress);
@@ -257,44 +242,23 @@ public class PeriodicFileScanner : IDisposable
             Logger.Warn("Error during enumerating FileCaches: " + ex.Message);
         }
 
-        using (var db = new FileCacheContext())
-        {
-            try
-            {
-                if (entitiesToRemove.Any())
-                {
-                    foreach (var entry in entitiesToRemove)
-                    {
-                        Logger.Debug("Removing " + entry.Item2);
-                        var toRemove = db.FileCaches.First(f => f.Filepath == entry.Item2 && f.Hash == entry.Item1);
-                        db.FileCaches.Remove(toRemove);
-                    }
-
-                }
-
-                if (entitiesToUpdate.Any())
-                {
-                    foreach (var entry in entitiesToUpdate)
-                    {
-                        Logger.Debug("Updating " + entry);
-                        _fileDbManager.GetFileCacheByPath(entry);
-                    }
-                }
-
-                if (entitiesToUpdate.Any() || entitiesToRemove.Any())
-                {
-                    db.SaveChanges();
-                }
-
-            }
-            catch (Exception ex)
-            {
-                Logger.Warn(ex.Message);
-            }
-        }
-
         Task.WaitAll(dbTasks);
 
+        if (entitiesToUpdate.Any() || entitiesToRemove.Any())
+        {
+            foreach (var entity in entitiesToUpdate)
+            {
+                _fileDbManager.UpdateHash(entity);
+            }
+
+            foreach (var entity in entitiesToRemove)
+            {
+                _fileDbManager.RemoveHash(entity);
+            }
+
+            _fileDbManager.WriteOutFullCsv();
+        }
+
         Logger.Debug("Scanner validated existing db files");
 
         if (ct.IsCancellationRequested) return;

MareSynchronos/FileCacheDB/FileCache.cs

@@ -1,48 +0,0 @@
-#nullable disable
-
-
-using System;
-
-namespace MareSynchronos.FileCacheDB
-{
-
-    public class FileCache
-    {
-        private FileCacheEntity entity;
-        public string Filepath { get; private set; }
-        public string Hash { get; private set; }
-        private string originalFilePathNoEntity = string.Empty;
-        private string originalHashNoEntity = string.Empty;
-        private string originalModifiedDate = string.Empty;
-        public string OriginalFilepath => entity == null ? originalFilePathNoEntity : entity.Filepath;
-        public string OriginalHash => entity == null ? originalHashNoEntity : entity.Hash;
-        public long LastModifiedDateTicks => long.Parse(entity == null ? originalModifiedDate : entity.LastModifiedDate);
-
-        public FileCache(string hash, string path, string lastModifiedDate)
-        {
-            originalHashNoEntity = hash;
-            originalFilePathNoEntity = path;
-            originalModifiedDate = lastModifiedDate;
-        }
-
-        public FileCache(FileCacheEntity entity)
-        {
-            this.entity = entity;
-        }
-
-        public void SetResolvedFilePath(string filePath)
-        {
-            Filepath = filePath.ToLowerInvariant().Replace("\\\\", "\\");
-        }
-
-        public void SetHash(string hash)
-        {
-            Hash = hash;
-        }
-
-        public void UpdateFileCache(FileCacheEntity entity)
-        {
-            this.entity = entity;
-        }
-    }
-}

MareSynchronos/FileCacheDB/FileCacheContext.cs

@@ -1,46 +0,0 @@
-using System.IO;
-using Microsoft.EntityFrameworkCore;
-
-#nullable disable
-
-namespace MareSynchronos.FileCacheDB
-{
-    public partial class FileCacheContext : DbContext
-    {
-        private string DbPath { get; set; }
-        public FileCacheContext()
-        {
-            DbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "FileCache.db");
-            Database.EnsureCreated();
-        }
-
-        public FileCacheContext(DbContextOptions<FileCacheContext> options)
-            : base(options)
-        {
-        }
-
-        public virtual DbSet<FileCacheEntity> FileCaches { get; set; }
-
-        protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
-        {
-            if (!optionsBuilder.IsConfigured)
-            {
-                optionsBuilder.UseSqlite("Data Source=" + DbPath+";Cache=Shared");
-            }
-        }
-
-        protected override void OnModelCreating(ModelBuilder modelBuilder)
-        {
-            modelBuilder.Entity<FileCacheEntity>(entity =>
-            {
-                entity.HasKey(e => new { e.Hash, e.Filepath });
-
-                entity.ToTable("FileCache");
-            });
-
-            OnModelCreatingPartial(modelBuilder);
-        }
-
-        partial void OnModelCreatingPartial(ModelBuilder modelBuilder);
-    }
-}

MareSynchronos/FileCacheDB/FileCacheEntity.cs

@@ -1,13 +0,0 @@
-#nullable disable
-
-
-namespace MareSynchronos.FileCacheDB
-{
-    public partial class FileCacheEntity
-    {
-        public string Hash { get; set; }
-        public string Filepath { get; set; }
-        public string LastModifiedDate { get; set; }
-        public int Version { get; set; }
-    }
-}

MareSynchronos/FileCacheDB/FileDbManager.cs

@@ -1,284 +0,0 @@
-using MareSynchronos.FileCacheDB;
-using MareSynchronos.Utils;
-using System;
-using System.Collections.Generic;
-using System.Globalization;
-using System.IO;
-using System.Linq;
-
-namespace MareSynchronos.Managers;
-
-public enum FileState
-{
-    Valid,
-    RequireUpdate,
-    RequireDeletion
-}
-
-public class FileDbManager
-{
-    private const string PenumbraPrefix = "{penumbra}";
-    private const string CachePrefix = "{cache}";
-    private readonly IpcManager _ipcManager;
-    private readonly Configuration _configuration;
-    private static object _lock = new();
-
-    public FileDbManager(IpcManager ipcManager, Configuration configuration)
-    {
-        _ipcManager = ipcManager;
-        _configuration = configuration;
-    }
-
-    public FileCache? GetFileCacheByHash(string hash)
-    {
-        List<FileCacheEntity> matchingEntries = new List<FileCacheEntity>();
-        using (var db = new FileCacheContext())
-        {
-            matchingEntries = db.FileCaches.Where(f => f.Hash.ToLower() == hash.ToLower()).ToList();
-        }
-
-        if (!matchingEntries.Any()) return null;
-
-        if (matchingEntries.Any(f => f.Filepath.Contains(PenumbraPrefix) && matchingEntries.Any(f => f.Filepath.Contains(CachePrefix))))
-        {
-            var cachedEntries = matchingEntries.Where(f => f.Filepath.Contains(CachePrefix)).ToList();
-            DeleteFromDatabase(cachedEntries.Select(f => new FileCache(f)));
-            foreach (var entry in cachedEntries)
-            {
-                matchingEntries.Remove(entry);
-            }
-        }
-
-        return GetValidatedFileCache(new FileCache(matchingEntries.First()));
-    }
-
-    public (FileState, string) ValidateFileCacheEntity(string hash, string path, string lastModifiedDate)
-    {
-        var fileCache = new FileCache(hash, path, lastModifiedDate);
-        if (!fileCache.OriginalFilepath.StartsWith(PenumbraPrefix + "\\") && !fileCache.OriginalFilepath.StartsWith(CachePrefix)) 
-            return (FileState.RequireUpdate, path);
-        fileCache = ReplacePathPrefixes(fileCache);
-        FileInfo fi = new FileInfo(fileCache.Filepath);
-        if (!fi.Exists) 
-            return (FileState.RequireDeletion, fileCache.Filepath);
-        if (fi.LastWriteTimeUtc.Ticks != fileCache.LastModifiedDateTicks) 
-            return (FileState.RequireUpdate, fileCache.Filepath);
-
-        return (FileState.Valid, fileCache.Filepath);
-    }
-
-    public FileCache? GetFileCacheByPath(string path)
-    {
-        FileCacheEntity? matchingEntries = null;
-        var cleanedPath = path.Replace("/", "\\").ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), "");
-        using (var db = new FileCacheContext())
-        {
-            matchingEntries = db.FileCaches.FirstOrDefault(f => f.Filepath.EndsWith(cleanedPath));
-        }
-
-        if (matchingEntries == null)
-        {
-            Logger.Debug("Found no entries for " + cleanedPath);
-            return CreateFileEntry(path);
-        }
-
-        var validatedCacheEntry = GetValidatedFileCache(new FileCache(matchingEntries));
-
-        return validatedCacheEntry;
-    }
-
-    public FileCache? CreateCacheEntry(string path)
-    {
-        Logger.Debug("Creating cache entry for " + path);
-        FileInfo fi = new FileInfo(path);
-        if (!fi.Exists) return null;
-        string prefixedPath = fi.FullName.ToLowerInvariant().Replace(_configuration.CacheFolder.ToLowerInvariant(), CachePrefix + "\\").Replace("\\\\", "\\");
-        return CreateFileCacheEntity(fi, prefixedPath);
-    }
-
-    public FileCache? CreateFileEntry(string path)
-    {
-        Logger.Debug("Creating file entry for " + path);
-        FileInfo fi = new FileInfo(path);
-        if (!fi.Exists) return null;
-        string prefixedPath = fi.FullName.ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), PenumbraPrefix + "\\").Replace("\\\\", "\\");
-        return CreateFileCacheEntity(fi, prefixedPath);
-    }
-
-    private FileCache? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath)
-    {
-        var hash = Crypto.GetFileHash(fileInfo.FullName);
-        lock (_lock)
-        {
-            var entity = new FileCacheEntity();
-            entity.Hash = hash;
-            entity.Filepath = prefixedPath;
-            entity.LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
-            try
-            {
-                using var db = new FileCacheContext();
-                db.FileCaches.Add(entity);
-                db.SaveChanges();
-            }
-            catch (Exception ex)
-            {
-                Logger.Warn("Could not add " + fileInfo.FullName ?? String.Empty);
-                Logger.Warn(ex.Message);
-            }
-        }
-        var result = GetFileCacheByPath(prefixedPath);
-        Logger.Debug("Creating file cache for " + fileInfo.FullName + " success: " + (result != null));
-        return result;
-    }
-
-    private FileCache? GetValidatedFileCache(FileCache fileCache)
-    {
-        var resulingFileCache = MigrateLegacy(fileCache);
-        if (resulingFileCache == null) return null;
-
-        resulingFileCache = ReplacePathPrefixes(resulingFileCache);
-        resulingFileCache = Validate(resulingFileCache);
-        return resulingFileCache;
-    }
-
-    private FileCache? Validate(FileCache fileCache)
-    {
-        var file = new FileInfo(fileCache.Filepath);
-        if (!file.Exists)
-        {
-            DeleteFromDatabase(new[] { fileCache });
-            return null;
-        }
-
-        if (file.LastWriteTimeUtc.Ticks != fileCache.LastModifiedDateTicks)
-        {
-            fileCache.SetHash(Crypto.GetFileHash(fileCache.Filepath));
-            UpdateCacheHash(fileCache, file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture));
-        }
-
-        return fileCache;
-    }
-
-    private FileCache? MigrateLegacy(FileCache fileCache)
-    {
-        if (fileCache.OriginalFilepath.StartsWith(PenumbraPrefix + "\\") || fileCache.OriginalFilepath.StartsWith(CachePrefix)) return fileCache;
-
-        var fileInfo = new FileInfo(fileCache.OriginalFilepath);
-        var penumbraDir = _ipcManager.PenumbraModDirectory()!;
-        if (penumbraDir.Last() != '\\') penumbraDir += "\\";
-        // check if it's a cache file
-        if (fileInfo.Exists && fileInfo.Name.Length == 40)
-        {
-            MigrateLegacyFilePath(fileCache, CachePrefix + "\\" + fileInfo.Name.ToLower());
-        }
-        else if (fileInfo.Exists && fileInfo.FullName.ToLowerInvariant().Contains(penumbraDir))
-        {
-            // attempt to replace penumbra mod folder path with {penumbra}
-            var newPath = PenumbraPrefix + "\\" + fileCache.OriginalFilepath.ToLowerInvariant().Replace(penumbraDir, string.Empty);
-            MigrateLegacyFilePath(fileCache, newPath);
-        }
-        else if (fileInfo.FullName.ToLowerInvariant().Contains(PenumbraPrefix))
-        {
-            var newPath = PenumbraPrefix + "\\" + fileCache.OriginalFilepath.ToLowerInvariant().Replace(PenumbraPrefix, string.Empty);
-            MigrateLegacyFilePath(fileCache, newPath);
-        }
-        else
-        {
-            DeleteFromDatabase(new[] { fileCache });
-            return null;
-        }
-
-        return fileCache;
-    }
-
-    private FileCache ReplacePathPrefixes(FileCache fileCache)
-    {
-        if (fileCache.OriginalFilepath.StartsWith(PenumbraPrefix))
-        {
-            fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory()));
-        }
-        else if (fileCache.OriginalFilepath.StartsWith(CachePrefix))
-        {
-            fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(CachePrefix, _configuration.CacheFolder));
-        }
-
-        return fileCache;
-    }
-
-    private void UpdateCacheHash(FileCache markedForUpdate, string lastModifiedDate)
-    {
-        lock (_lock)
-        {
-            try
-            {
-                Logger.Verbose("Updating Hash for " + markedForUpdate.OriginalFilepath);
-                using var db = new FileCacheContext();
-                var cache = db.FileCaches.First(f => f.Filepath == markedForUpdate.OriginalFilepath && f.Hash == markedForUpdate.OriginalHash);
-                var newcache = new FileCacheEntity()
-                {
-                    Filepath = cache.Filepath,
-                    Hash = markedForUpdate.Hash,
-                    LastModifiedDate = lastModifiedDate
-                };
-                db.Remove(cache);
-                db.FileCaches.Add(newcache);
-                markedForUpdate.UpdateFileCache(newcache);
-                db.SaveChanges();
-            }
-            catch (Exception ex)
-            {
-                Logger.Warn("Error updating file hash (" + ex.Message + "), returning currently existing" ?? string.Empty);
-                Logger.Warn(ex.InnerException?.Message ?? string.Empty);
-                Logger.Warn(ex.StackTrace ?? string.Empty);
-                using var db = new FileCacheContext();
-                var cache = db.FileCaches.First(f => f.Filepath == markedForUpdate.OriginalFilepath);
-                markedForUpdate.UpdateFileCache(cache);
-            }
-        }
-    }
-
-    private void MigrateLegacyFilePath(FileCache fileCacheToMigrate, string newPath)
-    {
-        lock (_lock)
-        {
-            Logger.Verbose("Migrating legacy file path for " + fileCacheToMigrate.OriginalFilepath);
-            using var db = new FileCacheContext();
-            var cache = db.FileCaches.First(f => f.Filepath == fileCacheToMigrate.OriginalFilepath && f.Hash == fileCacheToMigrate.OriginalHash);
-            var newcache = new FileCacheEntity()
-            {
-                Filepath = newPath,
-                Hash = cache.Hash,
-                LastModifiedDate = cache.LastModifiedDate
-            };
-            db.Remove(cache);
-            db.SaveChanges();
-            var existingCache = db.FileCaches.FirstOrDefault(f => f.Filepath == newPath && f.Hash == cache.Hash);
-            if (existingCache != null)
-            {
-                fileCacheToMigrate.UpdateFileCache(existingCache);
-            }
-            else
-            {
-                db.FileCaches.Add(newcache);
-                fileCacheToMigrate.UpdateFileCache(newcache);
-                db.SaveChanges();
-            }
-        }
-    }
-
-    private void DeleteFromDatabase(IEnumerable<FileCache> markedForDeletion)
-    {
-        lock (_lock)
-        {
-            using var db = new FileCacheContext();
-            foreach (var item in markedForDeletion)
-            {
-                Logger.Verbose("Removing " + item.OriginalFilepath);
-                var itemToRemove = db.FileCaches.FirstOrDefault(f => f.Hash == item.OriginalHash && f.Filepath == item.OriginalFilepath);
-                if (itemToRemove == null) continue;
-                db.FileCaches.Remove(itemToRemove);
-            }
-            db.SaveChanges();
-        }
-    }
-}

MareSynchronos/Managers/CachedPlayer.cs

@@ -6,6 +6,7 @@ using System.Threading.Tasks;
 using Dalamud.Logging;
 using FFXIVClientStructs.FFXIV.Client.Game.Character;
 using MareSynchronos.API;
+using MareSynchronos.FileCache;
 using MareSynchronos.Models;
 using MareSynchronos.Utils;
 using MareSynchronos.WebAPI;
@@ -15,12 +16,12 @@ namespace MareSynchronos.Managers;
 public class CachedPlayer
 {
     private readonly DalamudUtil _dalamudUtil;
-    private readonly FileDbManager fileDbManager;
+    private readonly FileCacheManager fileDbManager;
     private readonly IpcManager _ipcManager;
     private readonly ApiController _apiController;
     private bool _isVisible;
 
-    public CachedPlayer(string nameHash, IpcManager ipcManager, ApiController apiController, DalamudUtil dalamudUtil, FileDbManager fileDbManager)
+    public CachedPlayer(string nameHash, IpcManager ipcManager, ApiController apiController, DalamudUtil dalamudUtil, FileCacheManager fileDbManager)
     {
         PlayerNameHash = nameHash;
         _ipcManager = ipcManager;
@@ -201,7 +202,7 @@ public class CachedPlayer
                     var fileCache = fileDbManager.GetFileCacheByHash(item.Hash);
                     if (fileCache != null)
                     {
-                        moddedDictionary[gamePath] = fileCache.Filepath;
+                        moddedDictionary[gamePath] = fileCache.ResolvedFilepath;
                     }
                     else
                     {

MareSynchronos/Managers/OnlinePlayerManager.cs

@@ -5,6 +5,7 @@ using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using MareSynchronos.API;
+using MareSynchronos.FileCache;
 using MareSynchronos.Utils;
 using MareSynchronos.WebAPI;
 using MareSynchronos.WebAPI.Utils;
@@ -17,7 +18,7 @@ public class OnlinePlayerManager : IDisposable
     private readonly DalamudUtil _dalamudUtil;
     private readonly IpcManager _ipcManager;
     private readonly PlayerManager _playerManager;
-    private readonly FileDbManager _fileDbManager;
+    private readonly FileCacheManager _fileDbManager;
     private readonly ConcurrentDictionary<string, CachedPlayer> _onlineCachedPlayers = new();
     private readonly ConcurrentDictionary<string, CharacterCacheDto> _temporaryStoredCharacterCache = new();
     private readonly ConcurrentDictionary<CachedPlayer, CancellationTokenSource> _playerTokenDisposal = new();
@@ -25,7 +26,7 @@ public class OnlinePlayerManager : IDisposable
     private List<string> OnlineVisiblePlayerHashes => _onlineCachedPlayers.Select(p => p.Value).Where(p => p.PlayerCharacter != IntPtr.Zero)
         .Select(p => p.PlayerNameHash).ToList();
 
-    public OnlinePlayerManager(ApiController apiController, DalamudUtil dalamudUtil, IpcManager ipcManager, PlayerManager playerManager, FileDbManager fileDbManager)
+    public OnlinePlayerManager(ApiController apiController, DalamudUtil dalamudUtil, IpcManager ipcManager, PlayerManager playerManager, FileCacheManager fileDbManager)
     {
         Logger.Verbose("Creating " + nameof(OnlinePlayerManager));
 

MareSynchronos/Managers/PlayerManager.cs

@@ -9,7 +9,7 @@ using FFXIVClientStructs.FFXIV.Client.Game.Character;
 using System.Collections.Generic;
 using System.Linq;
 using MareSynchronos.Models;
-using MareSynchronos.FileCacheDB;
+using MareSynchronos.FileCache;
 #if DEBUG
 using Newtonsoft.Json;
 #endif

MareSynchronos/MareSynchronos.csproj

@@ -29,7 +29,6 @@
     <PackageReference Include="DalamudPackager" Version="2.1.8" />
     <PackageReference Include="lz4net" Version="1.0.15.93" />
     <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="6.0.8" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.8" />
   </ItemGroup>
 
   <PropertyGroup>

MareSynchronos/Models/FileReplacement.cs

@@ -4,15 +4,15 @@ using System.Text;
 using System.Threading.Tasks;
 using MareSynchronos.API;
 using System.Text.RegularExpressions;
-using MareSynchronos.Managers;
+using MareSynchronos.FileCache;
 
 namespace MareSynchronos.Models
 {
     public class FileReplacement
     {
-        private readonly FileDbManager fileDbManager;
+        private readonly FileCacheManager fileDbManager;
 
-        public FileReplacement(FileDbManager fileDbManager)
+        public FileReplacement(FileCacheManager fileDbManager)
         {
             this.fileDbManager = fileDbManager;
         }
@@ -37,7 +37,7 @@ namespace MareSynchronos.Models
             _ = Task.Run(() =>
             {
                 var cache = fileDbManager.GetFileCacheByPath(ResolvedPath);
-                Hash = cache.OriginalHash;
+                Hash = cache.Hash;
             });
         }
 

MareSynchronos/Plugin.cs

@@ -1,6 +1,5 @@
 using Dalamud.Game.Command;
 using Dalamud.Plugin;
-using MareSynchronos.FileCacheDB;
 using MareSynchronos.Factories;
 using System.Threading.Tasks;
 using Dalamud.Game;
@@ -13,8 +12,8 @@ using MareSynchronos.WebAPI;
 using Dalamud.Interface.Windowing;
 using MareSynchronos.UI;
 using MareSynchronos.Utils;
-using System.Runtime.InteropServices;
 using Dalamud.Game.ClientState.Conditions;
+using MareSynchronos.FileCache;
 
 namespace MareSynchronos
 {
@@ -27,7 +26,7 @@ namespace MareSynchronos
         private readonly PeriodicFileScanner _periodicFileScanner;
         private readonly IntroUi _introUi;
         private readonly IpcManager _ipcManager;
-        public static DalamudPluginInterface PluginInterface { get; set; }
+        private readonly DalamudPluginInterface _pluginInterface;
         private readonly SettingsUi _settingsUi;
         private readonly WindowSystem _windowSystem;
         private PlayerManager? _playerManager;
@@ -36,7 +35,7 @@ namespace MareSynchronos
         private OnlinePlayerManager? _characterCacheManager;
         private readonly DownloadUi _downloadUi;
         private readonly FileDialogManager _fileDialogManager;
-        private readonly FileDbManager _fileDbManager;
+        private readonly FileCacheManager _fileDbManager;
         private readonly CompactUi _compactUi;
         private readonly UiShared _uiSharedComponent;
         private readonly Dalamud.Localization _localization;
@@ -46,10 +45,10 @@ namespace MareSynchronos
             Framework framework, ObjectTable objectTable, ClientState clientState, Condition condition)
         {
             Logger.Debug("Launching " + Name);
-            PluginInterface = pluginInterface;
+            _pluginInterface = pluginInterface;
             _commandManager = commandManager;
-            _configuration = PluginInterface.GetPluginConfig() as Configuration ?? new Configuration();
+            _configuration = _pluginInterface.GetPluginConfig() as Configuration ?? new Configuration();
-            _configuration.Initialize(PluginInterface);
+            _configuration.Initialize(_pluginInterface);
             _configuration.Migrate();
 
             _localization = new Dalamud.Localization("MareSynchronos.Localization.", "", true);
@@ -57,19 +56,17 @@ namespace MareSynchronos
 
             _windowSystem = new WindowSystem("MareSynchronos");
 
-            new FileCacheContext().Dispose(); // make sure db is initialized I guess
-
             // those can be initialized outside of game login
             _dalamudUtil = new DalamudUtil(clientState, objectTable, framework, condition);
 
-            _ipcManager = new IpcManager(PluginInterface, _dalamudUtil);
+            _ipcManager = new IpcManager(_pluginInterface, _dalamudUtil);
             _fileDialogManager = new FileDialogManager();
-            _fileDbManager = new FileDbManager(_ipcManager, _configuration);
+            _fileDbManager = new FileCacheManager(_ipcManager, _configuration, _pluginInterface.ConfigDirectory.FullName);
             _apiController = new ApiController(_configuration, _dalamudUtil, _fileDbManager);
             _periodicFileScanner = new PeriodicFileScanner(_ipcManager, _configuration, _fileDbManager, _apiController, _dalamudUtil);
 
             _uiSharedComponent =
-                new UiShared(_ipcManager, _apiController, _periodicFileScanner, _fileDialogManager, _configuration, _dalamudUtil, PluginInterface, _localization);
+                new UiShared(_ipcManager, _apiController, _periodicFileScanner, _fileDialogManager, _configuration, _dalamudUtil, _pluginInterface, _localization);
             _settingsUi = new SettingsUi(_windowSystem, _uiSharedComponent, _configuration, _apiController);
             _compactUi = new CompactUi(_windowSystem, _uiSharedComponent, _configuration, _apiController);
 
@@ -120,6 +117,7 @@ namespace MareSynchronos
             _compactUi?.Dispose();
 
             _periodicFileScanner?.Dispose();
+            _fileDbManager?.Dispose();
             _playerManager?.Dispose();
             _characterCacheManager?.Dispose();
             _ipcManager?.Dispose();
@@ -133,8 +131,8 @@ namespace MareSynchronos
         {
             Logger.Debug("Client login");
 
-            PluginInterface.UiBuilder.Draw += Draw;
+            _pluginInterface.UiBuilder.Draw += Draw;
-            PluginInterface.UiBuilder.OpenConfigUi += OpenUi;
+            _pluginInterface.UiBuilder.OpenConfigUi += OpenUi;
             _commandManager.AddHandler(CommandName, new CommandInfo(OnCommand)
             {
                 HelpMessage = "Opens the Mare Synchronos UI"
@@ -157,8 +155,8 @@ namespace MareSynchronos
             _characterCacheManager?.Dispose();
             _playerManager?.Dispose();
             _transientResourceManager?.Dispose();
-            PluginInterface.UiBuilder.Draw -= Draw;
+            _pluginInterface.UiBuilder.Draw -= Draw;
-            PluginInterface.UiBuilder.OpenConfigUi -= OpenUi;
+            _pluginInterface.UiBuilder.OpenConfigUi -= OpenUi;
             _commandManager.RemoveHandler(CommandName);
         }
 

MareSynchronos/UI/IntroUI.cs

@@ -10,7 +10,7 @@ using ImGuiNET;
 using MareSynchronos.Utils;
 using MareSynchronos.Localization;
 using Dalamud.Utility;
-using MareSynchronos.FileCacheDB;
+using MareSynchronos.FileCache;
 
 namespace MareSynchronos.UI
 {

MareSynchronos/UI/UIShared.cs

@@ -11,7 +11,7 @@ using Dalamud.Interface.ImGuiFileDialog;
 using Dalamud.Plugin;
 using Dalamud.Utility;
 using ImGuiNET;
-using MareSynchronos.FileCacheDB;
+using MareSynchronos.FileCache;
 using MareSynchronos.Localization;
 using MareSynchronos.Managers;
 using MareSynchronos.Utils;

MareSynchronos/WebAPI/ApIController.Functions.Files.cs

@@ -9,7 +9,6 @@ using System.Threading;
 using System.Threading.Tasks;
 using LZ4;
 using MareSynchronos.API;
-using MareSynchronos.FileCacheDB;
 using MareSynchronos.Utils;
 using MareSynchronos.WebAPI.Utils;
 using Microsoft.AspNetCore.SignalR.Client;
@@ -177,7 +176,7 @@ namespace MareSynchronos.WebAPI
                     {
                         CurrentUploads.Add(new UploadFileTransfer(file)
                         {
-                            Total = new FileInfo(_fileDbManager.GetFileCacheByHash(file.Hash)!.Filepath).Length
+                            Total = new FileInfo(_fileDbManager.GetFileCacheByHash(file.Hash)!.ResolvedFilepath).Length
                         });
                     }
                     catch (Exception ex)
@@ -193,7 +192,7 @@ namespace MareSynchronos.WebAPI
                     {
                         ForbiddenTransfers.Add(new UploadFileTransfer(file)
                         {
-                            LocalFile = _fileDbManager.GetFileCacheByHash(file.Hash)?.Filepath ?? string.Empty
+                            LocalFile = _fileDbManager.GetFileCacheByHash(file.Hash)?.ResolvedFilepath ?? string.Empty
                         });
                     }
                 }
@@ -266,7 +265,7 @@ namespace MareSynchronos.WebAPI
 
         private async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken)
         {
-            var fileCache = _fileDbManager.GetFileCacheByHash(fileHash)!.Filepath;
+            var fileCache = _fileDbManager.GetFileCacheByHash(fileHash)!.ResolvedFilepath;
             return (fileHash, LZ4Codec.WrapHC(await File.ReadAllBytesAsync(fileCache, uploadToken), 0,
                 (int)new FileInfo(fileCache).Length));
         }

MareSynchronos/WebAPI/ApiController.Connectivity.cs

@@ -6,7 +6,7 @@ using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using MareSynchronos.API;
-using MareSynchronos.Managers;
+using MareSynchronos.FileCache;
 using MareSynchronos.Utils;
 using MareSynchronos.WebAPI.Utils;
 using Microsoft.AspNetCore.Http.Connections;
@@ -36,7 +36,7 @@ namespace MareSynchronos.WebAPI
 
         private readonly Configuration _pluginConfiguration;
         private readonly DalamudUtil _dalamudUtil;
-        private readonly FileDbManager _fileDbManager;
+        private readonly FileCacheManager _fileDbManager;
         private CancellationTokenSource _connectionCancellationTokenSource;
 
         private HubConnection? _mareHub;
@@ -49,7 +49,7 @@ namespace MareSynchronos.WebAPI
 
         public bool IsAdmin => _connectionDto?.IsAdmin ?? false;
 
-        public ApiController(Configuration pluginConfiguration, DalamudUtil dalamudUtil, FileDbManager fileDbManager)
+        public ApiController(Configuration pluginConfiguration, DalamudUtil dalamudUtil, FileCacheManager fileDbManager)
         {
             Logger.Verbose("Creating " + nameof(ApiController));