diff --git a/MareSynchronos/Factories/CharacterDataFactory.cs b/MareSynchronos/Factories/CharacterDataFactory.cs index c1b66d7..f7e767b 100644 --- a/MareSynchronos/Factories/CharacterDataFactory.cs +++ b/MareSynchronos/Factories/CharacterDataFactory.cs @@ -8,7 +8,7 @@ using FFXIVClientStructs.FFXIV.Client.Game.Character; using FFXIVClientStructs.FFXIV.Client.Graphics.Scene; using FFXIVClientStructs.FFXIV.Client.System.Resource; using MareSynchronos.API; -using MareSynchronos.FileCacheDB; +using MareSynchronos.FileCache; using MareSynchronos.Interop; using MareSynchronos.Managers; using MareSynchronos.Models; @@ -24,9 +24,9 @@ public class CharacterDataFactory private readonly DalamudUtil _dalamudUtil; private readonly IpcManager _ipcManager; private readonly TransientResourceManager transientResourceManager; - private readonly FileDbManager fileDbManager; + private readonly FileCacheManager fileDbManager; - public CharacterDataFactory(DalamudUtil dalamudUtil, IpcManager ipcManager, TransientResourceManager transientResourceManager, FileDbManager fileDbManager) + public CharacterDataFactory(DalamudUtil dalamudUtil, IpcManager ipcManager, TransientResourceManager transientResourceManager, FileCacheManager fileDbManager) { Logger.Verbose("Creating " + nameof(CharacterDataFactory)); this.fileDbManager = fileDbManager; diff --git a/MareSynchronos/FileCache/FileCache.cs b/MareSynchronos/FileCache/FileCache.cs new file mode 100644 index 0000000..f1f2ec4 --- /dev/null +++ b/MareSynchronos/FileCache/FileCache.cs @@ -0,0 +1,29 @@ +#nullable disable + + +using MareSynchronos; +using System.Globalization; + +namespace MareSynchronos.FileCache; + +public class FileCache +{ + public string ResolvedFilepath { get; private set; } + public string Hash { get; set; } + public string PrefixedFilePath { get; init; } + public string LastModifiedDateTicks { get; init; } + + public FileCache(string hash, string path, string lastModifiedDateTicks) + { + Hash = hash; + PrefixedFilePath = path; + LastModifiedDateTicks = lastModifiedDateTicks; + } + + public void SetResolvedFilePath(string filePath) + { + ResolvedFilepath = filePath.ToLowerInvariant().Replace("\\\\", "\\"); + } + + public string CsvEntry => $"{Hash}{FileCacheManager.CsvSplit}{PrefixedFilePath}{FileCacheManager.CsvSplit}{LastModifiedDateTicks.ToString(CultureInfo.InvariantCulture)}"; +} diff --git a/MareSynchronos/FileCache/FileDbManager.cs b/MareSynchronos/FileCache/FileDbManager.cs new file mode 100644 index 0000000..3cb5b6e --- /dev/null +++ b/MareSynchronos/FileCache/FileDbManager.cs @@ -0,0 +1,210 @@ +using MareSynchronos.Managers; +using MareSynchronos.Utils; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; + +namespace MareSynchronos.FileCache; + + +public enum FileState +{ + Valid, + RequireUpdate, + RequireDeletion +} + +public class FileCacheManager : IDisposable +{ + private const string PenumbraPrefix = "{penumbra}"; + private const string CachePrefix = "{cache}"; + private readonly IpcManager _ipcManager; + private readonly Configuration _configuration; + private readonly string CsvPath; + private string CsvBakPath => CsvPath + ".bak"; + private readonly ConcurrentDictionary FileCaches = new(); + public const string CsvSplit = "|"; + private object _fileWriteLock = new object(); + + public FileCacheManager(IpcManager ipcManager, Configuration configuration, string configDirectoryName) + { + _ipcManager = ipcManager; + _configuration = configuration; + CsvPath = Path.Combine(configDirectoryName, "FileCache.csv"); + + if (File.Exists(CsvBakPath)) + { + File.Move(CsvBakPath, CsvPath, true); + } + + if (File.Exists(CsvPath)) + { + var entries = File.ReadAllLines(CsvPath); + foreach (var entry in entries) + { + var splittedEntry = entry.Split(CsvSplit, StringSplitOptions.None); + var hash = splittedEntry[0]; + var path = splittedEntry[1]; + var time = splittedEntry[2]; + FileCaches[hash] = new FileCache(hash, path, time); + } + } + } + + public void WriteOutFullCsv() + { + StringBuilder sb = new StringBuilder(); + foreach (var entry in FileCaches) + { + sb.AppendLine(entry.Value.CsvEntry); + } + if (File.Exists(CsvPath)) + { + File.Copy(CsvPath, CsvBakPath, true); + } + lock (_fileWriteLock) + { + File.WriteAllText(CsvPath, sb.ToString()); + } + } + + public List GetAllFileCaches() => FileCaches.Values.ToList(); + + public FileCache? GetFileCacheByHash(string hash) + { + if (FileCaches.ContainsKey(hash)) + { + return GetValidatedFileCache(FileCaches[hash]); + } + + return null; + } + + public (FileState, FileCache) ValidateFileCacheEntity(FileCache fileCache) + { + fileCache = ReplacePathPrefixes(fileCache); + FileInfo fi = new(fileCache.ResolvedFilepath); + if (!fi.Exists) + { + return (FileState.RequireDeletion, fileCache); + } + if (fi.LastWriteTimeUtc.Ticks.ToString() != fileCache.LastModifiedDateTicks) + { + return (FileState.RequireUpdate, fileCache); + } + + return (FileState.Valid, fileCache); + } + + public FileCache? GetFileCacheByPath(string path) + { + var cleanedPath = path.Replace("/", "\\").ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), ""); + var entry = FileCaches.FirstOrDefault(f => f.Value.ResolvedFilepath.EndsWith(cleanedPath)).Value; + + if (entry == null) + { + Logger.Debug("Found no entries for " + cleanedPath); + return CreateFileEntry(path); + } + + var validatedCacheEntry = GetValidatedFileCache(entry); + + return validatedCacheEntry; + } + + public FileCache? CreateCacheEntry(string path) + { + Logger.Debug("Creating cache entry for " + path); + FileInfo fi = new(path); + if (!fi.Exists) return null; + var fullName = fi.FullName.ToLowerInvariant(); + if (!fullName.Contains(_configuration.CacheFolder.ToLowerInvariant())) return null; + string prefixedPath = fullName.Replace(_configuration.CacheFolder.ToLowerInvariant(), CachePrefix + "\\").Replace("\\\\", "\\"); + return CreateFileCacheEntity(fi, prefixedPath); + } + + public FileCache? CreateFileEntry(string path) + { + Logger.Debug("Creating file entry for " + path); + FileInfo fi = new(path); + if (!fi.Exists) return null; + var fullName = fi.FullName.ToLowerInvariant(); + if (!fullName.Contains(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant())) return null; + string prefixedPath = fullName.Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), PenumbraPrefix + "\\").Replace("\\\\", "\\"); + return CreateFileCacheEntity(fi, prefixedPath); + } + + private FileCache? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath) + { + var hash = Crypto.GetFileHash(fileInfo.FullName); + var entity = new FileCache(hash, prefixedPath, fileInfo.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture)); + FileCaches[hash] = entity; + lock (_fileWriteLock) + { + File.AppendAllLines(CsvPath, new[] { entity.CsvEntry }); + } + var result = GetFileCacheByPath(prefixedPath); + Logger.Debug("Creating file cache for " + fileInfo.FullName + " success: " + (result != null)); + return result; + } + + private FileCache? GetValidatedFileCache(FileCache fileCache) + { + var resulingFileCache = ReplacePathPrefixes(fileCache); + resulingFileCache = Validate(resulingFileCache); + return resulingFileCache; + } + + private FileCache? Validate(FileCache fileCache) + { + var file = new FileInfo(fileCache.ResolvedFilepath); + if (!file.Exists) + { + FileCaches.Remove(fileCache.Hash, out _); + return null; + } + + if (file.LastWriteTimeUtc.Ticks.ToString() != fileCache.LastModifiedDateTicks) + { + UpdateHash(fileCache); + } + + return fileCache; + } + + public void RemoveHash(FileCache entity) + { + FileCaches.Remove(entity.Hash, out _); + } + + public void UpdateHash(FileCache fileCache) + { + var prevHash = fileCache.Hash; + fileCache.Hash = Crypto.GetFileHash(fileCache.ResolvedFilepath); + FileCaches.Remove(prevHash, out _); + FileCaches[fileCache.Hash] = fileCache; + } + + private FileCache ReplacePathPrefixes(FileCache fileCache) + { + if (fileCache.PrefixedFilePath.StartsWith(PenumbraPrefix)) + { + fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory())); + } + else if (fileCache.PrefixedFilePath.StartsWith(CachePrefix)) + { + fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(CachePrefix, _configuration.CacheFolder)); + } + + return fileCache; + } + + public void Dispose() + { + WriteOutFullCsv(); + } +} diff --git a/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs b/MareSynchronos/FileCache/PeriodicFileScanner.cs similarity index 76% rename from MareSynchronos/FileCacheDB/PeriodicFileScanner.cs rename to MareSynchronos/FileCache/PeriodicFileScanner.cs index d723ecb..c83557a 100644 --- a/MareSynchronos/FileCacheDB/PeriodicFileScanner.cs +++ b/MareSynchronos/FileCache/PeriodicFileScanner.cs @@ -7,21 +7,20 @@ using System.Threading.Tasks; using MareSynchronos.Managers; using MareSynchronos.Utils; using MareSynchronos.WebAPI; -using Microsoft.EntityFrameworkCore; -namespace MareSynchronos.FileCacheDB; +namespace MareSynchronos.FileCache; public class PeriodicFileScanner : IDisposable { private readonly IpcManager _ipcManager; private readonly Configuration _pluginConfiguration; - private readonly FileDbManager _fileDbManager; + private readonly FileCacheManager _fileDbManager; private readonly ApiController _apiController; private readonly DalamudUtil _dalamudUtil; private int haltScanRequests = 0; private CancellationTokenSource? _scanCancellationTokenSource; private Task? _fileScannerTask = null; - public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileDbManager fileDbManager, ApiController apiController, DalamudUtil dalamudUtil) + public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileCacheManager fileDbManager, ApiController apiController, DalamudUtil dalamudUtil) { Logger.Verbose("Creating " + nameof(PeriodicFileScanner)); @@ -198,47 +197,33 @@ public class PeriodicFileScanner : IDisposable var cpuCount = (int)(Environment.ProcessorCount / 2.0f); Task[] dbTasks = Enumerable.Range(0, cpuCount).Select(c => Task.CompletedTask).ToArray(); - ConcurrentBag> entitiesToRemove = new(); - ConcurrentBag entitiesToUpdate = new(); + ConcurrentBag entitiesToRemove = new(); + ConcurrentBag entitiesToUpdate = new(); try { - using var ctx = new FileCacheContext(); - Logger.Debug("Database contains " + ctx.FileCaches.Count() + " files, local system contains " + TotalFiles); - using var cmd = ctx.Database.GetDbConnection().CreateCommand(); - cmd.CommandText = "SELECT Hash, FilePath, LastModifiedDate FROM FileCache"; - ctx.Database.OpenConnection(); - using var reader = cmd.ExecuteReader(); - while (reader.Read()) + foreach (var value in _fileDbManager.GetAllFileCaches()) { - var hash = reader["Hash"].ToString(); - var filePath = reader["FilePath"].ToString(); - var date = reader["LastModifiedDate"].ToString(); var idx = Task.WaitAny(dbTasks, ct); dbTasks[idx] = Task.Run(() => { try { - var fileState = _fileDbManager.ValidateFileCacheEntity(hash, filePath, date); - switch (fileState.Item1) + var result = _fileDbManager.ValidateFileCacheEntity(value); + scannedFiles[result.Item2.ResolvedFilepath] = true; + if (result.Item1 == FileState.RequireUpdate) { - case FileState.Valid: - scannedFiles[fileState.Item2] = true; - break; - case FileState.RequireDeletion: - entitiesToRemove.Add(new Tuple(hash, filePath)); - break; - case FileState.RequireUpdate: - scannedFiles[fileState.Item2] = true; - entitiesToUpdate.Add(fileState.Item2); - break; + entitiesToUpdate.Add(result.Item2); + } + else if (result.Item1 == FileState.RequireDeletion) + { + entitiesToRemove.Add(result.Item2); } } catch (Exception ex) { - Logger.Warn("Failed validating " + filePath); + Logger.Warn("Failed validating " + value.ResolvedFilepath); Logger.Warn(ex.Message); Logger.Warn(ex.StackTrace); - entitiesToRemove.Add(new Tuple(hash, filePath)); } Interlocked.Increment(ref currentFileProgress); @@ -257,44 +242,23 @@ public class PeriodicFileScanner : IDisposable Logger.Warn("Error during enumerating FileCaches: " + ex.Message); } - using (var db = new FileCacheContext()) - { - try - { - if (entitiesToRemove.Any()) - { - foreach (var entry in entitiesToRemove) - { - Logger.Debug("Removing " + entry.Item2); - var toRemove = db.FileCaches.First(f => f.Filepath == entry.Item2 && f.Hash == entry.Item1); - db.FileCaches.Remove(toRemove); - } - - } - - if (entitiesToUpdate.Any()) - { - foreach (var entry in entitiesToUpdate) - { - Logger.Debug("Updating " + entry); - _fileDbManager.GetFileCacheByPath(entry); - } - } - - if (entitiesToUpdate.Any() || entitiesToRemove.Any()) - { - db.SaveChanges(); - } - - } - catch (Exception ex) - { - Logger.Warn(ex.Message); - } - } - Task.WaitAll(dbTasks); + if (entitiesToUpdate.Any() || entitiesToRemove.Any()) + { + foreach (var entity in entitiesToUpdate) + { + _fileDbManager.UpdateHash(entity); + } + + foreach (var entity in entitiesToRemove) + { + _fileDbManager.RemoveHash(entity); + } + + _fileDbManager.WriteOutFullCsv(); + } + Logger.Debug("Scanner validated existing db files"); if (ct.IsCancellationRequested) return; diff --git a/MareSynchronos/FileCacheDB/FileCache.cs b/MareSynchronos/FileCacheDB/FileCache.cs deleted file mode 100644 index cc5cfee..0000000 --- a/MareSynchronos/FileCacheDB/FileCache.cs +++ /dev/null @@ -1,48 +0,0 @@ -#nullable disable - - -using System; - -namespace MareSynchronos.FileCacheDB -{ - - public class FileCache - { - private FileCacheEntity entity; - public string Filepath { get; private set; } - public string Hash { get; private set; } - private string originalFilePathNoEntity = string.Empty; - private string originalHashNoEntity = string.Empty; - private string originalModifiedDate = string.Empty; - public string OriginalFilepath => entity == null ? originalFilePathNoEntity : entity.Filepath; - public string OriginalHash => entity == null ? originalHashNoEntity : entity.Hash; - public long LastModifiedDateTicks => long.Parse(entity == null ? originalModifiedDate : entity.LastModifiedDate); - - public FileCache(string hash, string path, string lastModifiedDate) - { - originalHashNoEntity = hash; - originalFilePathNoEntity = path; - originalModifiedDate = lastModifiedDate; - } - - public FileCache(FileCacheEntity entity) - { - this.entity = entity; - } - - public void SetResolvedFilePath(string filePath) - { - Filepath = filePath.ToLowerInvariant().Replace("\\\\", "\\"); - } - - public void SetHash(string hash) - { - Hash = hash; - } - - public void UpdateFileCache(FileCacheEntity entity) - { - this.entity = entity; - } - } -} diff --git a/MareSynchronos/FileCacheDB/FileCacheContext.cs b/MareSynchronos/FileCacheDB/FileCacheContext.cs deleted file mode 100644 index 9ccc9a5..0000000 --- a/MareSynchronos/FileCacheDB/FileCacheContext.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System.IO; -using Microsoft.EntityFrameworkCore; - -#nullable disable - -namespace MareSynchronos.FileCacheDB -{ - public partial class FileCacheContext : DbContext - { - private string DbPath { get; set; } - public FileCacheContext() - { - DbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "FileCache.db"); - Database.EnsureCreated(); - } - - public FileCacheContext(DbContextOptions options) - : base(options) - { - } - - public virtual DbSet FileCaches { get; set; } - - protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) - { - if (!optionsBuilder.IsConfigured) - { - optionsBuilder.UseSqlite("Data Source=" + DbPath+";Cache=Shared"); - } - } - - protected override void OnModelCreating(ModelBuilder modelBuilder) - { - modelBuilder.Entity(entity => - { - entity.HasKey(e => new { e.Hash, e.Filepath }); - - entity.ToTable("FileCache"); - }); - - OnModelCreatingPartial(modelBuilder); - } - - partial void OnModelCreatingPartial(ModelBuilder modelBuilder); - } -} diff --git a/MareSynchronos/FileCacheDB/FileCacheEntity.cs b/MareSynchronos/FileCacheDB/FileCacheEntity.cs deleted file mode 100644 index e0deffc..0000000 --- a/MareSynchronos/FileCacheDB/FileCacheEntity.cs +++ /dev/null @@ -1,13 +0,0 @@ -#nullable disable - - -namespace MareSynchronos.FileCacheDB -{ - public partial class FileCacheEntity - { - public string Hash { get; set; } - public string Filepath { get; set; } - public string LastModifiedDate { get; set; } - public int Version { get; set; } - } -} diff --git a/MareSynchronos/FileCacheDB/FileDbManager.cs b/MareSynchronos/FileCacheDB/FileDbManager.cs deleted file mode 100644 index a54d516..0000000 --- a/MareSynchronos/FileCacheDB/FileDbManager.cs +++ /dev/null @@ -1,284 +0,0 @@ -using MareSynchronos.FileCacheDB; -using MareSynchronos.Utils; -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; - -namespace MareSynchronos.Managers; - -public enum FileState -{ - Valid, - RequireUpdate, - RequireDeletion -} - -public class FileDbManager -{ - private const string PenumbraPrefix = "{penumbra}"; - private const string CachePrefix = "{cache}"; - private readonly IpcManager _ipcManager; - private readonly Configuration _configuration; - private static object _lock = new(); - - public FileDbManager(IpcManager ipcManager, Configuration configuration) - { - _ipcManager = ipcManager; - _configuration = configuration; - } - - public FileCache? GetFileCacheByHash(string hash) - { - List matchingEntries = new List(); - using (var db = new FileCacheContext()) - { - matchingEntries = db.FileCaches.Where(f => f.Hash.ToLower() == hash.ToLower()).ToList(); - } - - if (!matchingEntries.Any()) return null; - - if (matchingEntries.Any(f => f.Filepath.Contains(PenumbraPrefix) && matchingEntries.Any(f => f.Filepath.Contains(CachePrefix)))) - { - var cachedEntries = matchingEntries.Where(f => f.Filepath.Contains(CachePrefix)).ToList(); - DeleteFromDatabase(cachedEntries.Select(f => new FileCache(f))); - foreach (var entry in cachedEntries) - { - matchingEntries.Remove(entry); - } - } - - return GetValidatedFileCache(new FileCache(matchingEntries.First())); - } - - public (FileState, string) ValidateFileCacheEntity(string hash, string path, string lastModifiedDate) - { - var fileCache = new FileCache(hash, path, lastModifiedDate); - if (!fileCache.OriginalFilepath.StartsWith(PenumbraPrefix + "\\") && !fileCache.OriginalFilepath.StartsWith(CachePrefix)) - return (FileState.RequireUpdate, path); - fileCache = ReplacePathPrefixes(fileCache); - FileInfo fi = new FileInfo(fileCache.Filepath); - if (!fi.Exists) - return (FileState.RequireDeletion, fileCache.Filepath); - if (fi.LastWriteTimeUtc.Ticks != fileCache.LastModifiedDateTicks) - return (FileState.RequireUpdate, fileCache.Filepath); - - return (FileState.Valid, fileCache.Filepath); - } - - public FileCache? GetFileCacheByPath(string path) - { - FileCacheEntity? matchingEntries = null; - var cleanedPath = path.Replace("/", "\\").ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), ""); - using (var db = new FileCacheContext()) - { - matchingEntries = db.FileCaches.FirstOrDefault(f => f.Filepath.EndsWith(cleanedPath)); - } - - if (matchingEntries == null) - { - Logger.Debug("Found no entries for " + cleanedPath); - return CreateFileEntry(path); - } - - var validatedCacheEntry = GetValidatedFileCache(new FileCache(matchingEntries)); - - return validatedCacheEntry; - } - - public FileCache? CreateCacheEntry(string path) - { - Logger.Debug("Creating cache entry for " + path); - FileInfo fi = new FileInfo(path); - if (!fi.Exists) return null; - string prefixedPath = fi.FullName.ToLowerInvariant().Replace(_configuration.CacheFolder.ToLowerInvariant(), CachePrefix + "\\").Replace("\\\\", "\\"); - return CreateFileCacheEntity(fi, prefixedPath); - } - - public FileCache? CreateFileEntry(string path) - { - Logger.Debug("Creating file entry for " + path); - FileInfo fi = new FileInfo(path); - if (!fi.Exists) return null; - string prefixedPath = fi.FullName.ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), PenumbraPrefix + "\\").Replace("\\\\", "\\"); - return CreateFileCacheEntity(fi, prefixedPath); - } - - private FileCache? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath) - { - var hash = Crypto.GetFileHash(fileInfo.FullName); - lock (_lock) - { - var entity = new FileCacheEntity(); - entity.Hash = hash; - entity.Filepath = prefixedPath; - entity.LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture); - try - { - using var db = new FileCacheContext(); - db.FileCaches.Add(entity); - db.SaveChanges(); - } - catch (Exception ex) - { - Logger.Warn("Could not add " + fileInfo.FullName ?? String.Empty); - Logger.Warn(ex.Message); - } - } - var result = GetFileCacheByPath(prefixedPath); - Logger.Debug("Creating file cache for " + fileInfo.FullName + " success: " + (result != null)); - return result; - } - - private FileCache? GetValidatedFileCache(FileCache fileCache) - { - var resulingFileCache = MigrateLegacy(fileCache); - if (resulingFileCache == null) return null; - - resulingFileCache = ReplacePathPrefixes(resulingFileCache); - resulingFileCache = Validate(resulingFileCache); - return resulingFileCache; - } - - private FileCache? Validate(FileCache fileCache) - { - var file = new FileInfo(fileCache.Filepath); - if (!file.Exists) - { - DeleteFromDatabase(new[] { fileCache }); - return null; - } - - if (file.LastWriteTimeUtc.Ticks != fileCache.LastModifiedDateTicks) - { - fileCache.SetHash(Crypto.GetFileHash(fileCache.Filepath)); - UpdateCacheHash(fileCache, file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture)); - } - - return fileCache; - } - - private FileCache? MigrateLegacy(FileCache fileCache) - { - if (fileCache.OriginalFilepath.StartsWith(PenumbraPrefix + "\\") || fileCache.OriginalFilepath.StartsWith(CachePrefix)) return fileCache; - - var fileInfo = new FileInfo(fileCache.OriginalFilepath); - var penumbraDir = _ipcManager.PenumbraModDirectory()!; - if (penumbraDir.Last() != '\\') penumbraDir += "\\"; - // check if it's a cache file - if (fileInfo.Exists && fileInfo.Name.Length == 40) - { - MigrateLegacyFilePath(fileCache, CachePrefix + "\\" + fileInfo.Name.ToLower()); - } - else if (fileInfo.Exists && fileInfo.FullName.ToLowerInvariant().Contains(penumbraDir)) - { - // attempt to replace penumbra mod folder path with {penumbra} - var newPath = PenumbraPrefix + "\\" + fileCache.OriginalFilepath.ToLowerInvariant().Replace(penumbraDir, string.Empty); - MigrateLegacyFilePath(fileCache, newPath); - } - else if (fileInfo.FullName.ToLowerInvariant().Contains(PenumbraPrefix)) - { - var newPath = PenumbraPrefix + "\\" + fileCache.OriginalFilepath.ToLowerInvariant().Replace(PenumbraPrefix, string.Empty); - MigrateLegacyFilePath(fileCache, newPath); - } - else - { - DeleteFromDatabase(new[] { fileCache }); - return null; - } - - return fileCache; - } - - private FileCache ReplacePathPrefixes(FileCache fileCache) - { - if (fileCache.OriginalFilepath.StartsWith(PenumbraPrefix)) - { - fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory())); - } - else if (fileCache.OriginalFilepath.StartsWith(CachePrefix)) - { - fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(CachePrefix, _configuration.CacheFolder)); - } - - return fileCache; - } - - private void UpdateCacheHash(FileCache markedForUpdate, string lastModifiedDate) - { - lock (_lock) - { - try - { - Logger.Verbose("Updating Hash for " + markedForUpdate.OriginalFilepath); - using var db = new FileCacheContext(); - var cache = db.FileCaches.First(f => f.Filepath == markedForUpdate.OriginalFilepath && f.Hash == markedForUpdate.OriginalHash); - var newcache = new FileCacheEntity() - { - Filepath = cache.Filepath, - Hash = markedForUpdate.Hash, - LastModifiedDate = lastModifiedDate - }; - db.Remove(cache); - db.FileCaches.Add(newcache); - markedForUpdate.UpdateFileCache(newcache); - db.SaveChanges(); - } - catch (Exception ex) - { - Logger.Warn("Error updating file hash (" + ex.Message + "), returning currently existing" ?? string.Empty); - Logger.Warn(ex.InnerException?.Message ?? string.Empty); - Logger.Warn(ex.StackTrace ?? string.Empty); - using var db = new FileCacheContext(); - var cache = db.FileCaches.First(f => f.Filepath == markedForUpdate.OriginalFilepath); - markedForUpdate.UpdateFileCache(cache); - } - } - } - - private void MigrateLegacyFilePath(FileCache fileCacheToMigrate, string newPath) - { - lock (_lock) - { - Logger.Verbose("Migrating legacy file path for " + fileCacheToMigrate.OriginalFilepath); - using var db = new FileCacheContext(); - var cache = db.FileCaches.First(f => f.Filepath == fileCacheToMigrate.OriginalFilepath && f.Hash == fileCacheToMigrate.OriginalHash); - var newcache = new FileCacheEntity() - { - Filepath = newPath, - Hash = cache.Hash, - LastModifiedDate = cache.LastModifiedDate - }; - db.Remove(cache); - db.SaveChanges(); - var existingCache = db.FileCaches.FirstOrDefault(f => f.Filepath == newPath && f.Hash == cache.Hash); - if (existingCache != null) - { - fileCacheToMigrate.UpdateFileCache(existingCache); - } - else - { - db.FileCaches.Add(newcache); - fileCacheToMigrate.UpdateFileCache(newcache); - db.SaveChanges(); - } - } - } - - private void DeleteFromDatabase(IEnumerable markedForDeletion) - { - lock (_lock) - { - using var db = new FileCacheContext(); - foreach (var item in markedForDeletion) - { - Logger.Verbose("Removing " + item.OriginalFilepath); - var itemToRemove = db.FileCaches.FirstOrDefault(f => f.Hash == item.OriginalHash && f.Filepath == item.OriginalFilepath); - if (itemToRemove == null) continue; - db.FileCaches.Remove(itemToRemove); - } - db.SaveChanges(); - } - } -} diff --git a/MareSynchronos/Managers/CachedPlayer.cs b/MareSynchronos/Managers/CachedPlayer.cs index b12cc07..58d28f4 100644 --- a/MareSynchronos/Managers/CachedPlayer.cs +++ b/MareSynchronos/Managers/CachedPlayer.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Dalamud.Logging; using FFXIVClientStructs.FFXIV.Client.Game.Character; using MareSynchronos.API; +using MareSynchronos.FileCache; using MareSynchronos.Models; using MareSynchronos.Utils; using MareSynchronos.WebAPI; @@ -15,12 +16,12 @@ namespace MareSynchronos.Managers; public class CachedPlayer { private readonly DalamudUtil _dalamudUtil; - private readonly FileDbManager fileDbManager; + private readonly FileCacheManager fileDbManager; private readonly IpcManager _ipcManager; private readonly ApiController _apiController; private bool _isVisible; - public CachedPlayer(string nameHash, IpcManager ipcManager, ApiController apiController, DalamudUtil dalamudUtil, FileDbManager fileDbManager) + public CachedPlayer(string nameHash, IpcManager ipcManager, ApiController apiController, DalamudUtil dalamudUtil, FileCacheManager fileDbManager) { PlayerNameHash = nameHash; _ipcManager = ipcManager; @@ -201,7 +202,7 @@ public class CachedPlayer var fileCache = fileDbManager.GetFileCacheByHash(item.Hash); if (fileCache != null) { - moddedDictionary[gamePath] = fileCache.Filepath; + moddedDictionary[gamePath] = fileCache.ResolvedFilepath; } else { diff --git a/MareSynchronos/Managers/OnlinePlayerManager.cs b/MareSynchronos/Managers/OnlinePlayerManager.cs index e648584..3b99e51 100644 --- a/MareSynchronos/Managers/OnlinePlayerManager.cs +++ b/MareSynchronos/Managers/OnlinePlayerManager.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using MareSynchronos.API; +using MareSynchronos.FileCache; using MareSynchronos.Utils; using MareSynchronos.WebAPI; using MareSynchronos.WebAPI.Utils; @@ -17,7 +18,7 @@ public class OnlinePlayerManager : IDisposable private readonly DalamudUtil _dalamudUtil; private readonly IpcManager _ipcManager; private readonly PlayerManager _playerManager; - private readonly FileDbManager _fileDbManager; + private readonly FileCacheManager _fileDbManager; private readonly ConcurrentDictionary _onlineCachedPlayers = new(); private readonly ConcurrentDictionary _temporaryStoredCharacterCache = new(); private readonly ConcurrentDictionary _playerTokenDisposal = new(); @@ -25,7 +26,7 @@ public class OnlinePlayerManager : IDisposable private List OnlineVisiblePlayerHashes => _onlineCachedPlayers.Select(p => p.Value).Where(p => p.PlayerCharacter != IntPtr.Zero) .Select(p => p.PlayerNameHash).ToList(); - public OnlinePlayerManager(ApiController apiController, DalamudUtil dalamudUtil, IpcManager ipcManager, PlayerManager playerManager, FileDbManager fileDbManager) + public OnlinePlayerManager(ApiController apiController, DalamudUtil dalamudUtil, IpcManager ipcManager, PlayerManager playerManager, FileCacheManager fileDbManager) { Logger.Verbose("Creating " + nameof(OnlinePlayerManager)); diff --git a/MareSynchronos/Managers/PlayerManager.cs b/MareSynchronos/Managers/PlayerManager.cs index 3db33fd..dd3481f 100644 --- a/MareSynchronos/Managers/PlayerManager.cs +++ b/MareSynchronos/Managers/PlayerManager.cs @@ -9,7 +9,7 @@ using FFXIVClientStructs.FFXIV.Client.Game.Character; using System.Collections.Generic; using System.Linq; using MareSynchronos.Models; -using MareSynchronos.FileCacheDB; +using MareSynchronos.FileCache; #if DEBUG using Newtonsoft.Json; #endif diff --git a/MareSynchronos/MareSynchronos.csproj b/MareSynchronos/MareSynchronos.csproj index 747b75a..662d33b 100644 --- a/MareSynchronos/MareSynchronos.csproj +++ b/MareSynchronos/MareSynchronos.csproj @@ -29,7 +29,6 @@ - diff --git a/MareSynchronos/Models/FileReplacement.cs b/MareSynchronos/Models/FileReplacement.cs index 495d96a..d4b2406 100644 --- a/MareSynchronos/Models/FileReplacement.cs +++ b/MareSynchronos/Models/FileReplacement.cs @@ -4,15 +4,15 @@ using System.Text; using System.Threading.Tasks; using MareSynchronos.API; using System.Text.RegularExpressions; -using MareSynchronos.Managers; +using MareSynchronos.FileCache; namespace MareSynchronos.Models { public class FileReplacement { - private readonly FileDbManager fileDbManager; + private readonly FileCacheManager fileDbManager; - public FileReplacement(FileDbManager fileDbManager) + public FileReplacement(FileCacheManager fileDbManager) { this.fileDbManager = fileDbManager; } @@ -37,7 +37,7 @@ namespace MareSynchronos.Models _ = Task.Run(() => { var cache = fileDbManager.GetFileCacheByPath(ResolvedPath); - Hash = cache.OriginalHash; + Hash = cache.Hash; }); } diff --git a/MareSynchronos/Plugin.cs b/MareSynchronos/Plugin.cs index 6760e1e..8db6542 100644 --- a/MareSynchronos/Plugin.cs +++ b/MareSynchronos/Plugin.cs @@ -1,6 +1,5 @@ using Dalamud.Game.Command; using Dalamud.Plugin; -using MareSynchronos.FileCacheDB; using MareSynchronos.Factories; using System.Threading.Tasks; using Dalamud.Game; @@ -13,8 +12,8 @@ using MareSynchronos.WebAPI; using Dalamud.Interface.Windowing; using MareSynchronos.UI; using MareSynchronos.Utils; -using System.Runtime.InteropServices; using Dalamud.Game.ClientState.Conditions; +using MareSynchronos.FileCache; namespace MareSynchronos { @@ -27,7 +26,7 @@ namespace MareSynchronos private readonly PeriodicFileScanner _periodicFileScanner; private readonly IntroUi _introUi; private readonly IpcManager _ipcManager; - public static DalamudPluginInterface PluginInterface { get; set; } + private readonly DalamudPluginInterface _pluginInterface; private readonly SettingsUi _settingsUi; private readonly WindowSystem _windowSystem; private PlayerManager? _playerManager; @@ -36,7 +35,7 @@ namespace MareSynchronos private OnlinePlayerManager? _characterCacheManager; private readonly DownloadUi _downloadUi; private readonly FileDialogManager _fileDialogManager; - private readonly FileDbManager _fileDbManager; + private readonly FileCacheManager _fileDbManager; private readonly CompactUi _compactUi; private readonly UiShared _uiSharedComponent; private readonly Dalamud.Localization _localization; @@ -46,10 +45,10 @@ namespace MareSynchronos Framework framework, ObjectTable objectTable, ClientState clientState, Condition condition) { Logger.Debug("Launching " + Name); - PluginInterface = pluginInterface; + _pluginInterface = pluginInterface; _commandManager = commandManager; - _configuration = PluginInterface.GetPluginConfig() as Configuration ?? new Configuration(); - _configuration.Initialize(PluginInterface); + _configuration = _pluginInterface.GetPluginConfig() as Configuration ?? new Configuration(); + _configuration.Initialize(_pluginInterface); _configuration.Migrate(); _localization = new Dalamud.Localization("MareSynchronos.Localization.", "", true); @@ -57,19 +56,17 @@ namespace MareSynchronos _windowSystem = new WindowSystem("MareSynchronos"); - new FileCacheContext().Dispose(); // make sure db is initialized I guess - // those can be initialized outside of game login _dalamudUtil = new DalamudUtil(clientState, objectTable, framework, condition); - _ipcManager = new IpcManager(PluginInterface, _dalamudUtil); + _ipcManager = new IpcManager(_pluginInterface, _dalamudUtil); _fileDialogManager = new FileDialogManager(); - _fileDbManager = new FileDbManager(_ipcManager, _configuration); + _fileDbManager = new FileCacheManager(_ipcManager, _configuration, _pluginInterface.ConfigDirectory.FullName); _apiController = new ApiController(_configuration, _dalamudUtil, _fileDbManager); _periodicFileScanner = new PeriodicFileScanner(_ipcManager, _configuration, _fileDbManager, _apiController, _dalamudUtil); _uiSharedComponent = - new UiShared(_ipcManager, _apiController, _periodicFileScanner, _fileDialogManager, _configuration, _dalamudUtil, PluginInterface, _localization); + new UiShared(_ipcManager, _apiController, _periodicFileScanner, _fileDialogManager, _configuration, _dalamudUtil, _pluginInterface, _localization); _settingsUi = new SettingsUi(_windowSystem, _uiSharedComponent, _configuration, _apiController); _compactUi = new CompactUi(_windowSystem, _uiSharedComponent, _configuration, _apiController); @@ -120,6 +117,7 @@ namespace MareSynchronos _compactUi?.Dispose(); _periodicFileScanner?.Dispose(); + _fileDbManager?.Dispose(); _playerManager?.Dispose(); _characterCacheManager?.Dispose(); _ipcManager?.Dispose(); @@ -133,8 +131,8 @@ namespace MareSynchronos { Logger.Debug("Client login"); - PluginInterface.UiBuilder.Draw += Draw; - PluginInterface.UiBuilder.OpenConfigUi += OpenUi; + _pluginInterface.UiBuilder.Draw += Draw; + _pluginInterface.UiBuilder.OpenConfigUi += OpenUi; _commandManager.AddHandler(CommandName, new CommandInfo(OnCommand) { HelpMessage = "Opens the Mare Synchronos UI" @@ -157,8 +155,8 @@ namespace MareSynchronos _characterCacheManager?.Dispose(); _playerManager?.Dispose(); _transientResourceManager?.Dispose(); - PluginInterface.UiBuilder.Draw -= Draw; - PluginInterface.UiBuilder.OpenConfigUi -= OpenUi; + _pluginInterface.UiBuilder.Draw -= Draw; + _pluginInterface.UiBuilder.OpenConfigUi -= OpenUi; _commandManager.RemoveHandler(CommandName); } diff --git a/MareSynchronos/UI/IntroUI.cs b/MareSynchronos/UI/IntroUI.cs index 187e95f..e15d85b 100644 --- a/MareSynchronos/UI/IntroUI.cs +++ b/MareSynchronos/UI/IntroUI.cs @@ -10,7 +10,7 @@ using ImGuiNET; using MareSynchronos.Utils; using MareSynchronos.Localization; using Dalamud.Utility; -using MareSynchronos.FileCacheDB; +using MareSynchronos.FileCache; namespace MareSynchronos.UI { diff --git a/MareSynchronos/UI/UIShared.cs b/MareSynchronos/UI/UIShared.cs index d5a42f6..c9634d0 100644 --- a/MareSynchronos/UI/UIShared.cs +++ b/MareSynchronos/UI/UIShared.cs @@ -11,7 +11,7 @@ using Dalamud.Interface.ImGuiFileDialog; using Dalamud.Plugin; using Dalamud.Utility; using ImGuiNET; -using MareSynchronos.FileCacheDB; +using MareSynchronos.FileCache; using MareSynchronos.Localization; using MareSynchronos.Managers; using MareSynchronos.Utils; diff --git a/MareSynchronos/WebAPI/ApIController.Functions.Files.cs b/MareSynchronos/WebAPI/ApIController.Functions.Files.cs index f183992..82a480e 100644 --- a/MareSynchronos/WebAPI/ApIController.Functions.Files.cs +++ b/MareSynchronos/WebAPI/ApIController.Functions.Files.cs @@ -9,7 +9,6 @@ using System.Threading; using System.Threading.Tasks; using LZ4; using MareSynchronos.API; -using MareSynchronos.FileCacheDB; using MareSynchronos.Utils; using MareSynchronos.WebAPI.Utils; using Microsoft.AspNetCore.SignalR.Client; @@ -177,7 +176,7 @@ namespace MareSynchronos.WebAPI { CurrentUploads.Add(new UploadFileTransfer(file) { - Total = new FileInfo(_fileDbManager.GetFileCacheByHash(file.Hash)!.Filepath).Length + Total = new FileInfo(_fileDbManager.GetFileCacheByHash(file.Hash)!.ResolvedFilepath).Length }); } catch (Exception ex) @@ -193,7 +192,7 @@ namespace MareSynchronos.WebAPI { ForbiddenTransfers.Add(new UploadFileTransfer(file) { - LocalFile = _fileDbManager.GetFileCacheByHash(file.Hash)?.Filepath ?? string.Empty + LocalFile = _fileDbManager.GetFileCacheByHash(file.Hash)?.ResolvedFilepath ?? string.Empty }); } } @@ -266,7 +265,7 @@ namespace MareSynchronos.WebAPI private async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken) { - var fileCache = _fileDbManager.GetFileCacheByHash(fileHash)!.Filepath; + var fileCache = _fileDbManager.GetFileCacheByHash(fileHash)!.ResolvedFilepath; return (fileHash, LZ4Codec.WrapHC(await File.ReadAllBytesAsync(fileCache, uploadToken), 0, (int)new FileInfo(fileCache).Length)); } diff --git a/MareSynchronos/WebAPI/ApiController.Connectivity.cs b/MareSynchronos/WebAPI/ApiController.Connectivity.cs index 5a22302..6088d99 100644 --- a/MareSynchronos/WebAPI/ApiController.Connectivity.cs +++ b/MareSynchronos/WebAPI/ApiController.Connectivity.cs @@ -6,7 +6,7 @@ using System.Net.Http; using System.Threading; using System.Threading.Tasks; using MareSynchronos.API; -using MareSynchronos.Managers; +using MareSynchronos.FileCache; using MareSynchronos.Utils; using MareSynchronos.WebAPI.Utils; using Microsoft.AspNetCore.Http.Connections; @@ -36,7 +36,7 @@ namespace MareSynchronos.WebAPI private readonly Configuration _pluginConfiguration; private readonly DalamudUtil _dalamudUtil; - private readonly FileDbManager _fileDbManager; + private readonly FileCacheManager _fileDbManager; private CancellationTokenSource _connectionCancellationTokenSource; private HubConnection? _mareHub; @@ -49,7 +49,7 @@ namespace MareSynchronos.WebAPI public bool IsAdmin => _connectionDto?.IsAdmin ?? false; - public ApiController(Configuration pluginConfiguration, DalamudUtil dalamudUtil, FileDbManager fileDbManager) + public ApiController(Configuration pluginConfiguration, DalamudUtil dalamudUtil, FileCacheManager fileDbManager) { Logger.Verbose("Creating " + nameof(ApiController));