t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove database, use in-memory

Browse Source
This commit is contained in:
Stanley Dimant
2022-09-28 15:44:29 +02:00
parent 287c1b0eff
commit c2e92c094c
18 changed files with 306 additions and 496 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
MareSynchronos/Factories/CharacterDataFactory.cs
View File

@@ -8,7 +8,7 @@ using FFXIVClientStructs.FFXIV.Client.Game.Character;
using FFXIVClientStructs.FFXIV.Client.Graphics.Scene;
using FFXIVClientStructs.FFXIV.Client.System.Resource;
using MareSynchronos.API;
using MareSynchronos.FileCacheDB;
using MareSynchronos.FileCache;
using MareSynchronos.Interop;
using MareSynchronos.Managers;
using MareSynchronos.Models;
@@ -24,9 +24,9 @@ public class CharacterDataFactory
private readonly DalamudUtil _dalamudUtil;
private readonly IpcManager _ipcManager;
private readonly TransientResourceManager transientResourceManager;
private readonly FileDbManager fileDbManager;
private readonly FileCacheManager fileDbManager;
public CharacterDataFactory(DalamudUtil dalamudUtil, IpcManager ipcManager, TransientResourceManager transientResourceManager, FileDbManager fileDbManager)
public CharacterDataFactory(DalamudUtil dalamudUtil, IpcManager ipcManager, TransientResourceManager transientResourceManager, FileCacheManager fileDbManager)
{
Logger.Verbose("Creating " + nameof(CharacterDataFactory));
this.fileDbManager = fileDbManager;

29
MareSynchronos/FileCache/FileCache.cs Normal file
View File

@@ -0,0 +1,29 @@
#nullable disable
using MareSynchronos;
using System.Globalization;
namespace MareSynchronos.FileCache;
public class FileCache
{
public string ResolvedFilepath { get; private set; }
public string Hash { get; set; }
public string PrefixedFilePath { get; init; }
public string LastModifiedDateTicks { get; init; }
public FileCache(string hash, string path, string lastModifiedDateTicks)
{
Hash = hash;
PrefixedFilePath = path;
LastModifiedDateTicks = lastModifiedDateTicks;
}
public void SetResolvedFilePath(string filePath)
{
ResolvedFilepath = filePath.ToLowerInvariant().Replace("\\\\", "\\");
}
public string CsvEntry => $"{Hash}{FileCacheManager.CsvSplit}{PrefixedFilePath}{FileCacheManager.CsvSplit}{LastModifiedDateTicks.ToString(CultureInfo.InvariantCulture)}";
}

210
MareSynchronos/FileCache/FileDbManager.cs Normal file
View File

@@ -0,0 +1,210 @@
using MareSynchronos.Managers;
using MareSynchronos.Utils;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
namespace MareSynchronos.FileCache;
public enum FileState
{
Valid,
RequireUpdate,
RequireDeletion
}
public class FileCacheManager : IDisposable
{
private const string PenumbraPrefix = "{penumbra}";
private const string CachePrefix = "{cache}";
private readonly IpcManager _ipcManager;
private readonly Configuration _configuration;
private readonly string CsvPath;
private string CsvBakPath => CsvPath + ".bak";
private readonly ConcurrentDictionary<string, FileCache> FileCaches = new();
public const string CsvSplit = "|";
private object _fileWriteLock = new object();
public FileCacheManager(IpcManager ipcManager, Configuration configuration, string configDirectoryName)
{
_ipcManager = ipcManager;
_configuration = configuration;
CsvPath = Path.Combine(configDirectoryName, "FileCache.csv");
if (File.Exists(CsvBakPath))
{
File.Move(CsvBakPath, CsvPath, true);
}
if (File.Exists(CsvPath))
{
var entries = File.ReadAllLines(CsvPath);
foreach (var entry in entries)
{
var splittedEntry = entry.Split(CsvSplit, StringSplitOptions.None);
var hash = splittedEntry[0];
var path = splittedEntry[1];
var time = splittedEntry[2];
FileCaches[hash] = new FileCache(hash, path, time);
}
}
}
public void WriteOutFullCsv()
{
StringBuilder sb = new StringBuilder();
foreach (var entry in FileCaches)
{
sb.AppendLine(entry.Value.CsvEntry);
}
if (File.Exists(CsvPath))
{
File.Copy(CsvPath, CsvBakPath, true);
}
lock (_fileWriteLock)
{
File.WriteAllText(CsvPath, sb.ToString());
}
}
public List<FileCache> GetAllFileCaches() => FileCaches.Values.ToList();
public FileCache? GetFileCacheByHash(string hash)
{
if (FileCaches.ContainsKey(hash))
{
return GetValidatedFileCache(FileCaches[hash]);
}
return null;
}
public (FileState, FileCache) ValidateFileCacheEntity(FileCache fileCache)
{
fileCache = ReplacePathPrefixes(fileCache);
FileInfo fi = new(fileCache.ResolvedFilepath);
if (!fi.Exists)
{
return (FileState.RequireDeletion, fileCache);
}
if (fi.LastWriteTimeUtc.Ticks.ToString() != fileCache.LastModifiedDateTicks)
{
return (FileState.RequireUpdate, fileCache);
}
return (FileState.Valid, fileCache);
}
public FileCache? GetFileCacheByPath(string path)
{
var cleanedPath = path.Replace("/", "\\").ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), "");
var entry = FileCaches.FirstOrDefault(f => f.Value.ResolvedFilepath.EndsWith(cleanedPath)).Value;
if (entry == null)
{
Logger.Debug("Found no entries for " + cleanedPath);
return CreateFileEntry(path);
}
var validatedCacheEntry = GetValidatedFileCache(entry);
return validatedCacheEntry;
}
public FileCache? CreateCacheEntry(string path)
{
Logger.Debug("Creating cache entry for " + path);
FileInfo fi = new(path);
if (!fi.Exists) return null;
var fullName = fi.FullName.ToLowerInvariant();
if (!fullName.Contains(_configuration.CacheFolder.ToLowerInvariant())) return null;
string prefixedPath = fullName.Replace(_configuration.CacheFolder.ToLowerInvariant(), CachePrefix + "\\").Replace("\\\\", "\\");
return CreateFileCacheEntity(fi, prefixedPath);
}
public FileCache? CreateFileEntry(string path)
{
Logger.Debug("Creating file entry for " + path);
FileInfo fi = new(path);
if (!fi.Exists) return null;
var fullName = fi.FullName.ToLowerInvariant();
if (!fullName.Contains(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant())) return null;
string prefixedPath = fullName.Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), PenumbraPrefix + "\\").Replace("\\\\", "\\");
return CreateFileCacheEntity(fi, prefixedPath);
}
private FileCache? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath)
{
var hash = Crypto.GetFileHash(fileInfo.FullName);
var entity = new FileCache(hash, prefixedPath, fileInfo.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture));
FileCaches[hash] = entity;
lock (_fileWriteLock)
{
File.AppendAllLines(CsvPath, new[] { entity.CsvEntry });
}
var result = GetFileCacheByPath(prefixedPath);
Logger.Debug("Creating file cache for " + fileInfo.FullName + " success: " + (result != null));
return result;
}
private FileCache? GetValidatedFileCache(FileCache fileCache)
{
var resulingFileCache = ReplacePathPrefixes(fileCache);
resulingFileCache = Validate(resulingFileCache);
return resulingFileCache;
}
private FileCache? Validate(FileCache fileCache)
{
var file = new FileInfo(fileCache.ResolvedFilepath);
if (!file.Exists)
{
FileCaches.Remove(fileCache.Hash, out _);
return null;
}
if (file.LastWriteTimeUtc.Ticks.ToString() != fileCache.LastModifiedDateTicks)
{
UpdateHash(fileCache);
}
return fileCache;
}
public void RemoveHash(FileCache entity)
{
FileCaches.Remove(entity.Hash, out _);
}
public void UpdateHash(FileCache fileCache)
{
var prevHash = fileCache.Hash;
fileCache.Hash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
FileCaches.Remove(prevHash, out _);
FileCaches[fileCache.Hash] = fileCache;
}
private FileCache ReplacePathPrefixes(FileCache fileCache)
{
if (fileCache.PrefixedFilePath.StartsWith(PenumbraPrefix))
{
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory()));
}
else if (fileCache.PrefixedFilePath.StartsWith(CachePrefix))
{
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(CachePrefix, _configuration.CacheFolder));
}
return fileCache;
}
public void Dispose()
{
WriteOutFullCsv();
}
}

96
MareSynchronos/FileCacheDB/PeriodicFileScanner.cs → MareSynchronos/FileCache/PeriodicFileScanner.cs
View File

@@ -7,21 +7,20 @@ using System.Threading.Tasks;
using MareSynchronos.Managers;
using MareSynchronos.Utils;
using MareSynchronos.WebAPI;
using Microsoft.EntityFrameworkCore;
namespace MareSynchronos.FileCacheDB;
namespace MareSynchronos.FileCache;
public class PeriodicFileScanner : IDisposable
{
private readonly IpcManager _ipcManager;
private readonly Configuration _pluginConfiguration;
private readonly FileDbManager _fileDbManager;
private readonly FileCacheManager _fileDbManager;
private readonly ApiController _apiController;
private readonly DalamudUtil _dalamudUtil;
private int haltScanRequests = 0;
private CancellationTokenSource? _scanCancellationTokenSource;
private Task? _fileScannerTask = null;
public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileDbManager fileDbManager, ApiController apiController, DalamudUtil dalamudUtil)
public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileCacheManager fileDbManager, ApiController apiController, DalamudUtil dalamudUtil)
{
Logger.Verbose("Creating " + nameof(PeriodicFileScanner));
@@ -198,47 +197,33 @@ public class PeriodicFileScanner : IDisposable
var cpuCount = (int)(Environment.ProcessorCount / 2.0f);
Task[] dbTasks = Enumerable.Range(0, cpuCount).Select(c => Task.CompletedTask).ToArray();
ConcurrentBag<Tuple<string, string>> entitiesToRemove = new();
ConcurrentBag<string> entitiesToUpdate = new();
ConcurrentBag<FileCache> entitiesToRemove = new();
ConcurrentBag<FileCache> entitiesToUpdate = new();
try
{
using var ctx = new FileCacheContext();
Logger.Debug("Database contains " + ctx.FileCaches.Count() + " files, local system contains " + TotalFiles);
using var cmd = ctx.Database.GetDbConnection().CreateCommand();
cmd.CommandText = "SELECT Hash, FilePath, LastModifiedDate FROM FileCache";
ctx.Database.OpenConnection();
using var reader = cmd.ExecuteReader();
while (reader.Read())
foreach (var value in _fileDbManager.GetAllFileCaches())
{
var hash = reader["Hash"].ToString();
var filePath = reader["FilePath"].ToString();
var date = reader["LastModifiedDate"].ToString();
var idx = Task.WaitAny(dbTasks, ct);
dbTasks[idx] = Task.Run(() =>
{
try
{
var fileState = _fileDbManager.ValidateFileCacheEntity(hash, filePath, date);
switch (fileState.Item1)
var result = _fileDbManager.ValidateFileCacheEntity(value);
scannedFiles[result.Item2.ResolvedFilepath] = true;
if (result.Item1 == FileState.RequireUpdate)
{
case FileState.Valid:
scannedFiles[fileState.Item2] = true;
break;
case FileState.RequireDeletion:
entitiesToRemove.Add(new Tuple<string, string>(hash, filePath));
break;
case FileState.RequireUpdate:
scannedFiles[fileState.Item2] = true;
entitiesToUpdate.Add(fileState.Item2);
break;
entitiesToUpdate.Add(result.Item2);
}
else if (result.Item1 == FileState.RequireDeletion)
{
entitiesToRemove.Add(result.Item2);
}
}
catch (Exception ex)
{
Logger.Warn("Failed validating " + filePath);
Logger.Warn("Failed validating " + value.ResolvedFilepath);
Logger.Warn(ex.Message);
Logger.Warn(ex.StackTrace);
entitiesToRemove.Add(new Tuple<string, string>(hash, filePath));
}
Interlocked.Increment(ref currentFileProgress);
@@ -257,44 +242,23 @@ public class PeriodicFileScanner : IDisposable
Logger.Warn("Error during enumerating FileCaches: " + ex.Message);
}
using (var db = new FileCacheContext())
{
try
{
if (entitiesToRemove.Any())
{
foreach (var entry in entitiesToRemove)
{
Logger.Debug("Removing " + entry.Item2);
var toRemove = db.FileCaches.First(f => f.Filepath == entry.Item2 && f.Hash == entry.Item1);
db.FileCaches.Remove(toRemove);
}
}
if (entitiesToUpdate.Any())
{
foreach (var entry in entitiesToUpdate)
{
Logger.Debug("Updating " + entry);
_fileDbManager.GetFileCacheByPath(entry);
}
}
if (entitiesToUpdate.Any() || entitiesToRemove.Any())
{
db.SaveChanges();
}
}
catch (Exception ex)
{
Logger.Warn(ex.Message);
}
}
Task.WaitAll(dbTasks);
if (entitiesToUpdate.Any() || entitiesToRemove.Any())
{
foreach (var entity in entitiesToUpdate)
{
_fileDbManager.UpdateHash(entity);
}
foreach (var entity in entitiesToRemove)
{
_fileDbManager.RemoveHash(entity);
}
_fileDbManager.WriteOutFullCsv();
}
Logger.Debug("Scanner validated existing db files");
if (ct.IsCancellationRequested) return;

48
MareSynchronos/FileCacheDB/FileCache.cs
View File

@@ -1,48 +0,0 @@
#nullable disable
using System;
namespace MareSynchronos.FileCacheDB
{
public class FileCache
{
private FileCacheEntity entity;
public string Filepath { get; private set; }
public string Hash { get; private set; }
private string originalFilePathNoEntity = string.Empty;
private string originalHashNoEntity = string.Empty;
private string originalModifiedDate = string.Empty;
public string OriginalFilepath => entity == null ? originalFilePathNoEntity : entity.Filepath;
public string OriginalHash => entity == null ? originalHashNoEntity : entity.Hash;
public long LastModifiedDateTicks => long.Parse(entity == null ? originalModifiedDate : entity.LastModifiedDate);
public FileCache(string hash, string path, string lastModifiedDate)
{
originalHashNoEntity = hash;
originalFilePathNoEntity = path;
originalModifiedDate = lastModifiedDate;
}
public FileCache(FileCacheEntity entity)
{
this.entity = entity;
}
public void SetResolvedFilePath(string filePath)
{
Filepath = filePath.ToLowerInvariant().Replace("\\\\", "\\");
}
public void SetHash(string hash)
{
Hash = hash;
}
public void UpdateFileCache(FileCacheEntity entity)
{
this.entity = entity;
}
}
}

46
MareSynchronos/FileCacheDB/FileCacheContext.cs
View File

@@ -1,46 +0,0 @@
using System.IO;
using Microsoft.EntityFrameworkCore;
#nullable disable
namespace MareSynchronos.FileCacheDB
{
public partial class FileCacheContext : DbContext
{
private string DbPath { get; set; }
public FileCacheContext()
{
DbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "FileCache.db");
Database.EnsureCreated();
}
public FileCacheContext(DbContextOptions<FileCacheContext> options)
: base(options)
{
}
public virtual DbSet<FileCacheEntity> FileCaches { get; set; }
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
if (!optionsBuilder.IsConfigured)
{
optionsBuilder.UseSqlite("Data Source=" + DbPath+";Cache=Shared");
}
}
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.Entity<FileCacheEntity>(entity =>
{
entity.HasKey(e => new { e.Hash, e.Filepath });
entity.ToTable("FileCache");
});
OnModelCreatingPartial(modelBuilder);
}
partial void OnModelCreatingPartial(ModelBuilder modelBuilder);
}
}

13
MareSynchronos/FileCacheDB/FileCacheEntity.cs
View File

@@ -1,13 +0,0 @@
#nullable disable
namespace MareSynchronos.FileCacheDB
{
public partial class FileCacheEntity
{
public string Hash { get; set; }
public string Filepath { get; set; }
public string LastModifiedDate { get; set; }
public int Version { get; set; }
}
}

284
MareSynchronos/FileCacheDB/FileDbManager.cs
View File

@@ -1,284 +0,0 @@
using MareSynchronos.FileCacheDB;
using MareSynchronos.Utils;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
namespace MareSynchronos.Managers;
public enum FileState
{
Valid,
RequireUpdate,
RequireDeletion
}
public class FileDbManager
{
private const string PenumbraPrefix = "{penumbra}";
private const string CachePrefix = "{cache}";
private readonly IpcManager _ipcManager;
private readonly Configuration _configuration;
private static object _lock = new();
public FileDbManager(IpcManager ipcManager, Configuration configuration)
{
_ipcManager = ipcManager;
_configuration = configuration;
}
public FileCache? GetFileCacheByHash(string hash)
{
List<FileCacheEntity> matchingEntries = new List<FileCacheEntity>();
using (var db = new FileCacheContext())
{
matchingEntries = db.FileCaches.Where(f => f.Hash.ToLower() == hash.ToLower()).ToList();
}
if (!matchingEntries.Any()) return null;
if (matchingEntries.Any(f => f.Filepath.Contains(PenumbraPrefix) && matchingEntries.Any(f => f.Filepath.Contains(CachePrefix))))
{
var cachedEntries = matchingEntries.Where(f => f.Filepath.Contains(CachePrefix)).ToList();
DeleteFromDatabase(cachedEntries.Select(f => new FileCache(f)));
foreach (var entry in cachedEntries)
{
matchingEntries.Remove(entry);
}
}
return GetValidatedFileCache(new FileCache(matchingEntries.First()));
}
public (FileState, string) ValidateFileCacheEntity(string hash, string path, string lastModifiedDate)
{
var fileCache = new FileCache(hash, path, lastModifiedDate);
if (!fileCache.OriginalFilepath.StartsWith(PenumbraPrefix + "\\") && !fileCache.OriginalFilepath.StartsWith(CachePrefix))
return (FileState.RequireUpdate, path);
fileCache = ReplacePathPrefixes(fileCache);
FileInfo fi = new FileInfo(fileCache.Filepath);
if (!fi.Exists)
return (FileState.RequireDeletion, fileCache.Filepath);
if (fi.LastWriteTimeUtc.Ticks != fileCache.LastModifiedDateTicks)
return (FileState.RequireUpdate, fileCache.Filepath);
return (FileState.Valid, fileCache.Filepath);
}
public FileCache? GetFileCacheByPath(string path)
{
FileCacheEntity? matchingEntries = null;
var cleanedPath = path.Replace("/", "\\").ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), "");
using (var db = new FileCacheContext())
{
matchingEntries = db.FileCaches.FirstOrDefault(f => f.Filepath.EndsWith(cleanedPath));
}
if (matchingEntries == null)
{
Logger.Debug("Found no entries for " + cleanedPath);
return CreateFileEntry(path);
}
var validatedCacheEntry = GetValidatedFileCache(new FileCache(matchingEntries));
return validatedCacheEntry;
}
public FileCache? CreateCacheEntry(string path)
{
Logger.Debug("Creating cache entry for " + path);
FileInfo fi = new FileInfo(path);
if (!fi.Exists) return null;
string prefixedPath = fi.FullName.ToLowerInvariant().Replace(_configuration.CacheFolder.ToLowerInvariant(), CachePrefix + "\\").Replace("\\\\", "\\");
return CreateFileCacheEntity(fi, prefixedPath);
}
public FileCache? CreateFileEntry(string path)
{
Logger.Debug("Creating file entry for " + path);
FileInfo fi = new FileInfo(path);
if (!fi.Exists) return null;
string prefixedPath = fi.FullName.ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), PenumbraPrefix + "\\").Replace("\\\\", "\\");
return CreateFileCacheEntity(fi, prefixedPath);
}
private FileCache? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath)
{
var hash = Crypto.GetFileHash(fileInfo.FullName);
lock (_lock)
{
var entity = new FileCacheEntity();
entity.Hash = hash;
entity.Filepath = prefixedPath;
entity.LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
try
{
using var db = new FileCacheContext();
db.FileCaches.Add(entity);
db.SaveChanges();
}
catch (Exception ex)
{
Logger.Warn("Could not add " + fileInfo.FullName ?? String.Empty);
Logger.Warn(ex.Message);
}
}
var result = GetFileCacheByPath(prefixedPath);
Logger.Debug("Creating file cache for " + fileInfo.FullName + " success: " + (result != null));
return result;
}
private FileCache? GetValidatedFileCache(FileCache fileCache)
{
var resulingFileCache = MigrateLegacy(fileCache);
if (resulingFileCache == null) return null;
resulingFileCache = ReplacePathPrefixes(resulingFileCache);
resulingFileCache = Validate(resulingFileCache);
return resulingFileCache;
}
private FileCache? Validate(FileCache fileCache)
{
var file = new FileInfo(fileCache.Filepath);
if (!file.Exists)
{
DeleteFromDatabase(new[] { fileCache });
return null;
}
if (file.LastWriteTimeUtc.Ticks != fileCache.LastModifiedDateTicks)
{
fileCache.SetHash(Crypto.GetFileHash(fileCache.Filepath));
UpdateCacheHash(fileCache, file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture));
}
return fileCache;
}
private FileCache? MigrateLegacy(FileCache fileCache)
{
if (fileCache.OriginalFilepath.StartsWith(PenumbraPrefix + "\\") || fileCache.OriginalFilepath.StartsWith(CachePrefix)) return fileCache;
var fileInfo = new FileInfo(fileCache.OriginalFilepath);
var penumbraDir = _ipcManager.PenumbraModDirectory()!;
if (penumbraDir.Last() != '\\') penumbraDir += "\\";
// check if it's a cache file
if (fileInfo.Exists && fileInfo.Name.Length == 40)
{
MigrateLegacyFilePath(fileCache, CachePrefix + "\\" + fileInfo.Name.ToLower());
}
else if (fileInfo.Exists && fileInfo.FullName.ToLowerInvariant().Contains(penumbraDir))
{
// attempt to replace penumbra mod folder path with {penumbra}
var newPath = PenumbraPrefix + "\\" + fileCache.OriginalFilepath.ToLowerInvariant().Replace(penumbraDir, string.Empty);
MigrateLegacyFilePath(fileCache, newPath);
}
else if (fileInfo.FullName.ToLowerInvariant().Contains(PenumbraPrefix))
{
var newPath = PenumbraPrefix + "\\" + fileCache.OriginalFilepath.ToLowerInvariant().Replace(PenumbraPrefix, string.Empty);
MigrateLegacyFilePath(fileCache, newPath);
}
else
{
DeleteFromDatabase(new[] { fileCache });
return null;
}
return fileCache;
}
private FileCache ReplacePathPrefixes(FileCache fileCache)
{
if (fileCache.OriginalFilepath.StartsWith(PenumbraPrefix))
{
fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory()));
}
else if (fileCache.OriginalFilepath.StartsWith(CachePrefix))
{
fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(CachePrefix, _configuration.CacheFolder));
}
return fileCache;
}
private void UpdateCacheHash(FileCache markedForUpdate, string lastModifiedDate)
{
lock (_lock)
{
try
{
Logger.Verbose("Updating Hash for " + markedForUpdate.OriginalFilepath);
using var db = new FileCacheContext();
var cache = db.FileCaches.First(f => f.Filepath == markedForUpdate.OriginalFilepath && f.Hash == markedForUpdate.OriginalHash);
var newcache = new FileCacheEntity()
{
Filepath = cache.Filepath,
Hash = markedForUpdate.Hash,
LastModifiedDate = lastModifiedDate
};
db.Remove(cache);
db.FileCaches.Add(newcache);
markedForUpdate.UpdateFileCache(newcache);
db.SaveChanges();
}
catch (Exception ex)
{
Logger.Warn("Error updating file hash (" + ex.Message + "), returning currently existing" ?? string.Empty);
Logger.Warn(ex.InnerException?.Message ?? string.Empty);
Logger.Warn(ex.StackTrace ?? string.Empty);
using var db = new FileCacheContext();
var cache = db.FileCaches.First(f => f.Filepath == markedForUpdate.OriginalFilepath);
markedForUpdate.UpdateFileCache(cache);
}
}
}
private void MigrateLegacyFilePath(FileCache fileCacheToMigrate, string newPath)
{
lock (_lock)
{
Logger.Verbose("Migrating legacy file path for " + fileCacheToMigrate.OriginalFilepath);
using var db = new FileCacheContext();
var cache = db.FileCaches.First(f => f.Filepath == fileCacheToMigrate.OriginalFilepath && f.Hash == fileCacheToMigrate.OriginalHash);
var newcache = new FileCacheEntity()
{
Filepath = newPath,
Hash = cache.Hash,
LastModifiedDate = cache.LastModifiedDate
};
db.Remove(cache);
db.SaveChanges();
var existingCache = db.FileCaches.FirstOrDefault(f => f.Filepath == newPath && f.Hash == cache.Hash);
if (existingCache != null)
{
fileCacheToMigrate.UpdateFileCache(existingCache);
}
else
{
db.FileCaches.Add(newcache);
fileCacheToMigrate.UpdateFileCache(newcache);
db.SaveChanges();
}
}
}
private void DeleteFromDatabase(IEnumerable<FileCache> markedForDeletion)
{
lock (_lock)
{
using var db = new FileCacheContext();
foreach (var item in markedForDeletion)
{
Logger.Verbose("Removing " + item.OriginalFilepath);
var itemToRemove = db.FileCaches.FirstOrDefault(f => f.Hash == item.OriginalHash && f.Filepath == item.OriginalFilepath);
if (itemToRemove == null) continue;
db.FileCaches.Remove(itemToRemove);
}
db.SaveChanges();
}
}
}

7
MareSynchronos/Managers/CachedPlayer.cs
View File

@@ -6,6 +6,7 @@ using System.Threading.Tasks;
using Dalamud.Logging;
using FFXIVClientStructs.FFXIV.Client.Game.Character;
using MareSynchronos.API;
using MareSynchronos.FileCache;
using MareSynchronos.Models;
using MareSynchronos.Utils;
using MareSynchronos.WebAPI;
@@ -15,12 +16,12 @@ namespace MareSynchronos.Managers;
public class CachedPlayer
{
private readonly DalamudUtil _dalamudUtil;
private readonly FileDbManager fileDbManager;
private readonly FileCacheManager fileDbManager;
private readonly IpcManager _ipcManager;
private readonly ApiController _apiController;
private bool _isVisible;
public CachedPlayer(string nameHash, IpcManager ipcManager, ApiController apiController, DalamudUtil dalamudUtil, FileDbManager fileDbManager)
public CachedPlayer(string nameHash, IpcManager ipcManager, ApiController apiController, DalamudUtil dalamudUtil, FileCacheManager fileDbManager)
{
PlayerNameHash = nameHash;
_ipcManager = ipcManager;
@@ -201,7 +202,7 @@ public class CachedPlayer
var fileCache = fileDbManager.GetFileCacheByHash(item.Hash);
if (fileCache != null)
{
moddedDictionary[gamePath] = fileCache.Filepath;
moddedDictionary[gamePath] = fileCache.ResolvedFilepath;
}
else
{

5
MareSynchronos/Managers/OnlinePlayerManager.cs
View File

@@ -5,6 +5,7 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MareSynchronos.API;
using MareSynchronos.FileCache;
using MareSynchronos.Utils;
using MareSynchronos.WebAPI;
using MareSynchronos.WebAPI.Utils;
@@ -17,7 +18,7 @@ public class OnlinePlayerManager : IDisposable
private readonly DalamudUtil _dalamudUtil;
private readonly IpcManager _ipcManager;
private readonly PlayerManager _playerManager;
private readonly FileDbManager _fileDbManager;
private readonly FileCacheManager _fileDbManager;
private readonly ConcurrentDictionary<string, CachedPlayer> _onlineCachedPlayers = new();
private readonly ConcurrentDictionary<string, CharacterCacheDto> _temporaryStoredCharacterCache = new();
private readonly ConcurrentDictionary<CachedPlayer, CancellationTokenSource> _playerTokenDisposal = new();
@@ -25,7 +26,7 @@ public class OnlinePlayerManager : IDisposable
private List<string> OnlineVisiblePlayerHashes => _onlineCachedPlayers.Select(p => p.Value).Where(p => p.PlayerCharacter != IntPtr.Zero)
.Select(p => p.PlayerNameHash).ToList();
public OnlinePlayerManager(ApiController apiController, DalamudUtil dalamudUtil, IpcManager ipcManager, PlayerManager playerManager, FileDbManager fileDbManager)
public OnlinePlayerManager(ApiController apiController, DalamudUtil dalamudUtil, IpcManager ipcManager, PlayerManager playerManager, FileCacheManager fileDbManager)
{
Logger.Verbose("Creating " + nameof(OnlinePlayerManager));

2
MareSynchronos/Managers/PlayerManager.cs
View File

@@ -9,7 +9,7 @@ using FFXIVClientStructs.FFXIV.Client.Game.Character;
using System.Collections.Generic;
using System.Linq;
using MareSynchronos.Models;
using MareSynchronos.FileCacheDB;
using MareSynchronos.FileCache;
#if DEBUG
using Newtonsoft.Json;
#endif

1
MareSynchronos/MareSynchronos.csproj
View File

@@ -29,7 +29,6 @@
<PackageReference Include="DalamudPackager" Version="2.1.8" />
<PackageReference Include="lz4net" Version="1.0.15.93" />
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="6.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.8" />
</ItemGroup>
<PropertyGroup>

8
MareSynchronos/Models/FileReplacement.cs
View File

@@ -4,15 +4,15 @@ using System.Text;
using System.Threading.Tasks;
using MareSynchronos.API;
using System.Text.RegularExpressions;
using MareSynchronos.Managers;
using MareSynchronos.FileCache;
namespace MareSynchronos.Models
{
public class FileReplacement
{
private readonly FileDbManager fileDbManager;
private readonly FileCacheManager fileDbManager;
public FileReplacement(FileDbManager fileDbManager)
public FileReplacement(FileCacheManager fileDbManager)
{
this.fileDbManager = fileDbManager;
}
@@ -37,7 +37,7 @@ namespace MareSynchronos.Models
_ = Task.Run(() =>
{
var cache = fileDbManager.GetFileCacheByPath(ResolvedPath);
Hash = cache.OriginalHash;
Hash = cache.Hash;
});
}

30
MareSynchronos/Plugin.cs
View File

@@ -1,6 +1,5 @@
using Dalamud.Game.Command;
using Dalamud.Plugin;
using MareSynchronos.FileCacheDB;
using MareSynchronos.Factories;
using System.Threading.Tasks;
using Dalamud.Game;
@@ -13,8 +12,8 @@ using MareSynchronos.WebAPI;
using Dalamud.Interface.Windowing;
using MareSynchronos.UI;
using MareSynchronos.Utils;
using System.Runtime.InteropServices;
using Dalamud.Game.ClientState.Conditions;
using MareSynchronos.FileCache;
namespace MareSynchronos
{
@@ -27,7 +26,7 @@ namespace MareSynchronos
private readonly PeriodicFileScanner _periodicFileScanner;
private readonly IntroUi _introUi;
private readonly IpcManager _ipcManager;
public static DalamudPluginInterface PluginInterface { get; set; }
private readonly DalamudPluginInterface _pluginInterface;
private readonly SettingsUi _settingsUi;
private readonly WindowSystem _windowSystem;
private PlayerManager? _playerManager;
@@ -36,7 +35,7 @@ namespace MareSynchronos
private OnlinePlayerManager? _characterCacheManager;
private readonly DownloadUi _downloadUi;
private readonly FileDialogManager _fileDialogManager;
private readonly FileDbManager _fileDbManager;
private readonly FileCacheManager _fileDbManager;
private readonly CompactUi _compactUi;
private readonly UiShared _uiSharedComponent;
private readonly Dalamud.Localization _localization;
@@ -46,10 +45,10 @@ namespace MareSynchronos
Framework framework, ObjectTable objectTable, ClientState clientState, Condition condition)
{
Logger.Debug("Launching " + Name);
PluginInterface = pluginInterface;
_pluginInterface = pluginInterface;
_commandManager = commandManager;
_configuration = PluginInterface.GetPluginConfig() as Configuration ?? new Configuration();
_configuration.Initialize(PluginInterface);
_configuration = _pluginInterface.GetPluginConfig() as Configuration ?? new Configuration();
_configuration.Initialize(_pluginInterface);
_configuration.Migrate();
_localization = new Dalamud.Localization("MareSynchronos.Localization.", "", true);
@@ -57,19 +56,17 @@ namespace MareSynchronos
_windowSystem = new WindowSystem("MareSynchronos");
new FileCacheContext().Dispose(); // make sure db is initialized I guess
// those can be initialized outside of game login
_dalamudUtil = new DalamudUtil(clientState, objectTable, framework, condition);
_ipcManager = new IpcManager(PluginInterface, _dalamudUtil);
_ipcManager = new IpcManager(_pluginInterface, _dalamudUtil);
_fileDialogManager = new FileDialogManager();
_fileDbManager = new FileDbManager(_ipcManager, _configuration);
_fileDbManager = new FileCacheManager(_ipcManager, _configuration, _pluginInterface.ConfigDirectory.FullName);
_apiController = new ApiController(_configuration, _dalamudUtil, _fileDbManager);
_periodicFileScanner = new PeriodicFileScanner(_ipcManager, _configuration, _fileDbManager, _apiController, _dalamudUtil);
_uiSharedComponent =
new UiShared(_ipcManager, _apiController, _periodicFileScanner, _fileDialogManager, _configuration, _dalamudUtil, PluginInterface, _localization);
new UiShared(_ipcManager, _apiController, _periodicFileScanner, _fileDialogManager, _configuration, _dalamudUtil, _pluginInterface, _localization);
_settingsUi = new SettingsUi(_windowSystem, _uiSharedComponent, _configuration, _apiController);
_compactUi = new CompactUi(_windowSystem, _uiSharedComponent, _configuration, _apiController);
@@ -120,6 +117,7 @@ namespace MareSynchronos
_compactUi?.Dispose();
_periodicFileScanner?.Dispose();
_fileDbManager?.Dispose();
_playerManager?.Dispose();
_characterCacheManager?.Dispose();
_ipcManager?.Dispose();
@@ -133,8 +131,8 @@ namespace MareSynchronos
{
Logger.Debug("Client login");
PluginInterface.UiBuilder.Draw += Draw;
PluginInterface.UiBuilder.OpenConfigUi += OpenUi;
_pluginInterface.UiBuilder.Draw += Draw;
_pluginInterface.UiBuilder.OpenConfigUi += OpenUi;
_commandManager.AddHandler(CommandName, new CommandInfo(OnCommand)
{
HelpMessage = "Opens the Mare Synchronos UI"
@@ -157,8 +155,8 @@ namespace MareSynchronos
_characterCacheManager?.Dispose();
_playerManager?.Dispose();
_transientResourceManager?.Dispose();
PluginInterface.UiBuilder.Draw -= Draw;
PluginInterface.UiBuilder.OpenConfigUi -= OpenUi;
_pluginInterface.UiBuilder.Draw -= Draw;
_pluginInterface.UiBuilder.OpenConfigUi -= OpenUi;
_commandManager.RemoveHandler(CommandName);
}

2
MareSynchronos/UI/IntroUI.cs
View File

@@ -10,7 +10,7 @@ using ImGuiNET;
using MareSynchronos.Utils;
using MareSynchronos.Localization;
using Dalamud.Utility;
using MareSynchronos.FileCacheDB;
using MareSynchronos.FileCache;
namespace MareSynchronos.UI
{

2
MareSynchronos/UI/UIShared.cs
View File

@@ -11,7 +11,7 @@ using Dalamud.Interface.ImGuiFileDialog;
using Dalamud.Plugin;
using Dalamud.Utility;
using ImGuiNET;
using MareSynchronos.FileCacheDB;
using MareSynchronos.FileCache;
using MareSynchronos.Localization;
using MareSynchronos.Managers;
using MareSynchronos.Utils;

7
MareSynchronos/WebAPI/ApIController.Functions.Files.cs
View File

@@ -9,7 +9,6 @@ using System.Threading;
using System.Threading.Tasks;
using LZ4;
using MareSynchronos.API;
using MareSynchronos.FileCacheDB;
using MareSynchronos.Utils;
using MareSynchronos.WebAPI.Utils;
using Microsoft.AspNetCore.SignalR.Client;
@@ -177,7 +176,7 @@ namespace MareSynchronos.WebAPI
{
CurrentUploads.Add(new UploadFileTransfer(file)
{
Total = new FileInfo(_fileDbManager.GetFileCacheByHash(file.Hash)!.Filepath).Length
Total = new FileInfo(_fileDbManager.GetFileCacheByHash(file.Hash)!.ResolvedFilepath).Length
});
}
catch (Exception ex)
@@ -193,7 +192,7 @@ namespace MareSynchronos.WebAPI
{
ForbiddenTransfers.Add(new UploadFileTransfer(file)
{
LocalFile = _fileDbManager.GetFileCacheByHash(file.Hash)?.Filepath ?? string.Empty
LocalFile = _fileDbManager.GetFileCacheByHash(file.Hash)?.ResolvedFilepath ?? string.Empty
});
}
}
@@ -266,7 +265,7 @@ namespace MareSynchronos.WebAPI
private async Task<(string, byte[])> GetCompressedFileData(string fileHash, CancellationToken uploadToken)
{
var fileCache = _fileDbManager.GetFileCacheByHash(fileHash)!.Filepath;
var fileCache = _fileDbManager.GetFileCacheByHash(fileHash)!.ResolvedFilepath;
return (fileHash, LZ4Codec.WrapHC(await File.ReadAllBytesAsync(fileCache, uploadToken), 0,
(int)new FileInfo(fileCache).Length));
}

6
MareSynchronos/WebAPI/ApiController.Connectivity.cs
View File

@@ -6,7 +6,7 @@ using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using MareSynchronos.API;
using MareSynchronos.Managers;
using MareSynchronos.FileCache;
using MareSynchronos.Utils;
using MareSynchronos.WebAPI.Utils;
using Microsoft.AspNetCore.Http.Connections;
@@ -36,7 +36,7 @@ namespace MareSynchronos.WebAPI
private readonly Configuration _pluginConfiguration;
private readonly DalamudUtil _dalamudUtil;
private readonly FileDbManager _fileDbManager;
private readonly FileCacheManager _fileDbManager;
private CancellationTokenSource _connectionCancellationTokenSource;
private HubConnection? _mareHub;
@@ -49,7 +49,7 @@ namespace MareSynchronos.WebAPI
public bool IsAdmin => _connectionDto?.IsAdmin ?? false;
public ApiController(Configuration pluginConfiguration, DalamudUtil dalamudUtil, FileDbManager fileDbManager)
public ApiController(Configuration pluginConfiguration, DalamudUtil dalamudUtil, FileCacheManager fileDbManager)
{
Logger.Verbose("Creating " + nameof(ApiController));