remove database, use in-memory
This commit is contained in:
Stanley Dimant
29
MareSynchronos/FileCache/FileCache.cs
Normal file
29
MareSynchronos/FileCache/FileCache.cs
Normal file
@@ -0,0 +1,29 @@
|
||||
#nullable disable
|
||||
|
||||
|
||||
using MareSynchronos;
|
||||
using System.Globalization;
|
||||
|
||||
namespace MareSynchronos.FileCache;
|
||||
|
||||
public class FileCache
|
||||
{
|
||||
public string ResolvedFilepath { get; private set; }
|
||||
public string Hash { get; set; }
|
||||
public string PrefixedFilePath { get; init; }
|
||||
public string LastModifiedDateTicks { get; init; }
|
||||
|
||||
public FileCache(string hash, string path, string lastModifiedDateTicks)
|
||||
{
|
||||
Hash = hash;
|
||||
PrefixedFilePath = path;
|
||||
LastModifiedDateTicks = lastModifiedDateTicks;
|
||||
}
|
||||
|
||||
public void SetResolvedFilePath(string filePath)
|
||||
{
|
||||
ResolvedFilepath = filePath.ToLowerInvariant().Replace("\\\\", "\\");
|
||||
}
|
||||
|
||||
public string CsvEntry => $"{Hash}{FileCacheManager.CsvSplit}{PrefixedFilePath}{FileCacheManager.CsvSplit}{LastModifiedDateTicks.ToString(CultureInfo.InvariantCulture)}";
|
||||
}
|
||||
210
MareSynchronos/FileCache/FileDbManager.cs
Normal file
210
MareSynchronos/FileCache/FileDbManager.cs
Normal file
@@ -0,0 +1,210 @@
|
||||
using MareSynchronos.Managers;
|
||||
using MareSynchronos.Utils;
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace MareSynchronos.FileCache;
|
||||
|
||||
|
||||
public enum FileState
|
||||
{
|
||||
Valid,
|
||||
RequireUpdate,
|
||||
RequireDeletion
|
||||
}
|
||||
|
||||
public class FileCacheManager : IDisposable
|
||||
{
|
||||
private const string PenumbraPrefix = "{penumbra}";
|
||||
private const string CachePrefix = "{cache}";
|
||||
private readonly IpcManager _ipcManager;
|
||||
private readonly Configuration _configuration;
|
||||
private readonly string CsvPath;
|
||||
private string CsvBakPath => CsvPath + ".bak";
|
||||
private readonly ConcurrentDictionary<string, FileCache> FileCaches = new();
|
||||
public const string CsvSplit = "|";
|
||||
private object _fileWriteLock = new object();
|
||||
|
||||
public FileCacheManager(IpcManager ipcManager, Configuration configuration, string configDirectoryName)
|
||||
{
|
||||
_ipcManager = ipcManager;
|
||||
_configuration = configuration;
|
||||
CsvPath = Path.Combine(configDirectoryName, "FileCache.csv");
|
||||
|
||||
if (File.Exists(CsvBakPath))
|
||||
{
|
||||
File.Move(CsvBakPath, CsvPath, true);
|
||||
}
|
||||
|
||||
if (File.Exists(CsvPath))
|
||||
{
|
||||
var entries = File.ReadAllLines(CsvPath);
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
var splittedEntry = entry.Split(CsvSplit, StringSplitOptions.None);
|
||||
var hash = splittedEntry[0];
|
||||
var path = splittedEntry[1];
|
||||
var time = splittedEntry[2];
|
||||
FileCaches[hash] = new FileCache(hash, path, time);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void WriteOutFullCsv()
|
||||
{
|
||||
StringBuilder sb = new StringBuilder();
|
||||
foreach (var entry in FileCaches)
|
||||
{
|
||||
sb.AppendLine(entry.Value.CsvEntry);
|
||||
}
|
||||
if (File.Exists(CsvPath))
|
||||
{
|
||||
File.Copy(CsvPath, CsvBakPath, true);
|
||||
}
|
||||
lock (_fileWriteLock)
|
||||
{
|
||||
File.WriteAllText(CsvPath, sb.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
public List<FileCache> GetAllFileCaches() => FileCaches.Values.ToList();
|
||||
|
||||
public FileCache? GetFileCacheByHash(string hash)
|
||||
{
|
||||
if (FileCaches.ContainsKey(hash))
|
||||
{
|
||||
return GetValidatedFileCache(FileCaches[hash]);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public (FileState, FileCache) ValidateFileCacheEntity(FileCache fileCache)
|
||||
{
|
||||
fileCache = ReplacePathPrefixes(fileCache);
|
||||
FileInfo fi = new(fileCache.ResolvedFilepath);
|
||||
if (!fi.Exists)
|
||||
{
|
||||
return (FileState.RequireDeletion, fileCache);
|
||||
}
|
||||
if (fi.LastWriteTimeUtc.Ticks.ToString() != fileCache.LastModifiedDateTicks)
|
||||
{
|
||||
return (FileState.RequireUpdate, fileCache);
|
||||
}
|
||||
|
||||
return (FileState.Valid, fileCache);
|
||||
}
|
||||
|
||||
public FileCache? GetFileCacheByPath(string path)
|
||||
{
|
||||
var cleanedPath = path.Replace("/", "\\").ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), "");
|
||||
var entry = FileCaches.FirstOrDefault(f => f.Value.ResolvedFilepath.EndsWith(cleanedPath)).Value;
|
||||
|
||||
if (entry == null)
|
||||
{
|
||||
Logger.Debug("Found no entries for " + cleanedPath);
|
||||
return CreateFileEntry(path);
|
||||
}
|
||||
|
||||
var validatedCacheEntry = GetValidatedFileCache(entry);
|
||||
|
||||
return validatedCacheEntry;
|
||||
}
|
||||
|
||||
public FileCache? CreateCacheEntry(string path)
|
||||
{
|
||||
Logger.Debug("Creating cache entry for " + path);
|
||||
FileInfo fi = new(path);
|
||||
if (!fi.Exists) return null;
|
||||
var fullName = fi.FullName.ToLowerInvariant();
|
||||
if (!fullName.Contains(_configuration.CacheFolder.ToLowerInvariant())) return null;
|
||||
string prefixedPath = fullName.Replace(_configuration.CacheFolder.ToLowerInvariant(), CachePrefix + "\\").Replace("\\\\", "\\");
|
||||
return CreateFileCacheEntity(fi, prefixedPath);
|
||||
}
|
||||
|
||||
public FileCache? CreateFileEntry(string path)
|
||||
{
|
||||
Logger.Debug("Creating file entry for " + path);
|
||||
FileInfo fi = new(path);
|
||||
if (!fi.Exists) return null;
|
||||
var fullName = fi.FullName.ToLowerInvariant();
|
||||
if (!fullName.Contains(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant())) return null;
|
||||
string prefixedPath = fullName.Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), PenumbraPrefix + "\\").Replace("\\\\", "\\");
|
||||
return CreateFileCacheEntity(fi, prefixedPath);
|
||||
}
|
||||
|
||||
private FileCache? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath)
|
||||
{
|
||||
var hash = Crypto.GetFileHash(fileInfo.FullName);
|
||||
var entity = new FileCache(hash, prefixedPath, fileInfo.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture));
|
||||
FileCaches[hash] = entity;
|
||||
lock (_fileWriteLock)
|
||||
{
|
||||
File.AppendAllLines(CsvPath, new[] { entity.CsvEntry });
|
||||
}
|
||||
var result = GetFileCacheByPath(prefixedPath);
|
||||
Logger.Debug("Creating file cache for " + fileInfo.FullName + " success: " + (result != null));
|
||||
return result;
|
||||
}
|
||||
|
||||
private FileCache? GetValidatedFileCache(FileCache fileCache)
|
||||
{
|
||||
var resulingFileCache = ReplacePathPrefixes(fileCache);
|
||||
resulingFileCache = Validate(resulingFileCache);
|
||||
return resulingFileCache;
|
||||
}
|
||||
|
||||
private FileCache? Validate(FileCache fileCache)
|
||||
{
|
||||
var file = new FileInfo(fileCache.ResolvedFilepath);
|
||||
if (!file.Exists)
|
||||
{
|
||||
FileCaches.Remove(fileCache.Hash, out _);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (file.LastWriteTimeUtc.Ticks.ToString() != fileCache.LastModifiedDateTicks)
|
||||
{
|
||||
UpdateHash(fileCache);
|
||||
}
|
||||
|
||||
return fileCache;
|
||||
}
|
||||
|
||||
public void RemoveHash(FileCache entity)
|
||||
{
|
||||
FileCaches.Remove(entity.Hash, out _);
|
||||
}
|
||||
|
||||
public void UpdateHash(FileCache fileCache)
|
||||
{
|
||||
var prevHash = fileCache.Hash;
|
||||
fileCache.Hash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
|
||||
FileCaches.Remove(prevHash, out _);
|
||||
FileCaches[fileCache.Hash] = fileCache;
|
||||
}
|
||||
|
||||
private FileCache ReplacePathPrefixes(FileCache fileCache)
|
||||
{
|
||||
if (fileCache.PrefixedFilePath.StartsWith(PenumbraPrefix))
|
||||
{
|
||||
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory()));
|
||||
}
|
||||
else if (fileCache.PrefixedFilePath.StartsWith(CachePrefix))
|
||||
{
|
||||
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(CachePrefix, _configuration.CacheFolder));
|
||||
}
|
||||
|
||||
return fileCache;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
WriteOutFullCsv();
|
||||
}
|
||||
}
|
||||
314
MareSynchronos/FileCache/PeriodicFileScanner.cs
Normal file
314
MareSynchronos/FileCache/PeriodicFileScanner.cs
Normal file
@@ -0,0 +1,314 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MareSynchronos.Managers;
|
||||
using MareSynchronos.Utils;
|
||||
using MareSynchronos.WebAPI;
|
||||
|
||||
namespace MareSynchronos.FileCache;
|
||||
|
||||
public class PeriodicFileScanner : IDisposable
|
||||
{
|
||||
private readonly IpcManager _ipcManager;
|
||||
private readonly Configuration _pluginConfiguration;
|
||||
private readonly FileCacheManager _fileDbManager;
|
||||
private readonly ApiController _apiController;
|
||||
private readonly DalamudUtil _dalamudUtil;
|
||||
private int haltScanRequests = 0;
|
||||
private CancellationTokenSource? _scanCancellationTokenSource;
|
||||
private Task? _fileScannerTask = null;
|
||||
public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileCacheManager fileDbManager, ApiController apiController, DalamudUtil dalamudUtil)
|
||||
{
|
||||
Logger.Verbose("Creating " + nameof(PeriodicFileScanner));
|
||||
|
||||
_ipcManager = ipcManager;
|
||||
_pluginConfiguration = pluginConfiguration;
|
||||
_fileDbManager = fileDbManager;
|
||||
_apiController = apiController;
|
||||
_dalamudUtil = dalamudUtil;
|
||||
_ipcManager.PenumbraInitialized += StartScan;
|
||||
if (!string.IsNullOrEmpty(_ipcManager.PenumbraModDirectory()))
|
||||
{
|
||||
StartScan();
|
||||
}
|
||||
_apiController.DownloadStarted += HaltScan;
|
||||
_apiController.DownloadFinished += ResumeScan;
|
||||
_dalamudUtil.ZoneSwitchStart += HaltScan;
|
||||
_dalamudUtil.ZoneSwitchEnd += ResumeScan;
|
||||
}
|
||||
|
||||
public void ResumeScan()
|
||||
{
|
||||
Interlocked.Decrement(ref haltScanRequests);
|
||||
|
||||
if (fileScanWasRunning && haltScanRequests == 0)
|
||||
{
|
||||
fileScanWasRunning = false;
|
||||
InvokeScan(true);
|
||||
}
|
||||
}
|
||||
|
||||
public void HaltScan()
|
||||
{
|
||||
Interlocked.Increment(ref haltScanRequests);
|
||||
|
||||
if (IsScanRunning && haltScanRequests >= 0)
|
||||
{
|
||||
_scanCancellationTokenSource?.Cancel();
|
||||
fileScanWasRunning = true;
|
||||
}
|
||||
}
|
||||
|
||||
private bool fileScanWasRunning = false;
|
||||
private long currentFileProgress = 0;
|
||||
public long CurrentFileProgress => currentFileProgress;
|
||||
|
||||
public long FileCacheSize { get; set; }
|
||||
|
||||
public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
|
||||
|
||||
public long TotalFiles { get; private set; }
|
||||
|
||||
public string TimeUntilNextScan => _timeUntilNextScan.ToString(@"mm\:ss");
|
||||
private TimeSpan _timeUntilNextScan = TimeSpan.Zero;
|
||||
private int timeBetweenScans => _pluginConfiguration.TimeSpanBetweenScansInSeconds;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Logger.Verbose("Disposing " + nameof(PeriodicFileScanner));
|
||||
|
||||
_ipcManager.PenumbraInitialized -= StartScan;
|
||||
_apiController.DownloadStarted -= HaltScan;
|
||||
_apiController.DownloadFinished -= ResumeScan;
|
||||
_dalamudUtil.ZoneSwitchStart -= HaltScan;
|
||||
_dalamudUtil.ZoneSwitchEnd -= ResumeScan;
|
||||
_scanCancellationTokenSource?.Cancel();
|
||||
}
|
||||
|
||||
public void InvokeScan(bool forced = false)
|
||||
{
|
||||
bool isForced = forced;
|
||||
TotalFiles = 0;
|
||||
currentFileProgress = 0;
|
||||
_scanCancellationTokenSource?.Cancel();
|
||||
_scanCancellationTokenSource = new CancellationTokenSource();
|
||||
var token = _scanCancellationTokenSource.Token;
|
||||
_fileScannerTask = Task.Run(async () =>
|
||||
{
|
||||
while (!token.IsCancellationRequested)
|
||||
{
|
||||
while (haltScanRequests > 0)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(1));
|
||||
}
|
||||
|
||||
isForced |= RecalculateFileCacheSize();
|
||||
if (!_pluginConfiguration.FileScanPaused || isForced)
|
||||
{
|
||||
isForced = false;
|
||||
TotalFiles = 0;
|
||||
currentFileProgress = 0;
|
||||
PeriodicFileScan(token);
|
||||
TotalFiles = 0;
|
||||
currentFileProgress = 0;
|
||||
}
|
||||
_timeUntilNextScan = TimeSpan.FromSeconds(timeBetweenScans);
|
||||
while (_timeUntilNextScan.TotalSeconds >= 0)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), token);
|
||||
_timeUntilNextScan -= TimeSpan.FromSeconds(1);
|
||||
}
|
||||
}
|
||||
}, token);
|
||||
}
|
||||
|
||||
internal void StartWatchers()
|
||||
{
|
||||
InvokeScan();
|
||||
}
|
||||
|
||||
public bool RecalculateFileCacheSize()
|
||||
{
|
||||
FileCacheSize = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder).Sum(f =>
|
||||
{
|
||||
try
|
||||
{
|
||||
return new FileInfo(f).Length;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
});
|
||||
|
||||
if (FileCacheSize < (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024) return false;
|
||||
|
||||
var allFiles = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder)
|
||||
.Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList();
|
||||
while (FileCacheSize > (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024)
|
||||
{
|
||||
var oldestFile = allFiles.First();
|
||||
FileCacheSize -= oldestFile.Length;
|
||||
File.Delete(oldestFile.FullName);
|
||||
allFiles.Remove(oldestFile);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private void PeriodicFileScan(CancellationToken ct)
|
||||
{
|
||||
TotalFiles = 1;
|
||||
var penumbraDir = _ipcManager.PenumbraModDirectory();
|
||||
bool penDirExists = true;
|
||||
bool cacheDirExists = true;
|
||||
if (string.IsNullOrEmpty(penumbraDir) || !Directory.Exists(penumbraDir))
|
||||
{
|
||||
penDirExists = false;
|
||||
Logger.Warn("Penumbra directory is not set or does not exist.");
|
||||
}
|
||||
if (string.IsNullOrEmpty(_pluginConfiguration.CacheFolder) || !Directory.Exists(_pluginConfiguration.CacheFolder))
|
||||
{
|
||||
cacheDirExists = false;
|
||||
Logger.Warn("Mare Cache directory is not set or does not exist.");
|
||||
}
|
||||
if (!penDirExists || !cacheDirExists)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Logger.Debug("Getting files from " + penumbraDir + " and " + _pluginConfiguration.CacheFolder);
|
||||
string[] ext = { ".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp" };
|
||||
|
||||
var scannedFiles = Directory.EnumerateFiles(penumbraDir, "*.*", SearchOption.AllDirectories)
|
||||
.Select(s => s.ToLowerInvariant())
|
||||
.Where(f => ext.Any(e => f.EndsWith(e)) && !f.Contains(@"\bg\") && !f.Contains(@"\bgcommon\") && !f.Contains(@"\ui\"))
|
||||
.Concat(Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.TopDirectoryOnly)
|
||||
.Where(f => new FileInfo(f).Name.Length == 40)
|
||||
.Select(s => s.ToLowerInvariant()).ToList())
|
||||
.ToDictionary(c => c, c => false);
|
||||
|
||||
TotalFiles = scannedFiles.Count;
|
||||
|
||||
// scan files from database
|
||||
var cpuCount = (int)(Environment.ProcessorCount / 2.0f);
|
||||
Task[] dbTasks = Enumerable.Range(0, cpuCount).Select(c => Task.CompletedTask).ToArray();
|
||||
|
||||
ConcurrentBag<FileCache> entitiesToRemove = new();
|
||||
ConcurrentBag<FileCache> entitiesToUpdate = new();
|
||||
try
|
||||
{
|
||||
foreach (var value in _fileDbManager.GetAllFileCaches())
|
||||
{
|
||||
var idx = Task.WaitAny(dbTasks, ct);
|
||||
dbTasks[idx] = Task.Run(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = _fileDbManager.ValidateFileCacheEntity(value);
|
||||
scannedFiles[result.Item2.ResolvedFilepath] = true;
|
||||
if (result.Item1 == FileState.RequireUpdate)
|
||||
{
|
||||
entitiesToUpdate.Add(result.Item2);
|
||||
}
|
||||
else if (result.Item1 == FileState.RequireDeletion)
|
||||
{
|
||||
entitiesToRemove.Add(result.Item2);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.Warn("Failed validating " + value.ResolvedFilepath);
|
||||
Logger.Warn(ex.Message);
|
||||
Logger.Warn(ex.StackTrace);
|
||||
}
|
||||
|
||||
Interlocked.Increment(ref currentFileProgress);
|
||||
Thread.Sleep(1);
|
||||
}, ct);
|
||||
|
||||
if (ct.IsCancellationRequested) return;
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.Warn("Error during enumerating FileCaches: " + ex.Message);
|
||||
}
|
||||
|
||||
Task.WaitAll(dbTasks);
|
||||
|
||||
if (entitiesToUpdate.Any() || entitiesToRemove.Any())
|
||||
{
|
||||
foreach (var entity in entitiesToUpdate)
|
||||
{
|
||||
_fileDbManager.UpdateHash(entity);
|
||||
}
|
||||
|
||||
foreach (var entity in entitiesToRemove)
|
||||
{
|
||||
_fileDbManager.RemoveHash(entity);
|
||||
}
|
||||
|
||||
_fileDbManager.WriteOutFullCsv();
|
||||
}
|
||||
|
||||
Logger.Debug("Scanner validated existing db files");
|
||||
|
||||
if (ct.IsCancellationRequested) return;
|
||||
|
||||
// scan new files
|
||||
foreach (var c in scannedFiles.Where(c => c.Value == false))
|
||||
{
|
||||
var idx = Task.WaitAny(dbTasks, ct);
|
||||
dbTasks[idx] = Task.Run(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
var entry = _fileDbManager.CreateFileEntry(c.Key);
|
||||
if (entry == null) _ = _fileDbManager.CreateCacheEntry(c.Key);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.Warn("Failed adding " + c.Key);
|
||||
Logger.Warn(ex.Message);
|
||||
}
|
||||
|
||||
Interlocked.Increment(ref currentFileProgress);
|
||||
Thread.Sleep(1);
|
||||
}, ct);
|
||||
|
||||
if (ct.IsCancellationRequested) return;
|
||||
}
|
||||
|
||||
Task.WaitAll(dbTasks);
|
||||
|
||||
Logger.Debug("Scanner added new files to db");
|
||||
|
||||
Logger.Debug("Scan complete");
|
||||
TotalFiles = 0;
|
||||
currentFileProgress = 0;
|
||||
entitiesToRemove.Clear();
|
||||
scannedFiles.Clear();
|
||||
dbTasks = Array.Empty<Task>();
|
||||
|
||||
if (!_pluginConfiguration.InitialScanComplete)
|
||||
{
|
||||
_pluginConfiguration.InitialScanComplete = true;
|
||||
_pluginConfiguration.Save();
|
||||
}
|
||||
}
|
||||
|
||||
private void StartScan()
|
||||
{
|
||||
if (!_ipcManager.Initialized || !_pluginConfiguration.HasValidSetup()) return;
|
||||
Logger.Verbose("Penumbra is active, configuration is valid, starting watchers and scan");
|
||||
InvokeScan(true);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user