add periodic file scanner, parallelize downloads, fix transient files being readded when not necessary, fix disposal of players on plugin shutdown

This commit is contained in:
Stanley Dimant
2022-09-25 14:38:06 +02:00
parent 25e87e6ec2
commit 0d7e173a97
20 changed files with 641 additions and 525 deletions

View File

@@ -1,12 +1,36 @@
#nullable disable
namespace MareSynchronos.FileCacheDB
{
public partial class FileCache
public class FileCache
{
public string Hash { get; set; }
public string Filepath { get; set; }
public string LastModifiedDate { get; set; }
public int Version { get; set; }
private FileCacheEntity entity;
public string Filepath { get; private set; }
public string Hash { get; private set; }
public string OriginalFilepath => entity.Filepath;
public string OriginalHash => entity.Hash;
public long LastModifiedDateTicks => long.Parse(entity.LastModifiedDate);
public FileCache(FileCacheEntity entity)
{
this.entity = entity;
}
public void SetResolvedFilePath(string filePath)
{
Filepath = filePath.ToLowerInvariant();
}
public void SetHash(string hash)
{
Hash = hash;
}
public void UpdateFileCache(FileCacheEntity entity)
{
this.entity = entity;
}
}
}

View File

@@ -9,7 +9,7 @@ namespace MareSynchronos.FileCacheDB
public partial class FileCacheContext : DbContext
{
private string DbPath { get; set; }
public FileCacheContext()
public FileCacheContext()
{
DbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "FileCache.db");
string oldDbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "FileCacheDebug.db");
@@ -36,7 +36,7 @@ namespace MareSynchronos.FileCacheDB
{
}
public virtual DbSet<FileCache> FileCaches { get; set; }
public virtual DbSet<FileCacheEntity> FileCaches { get; set; }
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
@@ -48,7 +48,7 @@ namespace MareSynchronos.FileCacheDB
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.Entity<FileCache>(entity =>
modelBuilder.Entity<FileCacheEntity>(entity =>
{
entity.HasKey(e => new { e.Hash, e.Filepath });

View File

@@ -0,0 +1,13 @@
#nullable disable
namespace MareSynchronos.FileCacheDB
{
public partial class FileCacheEntity
{
public string Hash { get; set; }
public string Filepath { get; set; }
public string LastModifiedDate { get; set; }
public int Version { get; set; }
}
}

View File

@@ -0,0 +1,227 @@
using MareSynchronos.FileCacheDB;
using MareSynchronos.Utils;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
namespace MareSynchronos.Managers;
public class FileDbManager
{
private const string PenumbraPrefix = "{penumbra}";
private const string CachePrefix = "{cache}";
private readonly IpcManager _ipcManager;
private readonly Configuration _configuration;
private static object _lock = new();
public FileDbManager(IpcManager ipcManager, Configuration configuration)
{
_ipcManager = ipcManager;
_configuration = configuration;
}
public FileCache? GetFileCacheByHash(string hash)
{
List<FileCacheEntity> matchingEntries = new List<FileCacheEntity>();
using (var db = new FileCacheContext())
{
matchingEntries = db.FileCaches.Where(f => f.Hash.ToLower() == hash.ToLower()).ToList();
}
if (!matchingEntries.Any()) return null;
if (matchingEntries.Any(f => f.Filepath.Contains(PenumbraPrefix) && matchingEntries.Any(f => f.Filepath.Contains(CachePrefix))))
{
var cachedEntries = matchingEntries.Where(f => f.Filepath.Contains(CachePrefix));
DeleteFromDatabase(cachedEntries.Select(f => new FileCache(f)));
foreach (var entry in cachedEntries)
{
matchingEntries.Remove(entry);
}
}
return GetValidatedFileCache(matchingEntries.First());
}
public FileCache? ValidateFileCache(FileCacheEntity fileCacheEntity)
{
return GetValidatedFileCache(fileCacheEntity);
}
public FileCache? GetFileCacheByPath(string path)
{
FileCacheEntity? matchingEntries = null;
var cleanedPath = path.Replace("/", "\\").ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), "");
using (var db = new FileCacheContext())
{
matchingEntries = db.FileCaches.FirstOrDefault(f => f.Filepath.EndsWith(cleanedPath));
}
if (matchingEntries == null)
{
return CreateFileCacheEntity(path);
}
var validatedCacheEntry = GetValidatedFileCache(matchingEntries);
return validatedCacheEntry;
}
public FileCache? CreateFileCacheEntity(string path)
{
Logger.Verbose("Creating entry for " + path);
FileInfo fi = new FileInfo(path);
if (!fi.Exists) return null;
string prefixedPath = fi.FullName.ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), PenumbraPrefix + "\\")
.Replace(_configuration.CacheFolder.ToLowerInvariant(), CachePrefix + "\\").Replace("\\\\", "\\");
var hash = Crypto.GetFileHash(path);
lock (_lock)
{
var entity = new FileCacheEntity();
entity.Hash = hash;
entity.Filepath = prefixedPath;
entity.LastModifiedDate = fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
try
{
using var db = new FileCacheContext();
db.FileCaches.Add(entity);
db.SaveChanges();
}
catch (Exception ex)
{
Logger.Warn("Could not add " + path);
}
}
return GetFileCacheByPath(prefixedPath)!;
}
private FileCache? GetValidatedFileCache(FileCacheEntity e)
{
var fileCache = new FileCache(e);
var resulingFileCache = MigrateLegacy(fileCache);
if (resulingFileCache == null) return null;
resulingFileCache = ReplacePathPrefixes(resulingFileCache);
resulingFileCache = Validate(resulingFileCache);
return resulingFileCache;
}
private FileCache? Validate(FileCache fileCache)
{
var file = new FileInfo(fileCache.Filepath);
if (!file.Exists)
{
DeleteFromDatabase(new[] { fileCache });
return null;
}
if (file.LastWriteTimeUtc.Ticks != fileCache.LastModifiedDateTicks)
{
fileCache.SetHash(Crypto.GetFileHash(fileCache.Filepath));
UpdateCacheHash(fileCache, file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture));
}
return fileCache;
}
private FileCache? MigrateLegacy(FileCache fileCache)
{
if (fileCache.OriginalFilepath.Contains(PenumbraPrefix) || fileCache.OriginalFilepath.Contains(CachePrefix)) return fileCache;
var fileInfo = new FileInfo(fileCache.OriginalFilepath);
var penumbraDir = _ipcManager.PenumbraModDirectory()!;
// check if it's a cache file
if (fileInfo.Exists && fileInfo.Name.Length == 40)
{
MigrateLegacyFilePath(fileCache, CachePrefix + "\\" + fileInfo.Name.ToLower());
}
else if (fileInfo.Exists && fileInfo.FullName.ToLowerInvariant().Contains(penumbraDir))
{
// attempt to replace penumbra mod folder path with {penumbra}
var newPath = PenumbraPrefix + fileCache.OriginalFilepath.ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!, string.Empty);
MigrateLegacyFilePath(fileCache, newPath);
}
else
{
DeleteFromDatabase(new[] { fileCache });
return null;
}
return fileCache;
}
private FileCache ReplacePathPrefixes(FileCache fileCache)
{
if (fileCache.OriginalFilepath.Contains(PenumbraPrefix))
{
fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory()));
}
else if (fileCache.OriginalFilepath.Contains(CachePrefix))
{
fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(CachePrefix, _configuration.CacheFolder));
}
return fileCache;
}
private void UpdateCacheHash(FileCache markedForUpdate, string lastModifiedDate)
{
lock (_lock)
{
Logger.Verbose("Updating Hash for " + markedForUpdate.OriginalFilepath);
using var db = new FileCacheContext();
var cache = db.FileCaches.First(f => f.Filepath == markedForUpdate.OriginalFilepath && f.Hash == markedForUpdate.OriginalHash);
var newcache = new FileCacheEntity()
{
Filepath = cache.Filepath,
Hash = markedForUpdate.Hash,
LastModifiedDate = lastModifiedDate
};
db.Remove(cache);
db.FileCaches.Add(newcache);
markedForUpdate.UpdateFileCache(newcache);
db.SaveChanges();
}
}
private void MigrateLegacyFilePath(FileCache fileCacheToMigrate, string newPath)
{
lock (_lock)
{
Logger.Verbose("Migrating legacy file path for " + fileCacheToMigrate.OriginalFilepath);
using var db = new FileCacheContext();
var cache = db.FileCaches.First(f => f.Filepath == fileCacheToMigrate.OriginalFilepath && f.Hash == fileCacheToMigrate.OriginalHash);
var newcache = new FileCacheEntity()
{
Filepath = newPath,
Hash = cache.Hash,
LastModifiedDate = cache.LastModifiedDate
};
db.Remove(cache);
db.FileCaches.Add(newcache);
fileCacheToMigrate.UpdateFileCache(newcache);
db.SaveChanges();
}
}
private void DeleteFromDatabase(IEnumerable<FileCache> markedForDeletion)
{
lock (_lock)
{
using var db = new FileCacheContext();
foreach (var item in markedForDeletion)
{
Logger.Verbose("Removing " + item.OriginalFilepath);
var itemToRemove = db.FileCaches.FirstOrDefault(f => f.Hash == item.OriginalHash && f.Filepath == item.OriginalFilepath);
if (itemToRemove == null) continue;
db.FileCaches.Remove(itemToRemove);
}
db.SaveChanges();
}
}
}

View File

@@ -0,0 +1,233 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MareSynchronos.Managers;
using MareSynchronos.Utils;
using MareSynchronos.WebAPI;
using Microsoft.EntityFrameworkCore;
namespace MareSynchronos.FileCacheDB;
public class PeriodicFileScanner : IDisposable
{
private readonly IpcManager _ipcManager;
private readonly Configuration _pluginConfiguration;
private readonly FileDbManager _fileDbManager;
private readonly ApiController _apiController;
private CancellationTokenSource? _scanCancellationTokenSource;
private Task? _fileScannerTask = null;
public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileDbManager fileDbManager, ApiController apiController)
{
Logger.Verbose("Creating " + nameof(PeriodicFileScanner));
_ipcManager = ipcManager;
_pluginConfiguration = pluginConfiguration;
_fileDbManager = fileDbManager;
_apiController = apiController;
_ipcManager.PenumbraInitialized += StartScan;
if (!string.IsNullOrEmpty(_ipcManager.PenumbraModDirectory()))
{
StartScan();
}
_apiController.DownloadStarted += _apiController_DownloadStarted;
_apiController.DownloadFinished += _apiController_DownloadFinished;
}
private void _apiController_DownloadFinished()
{
InvokeScan();
}
private void _apiController_DownloadStarted()
{
_scanCancellationTokenSource?.Cancel();
}
private long currentFileProgress = 0;
public long CurrentFileProgress => currentFileProgress;
public long FileCacheSize { get; set; }
public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
public long TotalFiles { get; private set; }
public string TimeUntilNextScan => _timeUntilNextScan.ToString(@"mm\:ss");
private TimeSpan _timeUntilNextScan = TimeSpan.Zero;
private int timeBetweenScans => _pluginConfiguration.TimeSpanBetweenScansInSeconds;
public void Dispose()
{
Logger.Verbose("Disposing " + nameof(PeriodicFileScanner));
_ipcManager.PenumbraInitialized -= StartScan;
_apiController.DownloadStarted -= _apiController_DownloadStarted;
_apiController.DownloadFinished -= _apiController_DownloadFinished;
_scanCancellationTokenSource?.Cancel();
}
public void InvokeScan()
{
_scanCancellationTokenSource?.Cancel();
_scanCancellationTokenSource = new CancellationTokenSource();
var token = _scanCancellationTokenSource.Token;
_fileScannerTask = Task.Run(async () =>
{
while (!token.IsCancellationRequested)
{
RecalculateFileCacheSize();
if (!_pluginConfiguration.FileScanPaused)
{
await PeriodicFileScan(token);
}
_timeUntilNextScan = TimeSpan.FromSeconds(timeBetweenScans);
while (_timeUntilNextScan.TotalSeconds >= 0)
{
await Task.Delay(TimeSpan.FromSeconds(1), token);
_timeUntilNextScan -= TimeSpan.FromSeconds(1);
}
}
});
}
internal void StartWatchers()
{
InvokeScan();
}
public void RecalculateFileCacheSize()
{
FileCacheSize = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder).Sum(f =>
{
try
{
return new FileInfo(f).Length;
}
catch
{
return 0;
}
});
if (FileCacheSize < (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024) return;
var allFiles = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder)
.Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList();
while (FileCacheSize > (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024)
{
var oldestFile = allFiles.First();
FileCacheSize -= oldestFile.Length;
File.Delete(oldestFile.FullName);
allFiles.Remove(oldestFile);
}
}
private async Task PeriodicFileScan(CancellationToken ct)
{
TotalFiles = 1;
var penumbraDir = _ipcManager.PenumbraModDirectory();
bool penDirExists = true;
bool cacheDirExists = true;
if (string.IsNullOrEmpty(penumbraDir) || !Directory.Exists(penumbraDir))
{
penDirExists = false;
Logger.Warn("Penumbra directory is not set or does not exist.");
}
if (string.IsNullOrEmpty(_pluginConfiguration.CacheFolder) || !Directory.Exists(_pluginConfiguration.CacheFolder))
{
cacheDirExists = false;
Logger.Warn("Mare Cache directory is not set or does not exist.");
}
if (!penDirExists || !cacheDirExists)
{
return;
}
Logger.Debug("Getting files from " + penumbraDir + " and " + _pluginConfiguration.CacheFolder);
string[] ext = { ".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp" };
var scannedFiles = new ConcurrentDictionary<string, bool>(
Directory.EnumerateFiles(penumbraDir, "*.*", SearchOption.AllDirectories)
.Select(s => new FileInfo(s))
.Where(f => ext.Contains(f.Extension) && !f.FullName.Contains(@"\bg\") && !f.FullName.Contains(@"\bgcommon\") && !f.FullName.Contains(@"\ui\"))
.Select(f => f.FullName.ToLowerInvariant())
.Concat(Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.AllDirectories)
.Where(f => new FileInfo(f).Name.Length == 40)
.Select(s => s.ToLowerInvariant()))
.Select(p => new KeyValuePair<string, bool>(p, false)).ToList());
List<FileCacheEntity> fileDbEntries;
using (var db = new FileCacheContext())
{
fileDbEntries = await db.FileCaches.ToListAsync(cancellationToken: ct);
}
TotalFiles = scannedFiles.Count;
Logger.Debug("Database contains " + fileDbEntries.Count + " files, local system contains " + TotalFiles);
// scan files from database
Parallel.ForEach(fileDbEntries.ToList(), new ParallelOptions()
{
MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
CancellationToken = ct,
},
dbEntry =>
{
if (ct.IsCancellationRequested) return;
try
{
var file = _fileDbManager.ValidateFileCache(dbEntry);
if (file != null && scannedFiles.ContainsKey(file.Filepath))
{
scannedFiles[file.Filepath] = true;
}
}
catch (Exception ex)
{
Logger.Warn(ex.Message);
Logger.Warn(ex.StackTrace);
}
Interlocked.Increment(ref currentFileProgress);
});
Logger.Debug("Scanner validated existing db files");
if (ct.IsCancellationRequested) return;
// scan new files
Parallel.ForEach(scannedFiles.Where(c => c.Value == false), new ParallelOptions()
{
MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
CancellationToken = ct
},
file =>
{
if (ct.IsCancellationRequested) return;
_ = _fileDbManager.CreateFileCacheEntity(file.Key);
Interlocked.Increment(ref currentFileProgress);
});
Logger.Debug("Scanner added new files to db");
Logger.Debug("Scan complete");
TotalFiles = 0;
currentFileProgress = 0;
if (!_pluginConfiguration.InitialScanComplete)
{
_pluginConfiguration.InitialScanComplete = true;
_pluginConfiguration.Save();
}
}
private void StartScan()
{
if (!_ipcManager.Initialized || !_pluginConfiguration.HasValidSetup()) return;
Logger.Verbose("Penumbra is active, configuration is valid, starting watchers and scan");
InvokeScan();
}
}