using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using MareSynchronos.Managers; using MareSynchronos.Utils; using MareSynchronos.WebAPI; using Microsoft.EntityFrameworkCore; namespace MareSynchronos.FileCacheDB; public class PeriodicFileScanner : IDisposable { private readonly IpcManager _ipcManager; private readonly Configuration _pluginConfiguration; private readonly FileDbManager _fileDbManager; private readonly ApiController _apiController; private CancellationTokenSource? _scanCancellationTokenSource; private Task? _fileScannerTask = null; public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileDbManager fileDbManager, ApiController apiController) { Logger.Verbose("Creating " + nameof(PeriodicFileScanner)); _ipcManager = ipcManager; _pluginConfiguration = pluginConfiguration; _fileDbManager = fileDbManager; _apiController = apiController; _ipcManager.PenumbraInitialized += StartScan; if (!string.IsNullOrEmpty(_ipcManager.PenumbraModDirectory())) { StartScan(); } _apiController.DownloadStarted += _apiController_DownloadStarted; _apiController.DownloadFinished += _apiController_DownloadFinished; } private void _apiController_DownloadFinished() { InvokeScan(); } private void _apiController_DownloadStarted() { _scanCancellationTokenSource?.Cancel(); } private long currentFileProgress = 0; public long CurrentFileProgress => currentFileProgress; public long FileCacheSize { get; set; } public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0; public long TotalFiles { get; private set; } public string TimeUntilNextScan => _timeUntilNextScan.ToString(@"mm\:ss"); private TimeSpan _timeUntilNextScan = TimeSpan.Zero; private int timeBetweenScans => _pluginConfiguration.TimeSpanBetweenScansInSeconds; public void Dispose() { Logger.Verbose("Disposing " + nameof(PeriodicFileScanner)); _ipcManager.PenumbraInitialized -= StartScan; _apiController.DownloadStarted -= _apiController_DownloadStarted; _apiController.DownloadFinished -= _apiController_DownloadFinished; _scanCancellationTokenSource?.Cancel(); } public void InvokeScan(bool forced = false) { bool isForced = forced; _scanCancellationTokenSource?.Cancel(); _scanCancellationTokenSource = new CancellationTokenSource(); var token = _scanCancellationTokenSource.Token; _fileScannerTask = Task.Run(async () => { while (!token.IsCancellationRequested) { isForced |= RecalculateFileCacheSize(); if (!_pluginConfiguration.FileScanPaused || isForced) { isForced = false; await PeriodicFileScan(token); } _timeUntilNextScan = TimeSpan.FromSeconds(timeBetweenScans); while (_timeUntilNextScan.TotalSeconds >= 0) { await Task.Delay(TimeSpan.FromSeconds(1), token); _timeUntilNextScan -= TimeSpan.FromSeconds(1); } } }); } internal void StartWatchers() { InvokeScan(); } public bool RecalculateFileCacheSize() { FileCacheSize = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder).Sum(f => { try { return new FileInfo(f).Length; } catch { return 0; } }); if (FileCacheSize < (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024) return false; var allFiles = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder) .Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList(); while (FileCacheSize > (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024) { var oldestFile = allFiles.First(); FileCacheSize -= oldestFile.Length; File.Delete(oldestFile.FullName); allFiles.Remove(oldestFile); } return true; } private async Task PeriodicFileScan(CancellationToken ct) { TotalFiles = 1; var penumbraDir = _ipcManager.PenumbraModDirectory(); bool penDirExists = true; bool cacheDirExists = true; if (string.IsNullOrEmpty(penumbraDir) || !Directory.Exists(penumbraDir)) { penDirExists = false; Logger.Warn("Penumbra directory is not set or does not exist."); } if (string.IsNullOrEmpty(_pluginConfiguration.CacheFolder) || !Directory.Exists(_pluginConfiguration.CacheFolder)) { cacheDirExists = false; Logger.Warn("Mare Cache directory is not set or does not exist."); } if (!penDirExists || !cacheDirExists) { return; } Logger.Debug("Getting files from " + penumbraDir + " and " + _pluginConfiguration.CacheFolder); string[] ext = { ".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp" }; var scannedFiles = new ConcurrentDictionary( Directory.EnumerateFiles(penumbraDir, "*.*", SearchOption.AllDirectories) .Select(s => new FileInfo(s)) .Where(f => ext.Contains(f.Extension) && !f.FullName.Contains(@"\bg\") && !f.FullName.Contains(@"\bgcommon\") && !f.FullName.Contains(@"\ui\")) .Select(f => f.FullName.ToLowerInvariant()) .Concat(Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.AllDirectories) .Where(f => new FileInfo(f).Name.Length == 40) .Select(s => s.ToLowerInvariant())) .Select(p => new KeyValuePair(p, false)).ToList()); List fileDbEntries; using (var db = new FileCacheContext()) { fileDbEntries = await db.FileCaches.ToListAsync(cancellationToken: ct); } TotalFiles = scannedFiles.Count; Logger.Debug("Database contains " + fileDbEntries.Count + " files, local system contains " + TotalFiles); // scan files from database Parallel.ForEach(fileDbEntries.ToList(), new ParallelOptions() { MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan, CancellationToken = ct, }, dbEntry => { if (ct.IsCancellationRequested) return; try { var file = _fileDbManager.ValidateFileCache(dbEntry); if (file != null && scannedFiles.ContainsKey(file.Filepath)) { scannedFiles[file.Filepath] = true; } } catch (Exception ex) { Logger.Warn(ex.Message); Logger.Warn(ex.StackTrace); } Interlocked.Increment(ref currentFileProgress); }); Logger.Debug("Scanner validated existing db files"); if (ct.IsCancellationRequested) return; // scan new files Parallel.ForEach(scannedFiles.Where(c => c.Value == false), new ParallelOptions() { MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan, CancellationToken = ct }, file => { if (ct.IsCancellationRequested) return; _ = _fileDbManager.CreateFileCacheEntity(file.Key); Interlocked.Increment(ref currentFileProgress); }); Logger.Debug("Scanner added new files to db"); Logger.Debug("Scan complete"); TotalFiles = 0; currentFileProgress = 0; if (!_pluginConfiguration.InitialScanComplete) { _pluginConfiguration.InitialScanComplete = true; _pluginConfiguration.Save(); } } private void StartScan() { if (!_ipcManager.Initialized || !_pluginConfiguration.HasValidSetup()) return; Logger.Verbose("Penumbra is active, configuration is valid, starting watchers and scan"); InvokeScan(true); } }