t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add periodic file scanner, parallelize downloads, fix transient files being readded when not necessary, fix disposal of players on plugin shutdown

Browse Source
This commit is contained in:
Stanley Dimant
2022-09-25 14:38:06 +02:00
parent 25e87e6ec2
commit 0d7e173a97
20 changed files with 641 additions and 525 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
MareSynchronos/Managers/CachedPlayer.cs
View File

@@ -3,32 +3,30 @@ using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Dalamud.Game.ClientState.Objects.SubKinds;
using Dalamud.Logging;
using FFXIVClientStructs.FFXIV.Client.Game.Character;
using MareSynchronos.API;
using MareSynchronos.FileCacheDB;
using MareSynchronos.Interop;
using MareSynchronos.Models;
using MareSynchronos.Utils;
using MareSynchronos.WebAPI;
using Newtonsoft.Json;
namespace MareSynchronos.Managers;
public class CachedPlayer
{
private readonly DalamudUtil _dalamudUtil;
private readonly FileDbManager fileDbManager;
private readonly IpcManager _ipcManager;
private readonly ApiController _apiController;
private bool _isVisible;
public CachedPlayer(string nameHash, IpcManager ipcManager, ApiController apiController, DalamudUtil dalamudUtil)
public CachedPlayer(string nameHash, IpcManager ipcManager, ApiController apiController, DalamudUtil dalamudUtil, FileDbManager fileDbManager)
{
PlayerNameHash = nameHash;
_ipcManager = ipcManager;
_apiController = apiController;
_dalamudUtil = dalamudUtil;
this.fileDbManager = fileDbManager;
}
public bool IsVisible
@@ -196,12 +194,11 @@ public class CachedPlayer
moddedDictionary = new Dictionary<string, string>();
try
{
using var db = new FileCacheContext();
foreach (var item in _cachedData.FileReplacements.SelectMany(k => k.Value.Where(v => string.IsNullOrEmpty(v.FileSwapPath))).ToList())
{
foreach (var gamePath in item.GamePaths)
{
var fileCache = db.FileCaches.FirstOrDefault(f => f.Hash == item.Hash);
var fileCache = fileDbManager.GetFileCacheByHash(item.Hash);
if (fileCache != null)
{
moddedDictionary[gamePath] = fileCache.Filepath;
@@ -311,7 +308,7 @@ public class CachedPlayer
if (companion != IntPtr.Zero)
{
Logger.Debug("Request Redraw for Companion");
_dalamudUtil.WaitWhileCharacterIsDrawing(PlayerName! + " companion", companion, 10000,ct);
_dalamudUtil.WaitWhileCharacterIsDrawing(PlayerName! + " companion", companion, 10000, ct);
ct.ThrowIfCancellationRequested();
if (_ipcManager.CheckGlamourerApi() && !string.IsNullOrEmpty(glamourerData))
{

381
MareSynchronos/Managers/FileCacheManager.cs
View File

@@ -1,381 +0,0 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Dalamud.Logging;
using MareSynchronos.FileCacheDB;
using MareSynchronos.Utils;
using Microsoft.EntityFrameworkCore;
namespace MareSynchronos.Managers
{
public class FileCacheManager : IDisposable
{
private readonly IpcManager _ipcManager;
private readonly ConcurrentBag<string> _modifiedFiles = new();
private readonly Configuration _pluginConfiguration;
private FileSystemWatcher? _cacheDirWatcher;
private FileSystemWatcher? _penumbraDirWatcher;
private Task? _rescanTask;
private readonly CancellationTokenSource _rescanTaskCancellationTokenSource = new();
private CancellationTokenSource _rescanTaskRunCancellationTokenSource = new();
private CancellationTokenSource? _scanCancellationTokenSource;
private object modifiedFilesLock = new object();
public FileCacheManager(IpcManager ipcManager, Configuration pluginConfiguration)
{
Logger.Verbose("Creating " + nameof(FileCacheManager));
_ipcManager = ipcManager;
_pluginConfiguration = pluginConfiguration;
StartWatchersAndScan();
_ipcManager.PenumbraInitialized += StartWatchersAndScan;
_ipcManager.PenumbraDisposed += StopWatchersAndScan;
}
public long CurrentFileProgress { get; private set; }
public long FileCacheSize { get; set; }
public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
public long TotalFiles { get; private set; }
public string WatchedCacheDirectory => (_cacheDirWatcher?.EnableRaisingEvents ?? false) ? _cacheDirWatcher!.Path : "Not watched";
public string WatchedPenumbraDirectory => (_penumbraDirWatcher?.EnableRaisingEvents ?? false) ? _penumbraDirWatcher!.Path : "Not watched";
public FileCache? Create(string file, CancellationToken? token)
{
FileInfo fileInfo = new(file);
int attempt = 0;
while (IsFileLocked(fileInfo) && attempt++ <= 10)
{
Thread.Sleep(1000);
Logger.Debug("Waiting for file release " + fileInfo.FullName + " attempt " + attempt);
token?.ThrowIfCancellationRequested();
}
if (attempt >= 10) return null;
var sha1Hash = Crypto.GetFileHash(fileInfo.FullName);
return new FileCache()
{
Filepath = fileInfo.FullName.ToLowerInvariant(),
Hash = sha1Hash,
LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString(),
};
}
public void Dispose()
{
Logger.Verbose("Disposing " + nameof(FileCacheManager));
_ipcManager.PenumbraInitialized -= StartWatchersAndScan;
_ipcManager.PenumbraDisposed -= StopWatchersAndScan;
_rescanTaskCancellationTokenSource?.Cancel();
_rescanTaskRunCancellationTokenSource?.Cancel();
_scanCancellationTokenSource?.Cancel();
StopWatchersAndScan();
}
public void StartInitialScan()
{
_scanCancellationTokenSource = new CancellationTokenSource();
Task.Run(() => StartFileScan(_scanCancellationTokenSource.Token));
}
public void StartWatchers()
{
if (!_ipcManager.Initialized || string.IsNullOrEmpty(_pluginConfiguration.CacheFolder)) return;
Logger.Verbose("Starting File System Watchers");
_penumbraDirWatcher?.Dispose();
_cacheDirWatcher?.Dispose();
_penumbraDirWatcher = new FileSystemWatcher(_ipcManager.PenumbraModDirectory()!)
{
IncludeSubdirectories = true,
};
_penumbraDirWatcher.NotifyFilter = NotifyFilters.FileName | NotifyFilters.Size;
_penumbraDirWatcher.Deleted += OnModified;
_penumbraDirWatcher.Changed += OnModified;
_penumbraDirWatcher.Renamed += OnModified;
_penumbraDirWatcher.Filters.Add("*.mtrl");
_penumbraDirWatcher.Filters.Add("*.mdl");
_penumbraDirWatcher.Filters.Add("*.tex");
_penumbraDirWatcher.Error += (sender, args) => PluginLog.Error(args.GetException(), "Error in Penumbra Dir Watcher");
_penumbraDirWatcher.EnableRaisingEvents = true;
_cacheDirWatcher = new FileSystemWatcher(_pluginConfiguration.CacheFolder)
{
IncludeSubdirectories = false,
};
_cacheDirWatcher.NotifyFilter = NotifyFilters.FileName | NotifyFilters.Size;
_cacheDirWatcher.Deleted += OnModified;
_cacheDirWatcher.Changed += OnModified;
_cacheDirWatcher.Renamed += OnModified;
_cacheDirWatcher.Filters.Add("*");
_cacheDirWatcher.Error +=
(sender, args) => PluginLog.Error(args.GetException(), "Error in Cache Dir Watcher");
_cacheDirWatcher.EnableRaisingEvents = true;
Task.Run(RecalculateFileCacheSize);
}
private bool IsFileLocked(FileInfo file)
{
try
{
using var fs = file.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
}
catch
{
return true;
}
return false;
}
private void OnModified(object sender, FileSystemEventArgs e)
{
lock (modifiedFilesLock)
{
_modifiedFiles.Add(e.FullPath);
}
_ = StartRescan();
}
private void RecalculateFileCacheSize()
{
FileCacheSize = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder).Sum(f =>
{
try
{
return new FileInfo(f).Length;
}
catch
{
return 0;
}
});
if (FileCacheSize < (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024) return;
var allFiles = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder)
.Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList();
while (FileCacheSize > (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024)
{
var oldestFile = allFiles.First();
FileCacheSize -= oldestFile.Length;
File.Delete(oldestFile.FullName);
allFiles.Remove(oldestFile);
}
}
public async Task StartRescan(bool force = false)
{
_rescanTaskRunCancellationTokenSource.Cancel();
_rescanTaskRunCancellationTokenSource = new CancellationTokenSource();
var token = _rescanTaskRunCancellationTokenSource.Token;
if (!force)
await Task.Delay(TimeSpan.FromSeconds(1), token);
while ((!_rescanTask?.IsCompleted ?? false) && !token.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(1), token);
}
if (token.IsCancellationRequested) return;
Logger.Debug("File changes detected");
lock (modifiedFilesLock)
{
if (!_modifiedFiles.Any()) return;
}
_rescanTask = Task.Run(async () =>
{
List<string> modifiedFilesCopy = new List<string>();
lock (modifiedFilesLock)
{
modifiedFilesCopy = _modifiedFiles.ToList();
_modifiedFiles.Clear();
}
await using var db = new FileCacheContext();
foreach (var item in modifiedFilesCopy.Distinct())
{
var fi = new FileInfo(item);
if (!fi.Exists)
{
PluginLog.Verbose("Removed: " + item);
db.RemoveRange(db.FileCaches.Where(f => f.Filepath.ToLower() == item.ToLowerInvariant()));
}
else
{
PluginLog.Verbose("Changed :" + item);
var fileCache = Create(item, _rescanTaskCancellationTokenSource.Token);
if (fileCache != null)
{
db.RemoveRange(db.FileCaches.Where(f => f.Filepath.ToLower() == fileCache.Filepath.ToLowerInvariant()));
await db.AddAsync(fileCache, _rescanTaskCancellationTokenSource.Token);
}
}
}
await db.SaveChangesAsync(_rescanTaskCancellationTokenSource.Token);
RecalculateFileCacheSize();
}, _rescanTaskCancellationTokenSource.Token);
}
private async Task StartFileScan(CancellationToken ct)
{
TotalFiles = 1;
_scanCancellationTokenSource = new CancellationTokenSource();
var penumbraDir = _ipcManager.PenumbraModDirectory()!;
Logger.Debug("Getting files from " + penumbraDir + " and " + _pluginConfiguration.CacheFolder);
var scannedFiles = new ConcurrentDictionary<string, bool>(
Directory.EnumerateFiles(penumbraDir, "*.*", SearchOption.AllDirectories)
.Select(s => s.ToLowerInvariant())
.Where(f => f.Contains(@"\chara\"))
.Where(f =>
(f.EndsWith(".tex", StringComparison.OrdinalIgnoreCase)
|| f.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase)
|| f.EndsWith(".mtrl", StringComparison.OrdinalIgnoreCase)))
.Concat(Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.AllDirectories)
.Where(f => new FileInfo(f).Name.Length == 40)
.Select(s => s.ToLowerInvariant()))
.Select(p => new KeyValuePair<string, bool>(p, false)).ToList());
List<FileCache> fileCaches;
await using (var db = new FileCacheContext())
fileCaches = db.FileCaches.ToList();
TotalFiles = scannedFiles.Count;
var fileCachesToDelete = new ConcurrentBag<FileCache>();
var fileCachesToAdd = new ConcurrentBag<FileCache>();
Logger.Debug("Database contains " + fileCaches.Count + " files, local system contains " + TotalFiles);
// scan files from database
Parallel.ForEach(fileCaches, new ParallelOptions()
{
MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
CancellationToken = ct,
},
cache =>
{
if (ct.IsCancellationRequested) return;
if (!File.Exists(cache.Filepath))
{
fileCachesToDelete.Add(cache);
}
else
{
if (scannedFiles.ContainsKey(cache.Filepath))
{
scannedFiles[cache.Filepath] = true;
}
FileInfo fileInfo = new(cache.Filepath);
if (fileInfo.LastWriteTimeUtc.Ticks == long.Parse(cache.LastModifiedDate)) return;
var newCache = Create(cache.Filepath, ct);
if (newCache != null)
{
fileCachesToAdd.Add(newCache);
fileCachesToDelete.Add(cache);
}
}
var files = CurrentFileProgress;
Interlocked.Increment(ref files);
CurrentFileProgress = files;
});
if (ct.IsCancellationRequested) return;
// scan new files
Parallel.ForEach(scannedFiles.Where(c => c.Value == false), new ParallelOptions()
{
MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
CancellationToken = ct
},
file =>
{
var newCache = Create(file.Key, ct);
if (newCache != null)
{
fileCachesToAdd.Add(newCache);
}
var files = CurrentFileProgress;
Interlocked.Increment(ref files);
CurrentFileProgress = files;
});
if (fileCachesToAdd.Any() || fileCachesToDelete.Any())
{
await using FileCacheContext db = new();
Logger.Debug("Found " + fileCachesToAdd.Count + " additions and " + fileCachesToDelete.Count + " deletions");
try
{
foreach (var deletion in fileCachesToDelete)
{
var entries = db.FileCaches.Where(f =>
f.Hash == deletion.Hash && f.Filepath.ToLower() == deletion.Filepath.ToLower());
if (await entries.AnyAsync(ct))
{
Logger.Verbose("Removing file from DB: " + deletion.Filepath);
db.FileCaches.RemoveRange(entries);
}
}
await db.SaveChangesAsync(ct);
foreach (var entry in fileCachesToAdd)
{
try
{
db.FileCaches.Add(entry);
}
catch
{
// ignored
}
}
await db.SaveChangesAsync(ct);
}
catch (Exception ex)
{
PluginLog.Error(ex, ex.Message);
}
}
Logger.Debug("Scan complete");
TotalFiles = 0;
CurrentFileProgress = 0;
if (!_pluginConfiguration.InitialScanComplete)
{
_pluginConfiguration.InitialScanComplete = true;
_pluginConfiguration.Save();
}
}
private void StartWatchersAndScan()
{
if (!_ipcManager.Initialized || !_pluginConfiguration.HasValidSetup()) return;
Logger.Verbose("Penumbra is active, configuration is valid, starting watchers and scan");
StartWatchers();
StartInitialScan();
}
private void StopWatchersAndScan()
{
_cacheDirWatcher?.Dispose();
}
}
}

9
MareSynchronos/Managers/IpcManager.cs
View File

@@ -174,11 +174,16 @@ namespace MareSynchronos.Managers
while (actionQueue.Count > 0 && totalSleepTime < 2000)
{
Logger.Verbose("Waiting for actionqueue to clear...");
HandleActionQueue();
System.Threading.Thread.Sleep(16);
totalSleepTime += 16;
}
Logger.Verbose("Action queue clear or not, disposing");
if (totalSleepTime >= 2000)
{
Logger.Verbose("Action queue clear or not, disposing");
}
_dalamudUtil.FrameworkUpdate -= HandleActionQueue;
_dalamudUtil.ZoneSwitchEnd -= ClearActionQueue;
actionQueue.Clear();
@@ -304,7 +309,7 @@ namespace MareSynchronos.Managers
public string? PenumbraModDirectory()
{
if (!CheckPenumbraApi()) return null;
return _penumbraResolveModDir!.InvokeFunc();
return _penumbraResolveModDir!.InvokeFunc().ToLowerInvariant();
}
public void PenumbraRedraw(IntPtr obj)

7
MareSynchronos/Managers/OnlinePlayerManager.cs
View File

@@ -17,6 +17,7 @@ public class OnlinePlayerManager : IDisposable
private readonly DalamudUtil _dalamudUtil;
private readonly IpcManager _ipcManager;
private readonly PlayerManager _playerManager;
private readonly FileDbManager _fileDbManager;
private readonly ConcurrentDictionary<string, CachedPlayer> _onlineCachedPlayers = new();
private readonly ConcurrentDictionary<string, CharacterCacheDto> _temporaryStoredCharacterCache = new();
private readonly ConcurrentDictionary<CachedPlayer, CancellationTokenSource> _playerTokenDisposal = new();
@@ -24,7 +25,7 @@ public class OnlinePlayerManager : IDisposable
private List<string> OnlineVisiblePlayerHashes => _onlineCachedPlayers.Select(p => p.Value).Where(p => p.PlayerCharacter != IntPtr.Zero)
.Select(p => p.PlayerNameHash).ToList();
public OnlinePlayerManager(ApiController apiController, DalamudUtil dalamudUtil, IpcManager ipcManager, PlayerManager playerManager)
public OnlinePlayerManager(ApiController apiController, DalamudUtil dalamudUtil, IpcManager ipcManager, PlayerManager playerManager, FileDbManager fileDbManager)
{
Logger.Verbose("Creating " + nameof(OnlinePlayerManager));
@@ -32,7 +33,7 @@ public class OnlinePlayerManager : IDisposable
_dalamudUtil = dalamudUtil;
_ipcManager = ipcManager;
_playerManager = playerManager;
_fileDbManager = fileDbManager;
_apiController.PairedClientOnline += ApiControllerOnPairedClientOnline;
_apiController.PairedClientOffline += ApiControllerOnPairedClientOffline;
_apiController.PairedWithOther += ApiControllerOnPairedWithOther;
@@ -249,6 +250,6 @@ public class OnlinePlayerManager : IDisposable
private CachedPlayer CreateCachedPlayer(string hashedName)
{
return new CachedPlayer(hashedName, _ipcManager, _apiController, _dalamudUtil);
return new CachedPlayer(hashedName, _ipcManager, _apiController, _dalamudUtil, _fileDbManager);
}
}

2
MareSynchronos/Managers/TransientResourceManager.cs
View File

@@ -101,7 +101,7 @@ namespace MareSynchronos.Managers
filePath = filePath.Split("|")[2];
}
filePath = filePath.ToLowerInvariant();
filePath = filePath.ToLowerInvariant().Replace("\\", "/");
var replacedGamePath = gamePath.ToLowerInvariant().Replace("\\", "/");