t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add periodic file scanner, parallelize downloads, fix transient files being readded when not necessary, fix disposal of players on plugin shutdown

Browse Source
This commit is contained in:
Stanley Dimant
2022-09-25 14:38:06 +02:00
parent 25e87e6ec2
commit 0d7e173a97
20 changed files with 641 additions and 525 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
MareSynchronos/Configuration.cs
View File

@@ -60,6 +60,9 @@ namespace MareSynchronos
public bool ReverseUserSort { get; set; } = true;
public int TimeSpanBetweenScansInSeconds { get; set; } = 30;
public bool FileScanPaused { get; set; } = false;
public bool InitialScanComplete { get; set; } = false;
public int MaxParallelScan
{
@@ -207,6 +210,12 @@ namespace MareSynchronos
Version = 5;
Save();
}
if (FileScanPaused)
{
FileScanPaused = false;
Save();
}
}
}
}

7
MareSynchronos/Factories/CharacterDataFactory.cs
View File

@@ -23,11 +23,12 @@ public class CharacterDataFactory
private readonly DalamudUtil _dalamudUtil;
private readonly IpcManager _ipcManager;
private readonly TransientResourceManager transientResourceManager;
private readonly FileDbManager fileDbManager;
public CharacterDataFactory(DalamudUtil dalamudUtil, IpcManager ipcManager, TransientResourceManager transientResourceManager)
public CharacterDataFactory(DalamudUtil dalamudUtil, IpcManager ipcManager, TransientResourceManager transientResourceManager, FileDbManager fileDbManager)
{
Logger.Verbose("Creating " + nameof(CharacterDataFactory));
this.fileDbManager = fileDbManager;
_dalamudUtil = dalamudUtil;
_ipcManager = ipcManager;
this.transientResourceManager = transientResourceManager;
@@ -416,7 +417,7 @@ public class CharacterDataFactory
private FileReplacement CreateFileReplacement(string path, bool doNotReverseResolve = false)
{
var fileReplacement = new FileReplacement(_ipcManager.PenumbraModDirectory()!);
var fileReplacement = new FileReplacement(fileDbManager);
if (!doNotReverseResolve)
{
fileReplacement.GamePaths =

34
MareSynchronos/FileCacheDB/FileCache.cs
View File

@@ -1,12 +1,36 @@
#nullable disable
namespace MareSynchronos.FileCacheDB
{
public partial class FileCache
public class FileCache
{
public string Hash { get; set; }
public string Filepath { get; set; }
public string LastModifiedDate { get; set; }
public int Version { get; set; }
private FileCacheEntity entity;
public string Filepath { get; private set; }
public string Hash { get; private set; }
public string OriginalFilepath => entity.Filepath;
public string OriginalHash => entity.Hash;
public long LastModifiedDateTicks => long.Parse(entity.LastModifiedDate);
public FileCache(FileCacheEntity entity)
{
this.entity = entity;
}
public void SetResolvedFilePath(string filePath)
{
Filepath = filePath.ToLowerInvariant();
}
public void SetHash(string hash)
{
Hash = hash;
}
public void UpdateFileCache(FileCacheEntity entity)
{
this.entity = entity;
}
}
}

4
MareSynchronos/FileCacheDB/FileCacheContext.cs
View File

@@ -36,7 +36,7 @@ namespace MareSynchronos.FileCacheDB
{
}
public virtual DbSet<FileCache> FileCaches { get; set; }
public virtual DbSet<FileCacheEntity> FileCaches { get; set; }
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
@@ -48,7 +48,7 @@ namespace MareSynchronos.FileCacheDB
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.Entity<FileCache>(entity =>
modelBuilder.Entity<FileCacheEntity>(entity =>
{
entity.HasKey(e => new { e.Hash, e.Filepath });

13
MareSynchronos/FileCacheDB/FileCacheEntity.cs Normal file
View File

@@ -0,0 +1,13 @@
#nullable disable
namespace MareSynchronos.FileCacheDB
{
public partial class FileCacheEntity
{
public string Hash { get; set; }
public string Filepath { get; set; }
public string LastModifiedDate { get; set; }
public int Version { get; set; }
}
}

227
MareSynchronos/FileCacheDB/FileDbManager.cs Normal file
View File

@@ -0,0 +1,227 @@
using MareSynchronos.FileCacheDB;
using MareSynchronos.Utils;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
namespace MareSynchronos.Managers;
public class FileDbManager
{
private const string PenumbraPrefix = "{penumbra}";
private const string CachePrefix = "{cache}";
private readonly IpcManager _ipcManager;
private readonly Configuration _configuration;
private static object _lock = new();
public FileDbManager(IpcManager ipcManager, Configuration configuration)
{
_ipcManager = ipcManager;
_configuration = configuration;
}
public FileCache? GetFileCacheByHash(string hash)
{
List<FileCacheEntity> matchingEntries = new List<FileCacheEntity>();
using (var db = new FileCacheContext())
{
matchingEntries = db.FileCaches.Where(f => f.Hash.ToLower() == hash.ToLower()).ToList();
}
if (!matchingEntries.Any()) return null;
if (matchingEntries.Any(f => f.Filepath.Contains(PenumbraPrefix) && matchingEntries.Any(f => f.Filepath.Contains(CachePrefix))))
{
var cachedEntries = matchingEntries.Where(f => f.Filepath.Contains(CachePrefix));
DeleteFromDatabase(cachedEntries.Select(f => new FileCache(f)));
foreach (var entry in cachedEntries)
{
matchingEntries.Remove(entry);
}
}
return GetValidatedFileCache(matchingEntries.First());
}
public FileCache? ValidateFileCache(FileCacheEntity fileCacheEntity)
{
return GetValidatedFileCache(fileCacheEntity);
}
public FileCache? GetFileCacheByPath(string path)
{
FileCacheEntity? matchingEntries = null;
var cleanedPath = path.Replace("/", "\\").ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), "");
using (var db = new FileCacheContext())
{
matchingEntries = db.FileCaches.FirstOrDefault(f => f.Filepath.EndsWith(cleanedPath));
}
if (matchingEntries == null)
{
return CreateFileCacheEntity(path);
}
var validatedCacheEntry = GetValidatedFileCache(matchingEntries);
return validatedCacheEntry;
}
public FileCache? CreateFileCacheEntity(string path)
{
Logger.Verbose("Creating entry for " + path);
FileInfo fi = new FileInfo(path);
if (!fi.Exists) return null;
string prefixedPath = fi.FullName.ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!.ToLowerInvariant(), PenumbraPrefix + "\\")
.Replace(_configuration.CacheFolder.ToLowerInvariant(), CachePrefix + "\\").Replace("\\\\", "\\");
var hash = Crypto.GetFileHash(path);
lock (_lock)
{
var entity = new FileCacheEntity();
entity.Hash = hash;
entity.Filepath = prefixedPath;
entity.LastModifiedDate = fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
try
{
using var db = new FileCacheContext();
db.FileCaches.Add(entity);
db.SaveChanges();
}
catch (Exception ex)
{
Logger.Warn("Could not add " + path);
}
}
return GetFileCacheByPath(prefixedPath)!;
}
private FileCache? GetValidatedFileCache(FileCacheEntity e)
{
var fileCache = new FileCache(e);
var resulingFileCache = MigrateLegacy(fileCache);
if (resulingFileCache == null) return null;
resulingFileCache = ReplacePathPrefixes(resulingFileCache);
resulingFileCache = Validate(resulingFileCache);
return resulingFileCache;
}
private FileCache? Validate(FileCache fileCache)
{
var file = new FileInfo(fileCache.Filepath);
if (!file.Exists)
{
DeleteFromDatabase(new[] { fileCache });
return null;
}
if (file.LastWriteTimeUtc.Ticks != fileCache.LastModifiedDateTicks)
{
fileCache.SetHash(Crypto.GetFileHash(fileCache.Filepath));
UpdateCacheHash(fileCache, file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture));
}
return fileCache;
}
private FileCache? MigrateLegacy(FileCache fileCache)
{
if (fileCache.OriginalFilepath.Contains(PenumbraPrefix) || fileCache.OriginalFilepath.Contains(CachePrefix)) return fileCache;
var fileInfo = new FileInfo(fileCache.OriginalFilepath);
var penumbraDir = _ipcManager.PenumbraModDirectory()!;
// check if it's a cache file
if (fileInfo.Exists && fileInfo.Name.Length == 40)
{
MigrateLegacyFilePath(fileCache, CachePrefix + "\\" + fileInfo.Name.ToLower());
}
else if (fileInfo.Exists && fileInfo.FullName.ToLowerInvariant().Contains(penumbraDir))
{
// attempt to replace penumbra mod folder path with {penumbra}
var newPath = PenumbraPrefix + fileCache.OriginalFilepath.ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory()!, string.Empty);
MigrateLegacyFilePath(fileCache, newPath);
}
else
{
DeleteFromDatabase(new[] { fileCache });
return null;
}
return fileCache;
}
private FileCache ReplacePathPrefixes(FileCache fileCache)
{
if (fileCache.OriginalFilepath.Contains(PenumbraPrefix))
{
fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory()));
}
else if (fileCache.OriginalFilepath.Contains(CachePrefix))
{
fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(CachePrefix, _configuration.CacheFolder));
}
return fileCache;
}
private void UpdateCacheHash(FileCache markedForUpdate, string lastModifiedDate)
{
lock (_lock)
{
Logger.Verbose("Updating Hash for " + markedForUpdate.OriginalFilepath);
using var db = new FileCacheContext();
var cache = db.FileCaches.First(f => f.Filepath == markedForUpdate.OriginalFilepath && f.Hash == markedForUpdate.OriginalHash);
var newcache = new FileCacheEntity()
{
Filepath = cache.Filepath,
Hash = markedForUpdate.Hash,
LastModifiedDate = lastModifiedDate
};
db.Remove(cache);
db.FileCaches.Add(newcache);
markedForUpdate.UpdateFileCache(newcache);
db.SaveChanges();
}
}
private void MigrateLegacyFilePath(FileCache fileCacheToMigrate, string newPath)
{
lock (_lock)
{
Logger.Verbose("Migrating legacy file path for " + fileCacheToMigrate.OriginalFilepath);
using var db = new FileCacheContext();
var cache = db.FileCaches.First(f => f.Filepath == fileCacheToMigrate.OriginalFilepath && f.Hash == fileCacheToMigrate.OriginalHash);
var newcache = new FileCacheEntity()
{
Filepath = newPath,
Hash = cache.Hash,
LastModifiedDate = cache.LastModifiedDate
};
db.Remove(cache);
db.FileCaches.Add(newcache);
fileCacheToMigrate.UpdateFileCache(newcache);
db.SaveChanges();
}
}
private void DeleteFromDatabase(IEnumerable<FileCache> markedForDeletion)
{
lock (_lock)
{
using var db = new FileCacheContext();
foreach (var item in markedForDeletion)
{
Logger.Verbose("Removing " + item.OriginalFilepath);
var itemToRemove = db.FileCaches.FirstOrDefault(f => f.Hash == item.OriginalHash && f.Filepath == item.OriginalFilepath);
if (itemToRemove == null) continue;
db.FileCaches.Remove(itemToRemove);
}
db.SaveChanges();
}
}
}

233
MareSynchronos/FileCacheDB/PeriodicFileScanner.cs Normal file
View File

@@ -0,0 +1,233 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MareSynchronos.Managers;
using MareSynchronos.Utils;
using MareSynchronos.WebAPI;
using Microsoft.EntityFrameworkCore;
namespace MareSynchronos.FileCacheDB;
public class PeriodicFileScanner : IDisposable
{
private readonly IpcManager _ipcManager;
private readonly Configuration _pluginConfiguration;
private readonly FileDbManager _fileDbManager;
private readonly ApiController _apiController;
private CancellationTokenSource? _scanCancellationTokenSource;
private Task? _fileScannerTask = null;
public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileDbManager fileDbManager, ApiController apiController)
{
Logger.Verbose("Creating " + nameof(PeriodicFileScanner));
_ipcManager = ipcManager;
_pluginConfiguration = pluginConfiguration;
_fileDbManager = fileDbManager;
_apiController = apiController;
_ipcManager.PenumbraInitialized += StartScan;
if (!string.IsNullOrEmpty(_ipcManager.PenumbraModDirectory()))
{
StartScan();
}
_apiController.DownloadStarted += _apiController_DownloadStarted;
_apiController.DownloadFinished += _apiController_DownloadFinished;
}
private void _apiController_DownloadFinished()
{
InvokeScan();
}
private void _apiController_DownloadStarted()
{
_scanCancellationTokenSource?.Cancel();
}
private long currentFileProgress = 0;
public long CurrentFileProgress => currentFileProgress;
public long FileCacheSize { get; set; }
public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
public long TotalFiles { get; private set; }
public string TimeUntilNextScan => _timeUntilNextScan.ToString(@"mm\:ss");
private TimeSpan _timeUntilNextScan = TimeSpan.Zero;
private int timeBetweenScans => _pluginConfiguration.TimeSpanBetweenScansInSeconds;
public void Dispose()
{
Logger.Verbose("Disposing " + nameof(PeriodicFileScanner));
_ipcManager.PenumbraInitialized -= StartScan;
_apiController.DownloadStarted -= _apiController_DownloadStarted;
_apiController.DownloadFinished -= _apiController_DownloadFinished;
_scanCancellationTokenSource?.Cancel();
}
public void InvokeScan()
{
_scanCancellationTokenSource?.Cancel();
_scanCancellationTokenSource = new CancellationTokenSource();
var token = _scanCancellationTokenSource.Token;
_fileScannerTask = Task.Run(async () =>
{
while (!token.IsCancellationRequested)
{
RecalculateFileCacheSize();
if (!_pluginConfiguration.FileScanPaused)
{
await PeriodicFileScan(token);
}
_timeUntilNextScan = TimeSpan.FromSeconds(timeBetweenScans);
while (_timeUntilNextScan.TotalSeconds >= 0)
{
await Task.Delay(TimeSpan.FromSeconds(1), token);
_timeUntilNextScan -= TimeSpan.FromSeconds(1);
}
}
});
}
internal void StartWatchers()
{
InvokeScan();
}
public void RecalculateFileCacheSize()
{
FileCacheSize = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder).Sum(f =>
{
try
{
return new FileInfo(f).Length;
}
catch
{
return 0;
}
});
if (FileCacheSize < (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024) return;
var allFiles = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder)
.Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList();
while (FileCacheSize > (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024)
{
var oldestFile = allFiles.First();
FileCacheSize -= oldestFile.Length;
File.Delete(oldestFile.FullName);
allFiles.Remove(oldestFile);
}
}
private async Task PeriodicFileScan(CancellationToken ct)
{
TotalFiles = 1;
var penumbraDir = _ipcManager.PenumbraModDirectory();
bool penDirExists = true;
bool cacheDirExists = true;
if (string.IsNullOrEmpty(penumbraDir) || !Directory.Exists(penumbraDir))
{
penDirExists = false;
Logger.Warn("Penumbra directory is not set or does not exist.");
}
if (string.IsNullOrEmpty(_pluginConfiguration.CacheFolder) || !Directory.Exists(_pluginConfiguration.CacheFolder))
{
cacheDirExists = false;
Logger.Warn("Mare Cache directory is not set or does not exist.");
}
if (!penDirExists || !cacheDirExists)
{
return;
}
Logger.Debug("Getting files from " + penumbraDir + " and " + _pluginConfiguration.CacheFolder);
string[] ext = { ".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp" };
var scannedFiles = new ConcurrentDictionary<string, bool>(
Directory.EnumerateFiles(penumbraDir, "*.*", SearchOption.AllDirectories)
.Select(s => new FileInfo(s))
.Where(f => ext.Contains(f.Extension) && !f.FullName.Contains(@"\bg\") && !f.FullName.Contains(@"\bgcommon\") && !f.FullName.Contains(@"\ui\"))
.Select(f => f.FullName.ToLowerInvariant())
.Concat(Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.AllDirectories)
.Where(f => new FileInfo(f).Name.Length == 40)
.Select(s => s.ToLowerInvariant()))
.Select(p => new KeyValuePair<string, bool>(p, false)).ToList());
List<FileCacheEntity> fileDbEntries;
using (var db = new FileCacheContext())
{
fileDbEntries = await db.FileCaches.ToListAsync(cancellationToken: ct);
}
TotalFiles = scannedFiles.Count;
Logger.Debug("Database contains " + fileDbEntries.Count + " files, local system contains " + TotalFiles);
// scan files from database
Parallel.ForEach(fileDbEntries.ToList(), new ParallelOptions()
{
MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
CancellationToken = ct,
},
dbEntry =>
{
if (ct.IsCancellationRequested) return;
try
{
var file = _fileDbManager.ValidateFileCache(dbEntry);
if (file != null && scannedFiles.ContainsKey(file.Filepath))
{
scannedFiles[file.Filepath] = true;
}
}
catch (Exception ex)
{
Logger.Warn(ex.Message);
Logger.Warn(ex.StackTrace);
}
Interlocked.Increment(ref currentFileProgress);
});
Logger.Debug("Scanner validated existing db files");
if (ct.IsCancellationRequested) return;
// scan new files
Parallel.ForEach(scannedFiles.Where(c => c.Value == false), new ParallelOptions()
{
MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
CancellationToken = ct
},
file =>
{
if (ct.IsCancellationRequested) return;
_ = _fileDbManager.CreateFileCacheEntity(file.Key);
Interlocked.Increment(ref currentFileProgress);
});
Logger.Debug("Scanner added new files to db");
Logger.Debug("Scan complete");
TotalFiles = 0;
currentFileProgress = 0;
if (!_pluginConfiguration.InitialScanComplete)
{
_pluginConfiguration.InitialScanComplete = true;
_pluginConfiguration.Save();
}
}
private void StartScan()
{
if (!_ipcManager.Initialized || !_pluginConfiguration.HasValidSetup()) return;
Logger.Verbose("Penumbra is active, configuration is valid, starting watchers and scan");
InvokeScan();
}
}

11
MareSynchronos/Managers/CachedPlayer.cs
View File

@@ -3,32 +3,30 @@ using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Dalamud.Game.ClientState.Objects.SubKinds;
using Dalamud.Logging;
using FFXIVClientStructs.FFXIV.Client.Game.Character;
using MareSynchronos.API;
using MareSynchronos.FileCacheDB;
using MareSynchronos.Interop;
using MareSynchronos.Models;
using MareSynchronos.Utils;
using MareSynchronos.WebAPI;
using Newtonsoft.Json;
namespace MareSynchronos.Managers;
public class CachedPlayer
{
private readonly DalamudUtil _dalamudUtil;
private readonly FileDbManager fileDbManager;
private readonly IpcManager _ipcManager;
private readonly ApiController _apiController;
private bool _isVisible;
public CachedPlayer(string nameHash, IpcManager ipcManager, ApiController apiController, DalamudUtil dalamudUtil)
public CachedPlayer(string nameHash, IpcManager ipcManager, ApiController apiController, DalamudUtil dalamudUtil, FileDbManager fileDbManager)
{
PlayerNameHash = nameHash;
_ipcManager = ipcManager;
_apiController = apiController;
_dalamudUtil = dalamudUtil;
this.fileDbManager = fileDbManager;
}
public bool IsVisible
@@ -196,12 +194,11 @@ public class CachedPlayer
moddedDictionary = new Dictionary<string, string>();
try
{
using var db = new FileCacheContext();
foreach (var item in _cachedData.FileReplacements.SelectMany(k => k.Value.Where(v => string.IsNullOrEmpty(v.FileSwapPath))).ToList())
{
foreach (var gamePath in item.GamePaths)
{
var fileCache = db.FileCaches.FirstOrDefault(f => f.Hash == item.Hash);
var fileCache = fileDbManager.GetFileCacheByHash(item.Hash);
if (fileCache != null)
{
moddedDictionary[gamePath] = fileCache.Filepath;

381
MareSynchronos/Managers/FileCacheManager.cs
View File

@@ -1,381 +0,0 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Dalamud.Logging;
using MareSynchronos.FileCacheDB;
using MareSynchronos.Utils;
using Microsoft.EntityFrameworkCore;
namespace MareSynchronos.Managers
{
public class FileCacheManager : IDisposable
{
private readonly IpcManager _ipcManager;
private readonly ConcurrentBag<string> _modifiedFiles = new();
private readonly Configuration _pluginConfiguration;
private FileSystemWatcher? _cacheDirWatcher;
private FileSystemWatcher? _penumbraDirWatcher;
private Task? _rescanTask;
private readonly CancellationTokenSource _rescanTaskCancellationTokenSource = new();
private CancellationTokenSource _rescanTaskRunCancellationTokenSource = new();
private CancellationTokenSource? _scanCancellationTokenSource;
private object modifiedFilesLock = new object();
public FileCacheManager(IpcManager ipcManager, Configuration pluginConfiguration)
{
Logger.Verbose("Creating " + nameof(FileCacheManager));
_ipcManager = ipcManager;
_pluginConfiguration = pluginConfiguration;
StartWatchersAndScan();
_ipcManager.PenumbraInitialized += StartWatchersAndScan;
_ipcManager.PenumbraDisposed += StopWatchersAndScan;
}
public long CurrentFileProgress { get; private set; }
public long FileCacheSize { get; set; }
public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0;
public long TotalFiles { get; private set; }
public string WatchedCacheDirectory => (_cacheDirWatcher?.EnableRaisingEvents ?? false) ? _cacheDirWatcher!.Path : "Not watched";
public string WatchedPenumbraDirectory => (_penumbraDirWatcher?.EnableRaisingEvents ?? false) ? _penumbraDirWatcher!.Path : "Not watched";
public FileCache? Create(string file, CancellationToken? token)
{
FileInfo fileInfo = new(file);
int attempt = 0;
while (IsFileLocked(fileInfo) && attempt++ <= 10)
{
Thread.Sleep(1000);
Logger.Debug("Waiting for file release " + fileInfo.FullName + " attempt " + attempt);
token?.ThrowIfCancellationRequested();
}
if (attempt >= 10) return null;
var sha1Hash = Crypto.GetFileHash(fileInfo.FullName);
return new FileCache()
{
Filepath = fileInfo.FullName.ToLowerInvariant(),
Hash = sha1Hash,
LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString(),
};
}
public void Dispose()
{
Logger.Verbose("Disposing " + nameof(FileCacheManager));
_ipcManager.PenumbraInitialized -= StartWatchersAndScan;
_ipcManager.PenumbraDisposed -= StopWatchersAndScan;
_rescanTaskCancellationTokenSource?.Cancel();
_rescanTaskRunCancellationTokenSource?.Cancel();
_scanCancellationTokenSource?.Cancel();
StopWatchersAndScan();
}
public void StartInitialScan()
{
_scanCancellationTokenSource = new CancellationTokenSource();
Task.Run(() => StartFileScan(_scanCancellationTokenSource.Token));
}
public void StartWatchers()
{
if (!_ipcManager.Initialized || string.IsNullOrEmpty(_pluginConfiguration.CacheFolder)) return;
Logger.Verbose("Starting File System Watchers");
_penumbraDirWatcher?.Dispose();
_cacheDirWatcher?.Dispose();
_penumbraDirWatcher = new FileSystemWatcher(_ipcManager.PenumbraModDirectory()!)
{
IncludeSubdirectories = true,
};
_penumbraDirWatcher.NotifyFilter = NotifyFilters.FileName | NotifyFilters.Size;
_penumbraDirWatcher.Deleted += OnModified;
_penumbraDirWatcher.Changed += OnModified;
_penumbraDirWatcher.Renamed += OnModified;
_penumbraDirWatcher.Filters.Add("*.mtrl");
_penumbraDirWatcher.Filters.Add("*.mdl");
_penumbraDirWatcher.Filters.Add("*.tex");
_penumbraDirWatcher.Error += (sender, args) => PluginLog.Error(args.GetException(), "Error in Penumbra Dir Watcher");
_penumbraDirWatcher.EnableRaisingEvents = true;
_cacheDirWatcher = new FileSystemWatcher(_pluginConfiguration.CacheFolder)
{
IncludeSubdirectories = false,
};
_cacheDirWatcher.NotifyFilter = NotifyFilters.FileName | NotifyFilters.Size;
_cacheDirWatcher.Deleted += OnModified;
_cacheDirWatcher.Changed += OnModified;
_cacheDirWatcher.Renamed += OnModified;
_cacheDirWatcher.Filters.Add("*");
_cacheDirWatcher.Error +=
(sender, args) => PluginLog.Error(args.GetException(), "Error in Cache Dir Watcher");
_cacheDirWatcher.EnableRaisingEvents = true;
Task.Run(RecalculateFileCacheSize);
}
private bool IsFileLocked(FileInfo file)
{
try
{
using var fs = file.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
}
catch
{
return true;
}
return false;
}
private void OnModified(object sender, FileSystemEventArgs e)
{
lock (modifiedFilesLock)
{
_modifiedFiles.Add(e.FullPath);
}
_ = StartRescan();
}
private void RecalculateFileCacheSize()
{
FileCacheSize = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder).Sum(f =>
{
try
{
return new FileInfo(f).Length;
}
catch
{
return 0;
}
});
if (FileCacheSize < (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024) return;
var allFiles = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder)
.Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList();
while (FileCacheSize > (long)_pluginConfiguration.MaxLocalCacheInGiB * 1024 * 1024 * 1024)
{
var oldestFile = allFiles.First();
FileCacheSize -= oldestFile.Length;
File.Delete(oldestFile.FullName);
allFiles.Remove(oldestFile);
}
}
public async Task StartRescan(bool force = false)
{
_rescanTaskRunCancellationTokenSource.Cancel();
_rescanTaskRunCancellationTokenSource = new CancellationTokenSource();
var token = _rescanTaskRunCancellationTokenSource.Token;
if (!force)
await Task.Delay(TimeSpan.FromSeconds(1), token);
while ((!_rescanTask?.IsCompleted ?? false) && !token.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(1), token);
}
if (token.IsCancellationRequested) return;
Logger.Debug("File changes detected");
lock (modifiedFilesLock)
{
if (!_modifiedFiles.Any()) return;
}
_rescanTask = Task.Run(async () =>
{
List<string> modifiedFilesCopy = new List<string>();
lock (modifiedFilesLock)
{
modifiedFilesCopy = _modifiedFiles.ToList();
_modifiedFiles.Clear();
}
await using var db = new FileCacheContext();
foreach (var item in modifiedFilesCopy.Distinct())
{
var fi = new FileInfo(item);
if (!fi.Exists)
{
PluginLog.Verbose("Removed: " + item);
db.RemoveRange(db.FileCaches.Where(f => f.Filepath.ToLower() == item.ToLowerInvariant()));
}
else
{
PluginLog.Verbose("Changed :" + item);
var fileCache = Create(item, _rescanTaskCancellationTokenSource.Token);
if (fileCache != null)
{
db.RemoveRange(db.FileCaches.Where(f => f.Filepath.ToLower() == fileCache.Filepath.ToLowerInvariant()));
await db.AddAsync(fileCache, _rescanTaskCancellationTokenSource.Token);
}
}
}
await db.SaveChangesAsync(_rescanTaskCancellationTokenSource.Token);
RecalculateFileCacheSize();
}, _rescanTaskCancellationTokenSource.Token);
}
private async Task StartFileScan(CancellationToken ct)
{
TotalFiles = 1;
_scanCancellationTokenSource = new CancellationTokenSource();
var penumbraDir = _ipcManager.PenumbraModDirectory()!;
Logger.Debug("Getting files from " + penumbraDir + " and " + _pluginConfiguration.CacheFolder);
var scannedFiles = new ConcurrentDictionary<string, bool>(
Directory.EnumerateFiles(penumbraDir, "*.*", SearchOption.AllDirectories)
.Select(s => s.ToLowerInvariant())
.Where(f => f.Contains(@"\chara\"))
.Where(f =>
(f.EndsWith(".tex", StringComparison.OrdinalIgnoreCase)
|| f.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase)
|| f.EndsWith(".mtrl", StringComparison.OrdinalIgnoreCase)))
.Concat(Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.AllDirectories)
.Where(f => new FileInfo(f).Name.Length == 40)
.Select(s => s.ToLowerInvariant()))
.Select(p => new KeyValuePair<string, bool>(p, false)).ToList());
List<FileCache> fileCaches;
await using (var db = new FileCacheContext())
fileCaches = db.FileCaches.ToList();
TotalFiles = scannedFiles.Count;
var fileCachesToDelete = new ConcurrentBag<FileCache>();
var fileCachesToAdd = new ConcurrentBag<FileCache>();
Logger.Debug("Database contains " + fileCaches.Count + " files, local system contains " + TotalFiles);
// scan files from database
Parallel.ForEach(fileCaches, new ParallelOptions()
{
MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
CancellationToken = ct,
},
cache =>
{
if (ct.IsCancellationRequested) return;
if (!File.Exists(cache.Filepath))
{
fileCachesToDelete.Add(cache);
}
else
{
if (scannedFiles.ContainsKey(cache.Filepath))
{
scannedFiles[cache.Filepath] = true;
}
FileInfo fileInfo = new(cache.Filepath);
if (fileInfo.LastWriteTimeUtc.Ticks == long.Parse(cache.LastModifiedDate)) return;
var newCache = Create(cache.Filepath, ct);
if (newCache != null)
{
fileCachesToAdd.Add(newCache);
fileCachesToDelete.Add(cache);
}
}
var files = CurrentFileProgress;
Interlocked.Increment(ref files);
CurrentFileProgress = files;
});
if (ct.IsCancellationRequested) return;
// scan new files
Parallel.ForEach(scannedFiles.Where(c => c.Value == false), new ParallelOptions()
{
MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
CancellationToken = ct
},
file =>
{
var newCache = Create(file.Key, ct);
if (newCache != null)
{
fileCachesToAdd.Add(newCache);
}
var files = CurrentFileProgress;
Interlocked.Increment(ref files);
CurrentFileProgress = files;
});
if (fileCachesToAdd.Any() || fileCachesToDelete.Any())
{
await using FileCacheContext db = new();
Logger.Debug("Found " + fileCachesToAdd.Count + " additions and " + fileCachesToDelete.Count + " deletions");
try
{
foreach (var deletion in fileCachesToDelete)
{
var entries = db.FileCaches.Where(f =>
f.Hash == deletion.Hash && f.Filepath.ToLower() == deletion.Filepath.ToLower());
if (await entries.AnyAsync(ct))
{
Logger.Verbose("Removing file from DB: " + deletion.Filepath);
db.FileCaches.RemoveRange(entries);
}
}
await db.SaveChangesAsync(ct);
foreach (var entry in fileCachesToAdd)
{
try
{
db.FileCaches.Add(entry);
}
catch
{
// ignored
}
}
await db.SaveChangesAsync(ct);
}
catch (Exception ex)
{
PluginLog.Error(ex, ex.Message);
}
}
Logger.Debug("Scan complete");
TotalFiles = 0;
CurrentFileProgress = 0;
if (!_pluginConfiguration.InitialScanComplete)
{
_pluginConfiguration.InitialScanComplete = true;
_pluginConfiguration.Save();
}
}
private void StartWatchersAndScan()
{
if (!_ipcManager.Initialized || !_pluginConfiguration.HasValidSetup()) return;
Logger.Verbose("Penumbra is active, configuration is valid, starting watchers and scan");
StartWatchers();
StartInitialScan();
}
private void StopWatchersAndScan()
{
_cacheDirWatcher?.Dispose();
}
}
}

7
MareSynchronos/Managers/IpcManager.cs
View File

@@ -174,11 +174,16 @@ namespace MareSynchronos.Managers
while (actionQueue.Count > 0 && totalSleepTime < 2000)
{
Logger.Verbose("Waiting for actionqueue to clear...");
HandleActionQueue();
System.Threading.Thread.Sleep(16);
totalSleepTime += 16;
}
if (totalSleepTime >= 2000)
{
Logger.Verbose("Action queue clear or not, disposing");
}
_dalamudUtil.FrameworkUpdate -= HandleActionQueue;
_dalamudUtil.ZoneSwitchEnd -= ClearActionQueue;
actionQueue.Clear();
@@ -304,7 +309,7 @@ namespace MareSynchronos.Managers
public string? PenumbraModDirectory()
{
if (!CheckPenumbraApi()) return null;
return _penumbraResolveModDir!.InvokeFunc();
return _penumbraResolveModDir!.InvokeFunc().ToLowerInvariant();
}
public void PenumbraRedraw(IntPtr obj)

7
MareSynchronos/Managers/OnlinePlayerManager.cs
View File

@@ -17,6 +17,7 @@ public class OnlinePlayerManager : IDisposable
private readonly DalamudUtil _dalamudUtil;
private readonly IpcManager _ipcManager;
private readonly PlayerManager _playerManager;
private readonly FileDbManager _fileDbManager;
private readonly ConcurrentDictionary<string, CachedPlayer> _onlineCachedPlayers = new();
private readonly ConcurrentDictionary<string, CharacterCacheDto> _temporaryStoredCharacterCache = new();
private readonly ConcurrentDictionary<CachedPlayer, CancellationTokenSource> _playerTokenDisposal = new();
@@ -24,7 +25,7 @@ public class OnlinePlayerManager : IDisposable
private List<string> OnlineVisiblePlayerHashes => _onlineCachedPlayers.Select(p => p.Value).Where(p => p.PlayerCharacter != IntPtr.Zero)
.Select(p => p.PlayerNameHash).ToList();
public OnlinePlayerManager(ApiController apiController, DalamudUtil dalamudUtil, IpcManager ipcManager, PlayerManager playerManager)
public OnlinePlayerManager(ApiController apiController, DalamudUtil dalamudUtil, IpcManager ipcManager, PlayerManager playerManager, FileDbManager fileDbManager)
{
Logger.Verbose("Creating " + nameof(OnlinePlayerManager));
@@ -32,7 +33,7 @@ public class OnlinePlayerManager : IDisposable
_dalamudUtil = dalamudUtil;
_ipcManager = ipcManager;
_playerManager = playerManager;
_fileDbManager = fileDbManager;
_apiController.PairedClientOnline += ApiControllerOnPairedClientOnline;
_apiController.PairedClientOffline += ApiControllerOnPairedClientOffline;
_apiController.PairedWithOther += ApiControllerOnPairedWithOther;
@@ -249,6 +250,6 @@ public class OnlinePlayerManager : IDisposable
private CachedPlayer CreateCachedPlayer(string hashedName)
{
return new CachedPlayer(hashedName, _ipcManager, _apiController, _dalamudUtil);
return new CachedPlayer(hashedName, _ipcManager, _apiController, _dalamudUtil, _fileDbManager);
}
}

2
MareSynchronos/Managers/TransientResourceManager.cs
View File

@@ -101,7 +101,7 @@ namespace MareSynchronos.Managers
filePath = filePath.Split("|")[2];
}
filePath = filePath.ToLowerInvariant();
filePath = filePath.ToLowerInvariant().Replace("\\", "/");
var replacedGamePath = gamePath.ToLowerInvariant().Replace("\\", "/");

67
MareSynchronos/Models/FileReplacement.cs
View File

@@ -9,16 +9,17 @@ using System.IO;
using MareSynchronos.API;
using MareSynchronos.Utils;
using System.Text.RegularExpressions;
using MareSynchronos.Managers;
namespace MareSynchronos.Models
{
public class FileReplacement
{
private readonly string _penumbraDirectory;
private readonly FileDbManager fileDbManager;
public FileReplacement(string penumbraDirectory)
public FileReplacement(FileDbManager fileDbManager)
{
_penumbraDirectory = penumbraDirectory;
this.fileDbManager = fileDbManager;
}
public bool Computed => IsFileSwap || !HasFileReplacement || !string.IsNullOrEmpty(Hash);
@@ -35,38 +36,14 @@ namespace MareSynchronos.Models
public void SetResolvedPath(string path)
{
ResolvedPath = path.ToLowerInvariant().Replace('\\', '/');//.Replace('/', '\\').Replace(_penumbraDirectory, "").Replace('\\', '/');
ResolvedPath = path.ToLowerInvariant().Replace('\\', '/');
if (!HasFileReplacement || IsFileSwap) return;
_ = Task.Run(() =>
{
FileCache? fileCache;
using (FileCacheContext db = new())
{
fileCache = db.FileCaches.FirstOrDefault(f => f.Filepath == path.Replace('/', '\\').ToLowerInvariant());
}
if (fileCache != null)
{
FileInfo fi = new(fileCache.Filepath);
if (fi.LastWriteTimeUtc.Ticks == long.Parse(fileCache.LastModifiedDate))
{
Hash = fileCache.Hash;
}
else
{
Hash = ComputeHash(fi);
using var db = new FileCacheContext();
var newTempCache = db.FileCaches.Single(f => f.Filepath == path.Replace('/', '\\').ToLowerInvariant());
newTempCache.Hash = Hash;
db.Update(newTempCache);
db.SaveChanges();
}
}
else
{
Hash = ComputeHash(new FileInfo(path.Replace('/', '\\').ToLowerInvariant()));
}
var cache = fileDbManager.GetFileCacheByPath(ResolvedPath);
cache ??= fileDbManager.CreateFileCacheEntity(ResolvedPath);
Hash = cache.OriginalHash;
});
}
@@ -85,33 +62,5 @@ namespace MareSynchronos.Models
builder.AppendLine($"Modded: {HasFileReplacement} - {string.Join(",", GamePaths)} => {ResolvedPath}");
return builder.ToString();
}
private string ComputeHash(FileInfo fi)
{
// compute hash if hash is not present
string hash = Crypto.GetFileHash(fi.FullName);
using FileCacheContext db = new();
var fileAddedDuringCompute = db.FileCaches.FirstOrDefault(f => f.Filepath == fi.FullName.ToLowerInvariant());
if (fileAddedDuringCompute != null) return fileAddedDuringCompute.Hash;
try
{
Logger.Debug("Adding new file to DB: " + fi.FullName + ", " + hash);
db.Add(new FileCache()
{
Hash = hash,
Filepath = fi.FullName.ToLowerInvariant(),
LastModifiedDate = fi.LastWriteTimeUtc.Ticks.ToString()
});
db.SaveChanges();
}
catch (Exception ex)
{
PluginLog.Error(ex, "Error adding files to database. Most likely not an issue though.");
}
return hash;
}
}
}

22
MareSynchronos/Plugin.cs
View File

@@ -25,7 +25,7 @@ namespace MareSynchronos
private readonly CommandManager _commandManager;
private readonly Framework _framework;
private readonly Configuration _configuration;
private readonly FileCacheManager _fileCacheManager;
private readonly PeriodicFileScanner _fileCacheManager;
private readonly IntroUi _introUi;
private readonly IpcManager _ipcManager;
public static DalamudPluginInterface PluginInterface { get; set; }
@@ -37,6 +37,7 @@ namespace MareSynchronos
private OnlinePlayerManager? _characterCacheManager;
private readonly DownloadUi _downloadUi;
private readonly FileDialogManager _fileDialogManager;
private readonly FileDbManager _fileDbManager;
private readonly CompactUi _compactUi;
private readonly UiShared _uiSharedComponent;
private readonly Dalamud.Localization _localization;
@@ -63,15 +64,16 @@ namespace MareSynchronos
// those can be initialized outside of game login
_dalamudUtil = new DalamudUtil(clientState, objectTable, framework, condition);
_apiController = new ApiController(_configuration, _dalamudUtil);
_ipcManager = new IpcManager(PluginInterface, _dalamudUtil);
// Compatibility for FileSystemWatchers under OSX
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
Environment.SetEnvironmentVariable("MONO_MANAGED_WATCHER", "enabled");
_fileCacheManager = new FileCacheManager(_ipcManager, _configuration);
_fileDialogManager = new FileDialogManager();
_fileDbManager = new FileDbManager(_ipcManager, _configuration);
_apiController = new ApiController(_configuration, _dalamudUtil, _fileDbManager);
_fileCacheManager = new PeriodicFileScanner(_ipcManager, _configuration, _fileDbManager, _apiController);
_uiSharedComponent =
new UiShared(_ipcManager, _apiController, _fileCacheManager, _fileDialogManager, _configuration, _dalamudUtil, PluginInterface, _localization);
@@ -101,7 +103,6 @@ namespace MareSynchronos
_dalamudUtil.LogIn += DalamudUtilOnLogIn;
_dalamudUtil.LogOut += DalamudUtilOnLogOut;
_apiController.RegisterFinalized += ApiControllerOnRegisterFinalized;
if (_dalamudUtil.IsLoggedIn)
{
@@ -109,17 +110,10 @@ namespace MareSynchronos
}
}
private void ApiControllerOnRegisterFinalized()
{
_introUi.IsOpen = false;
_compactUi.IsOpen = true;
}
public string Name => "Mare Synchronos";
public void Dispose()
{
Logger.Verbose("Disposing " + Name);
_apiController.RegisterFinalized -= ApiControllerOnRegisterFinalized;
_apiController?.Dispose();
_commandManager.RemoveHandler(CommandName);
@@ -133,9 +127,9 @@ namespace MareSynchronos
_compactUi?.Dispose();
_fileCacheManager?.Dispose();
_ipcManager?.Dispose();
_playerManager?.Dispose();
_characterCacheManager?.Dispose();
_ipcManager?.Dispose();
_transientResourceManager?.Dispose();
_dalamudUtil.Dispose();
Logger.Debug("Shut down");
@@ -195,11 +189,11 @@ namespace MareSynchronos
{
_transientResourceManager = new TransientResourceManager(_ipcManager, _dalamudUtil);
var characterCacheFactory =
new CharacterDataFactory(_dalamudUtil, _ipcManager, _transientResourceManager);
new CharacterDataFactory(_dalamudUtil, _ipcManager, _transientResourceManager, _fileDbManager);
_playerManager = new PlayerManager(_apiController, _ipcManager,
characterCacheFactory, _dalamudUtil, _transientResourceManager);
_characterCacheManager = new OnlinePlayerManager(_apiController,
_dalamudUtil, _ipcManager, _playerManager);
_dalamudUtil, _ipcManager, _playerManager, _fileDbManager);
}
catch (Exception ex)
{

1
MareSynchronos/UI/CompactUI.cs
View File

@@ -52,6 +52,7 @@ namespace MareSynchronos.UI
Logger.Warn(ex.StackTrace);
}
this.WindowName = "Mare Synchronos " + dateTime + "###MareSynchronosMainUI";
Toggle();
#else
this.WindowName = "Mare Synchronos " + Assembly.GetExecutingAssembly().GetName().Version;
#endif

8
MareSynchronos/UI/IntroUI.cs
View File

@@ -7,10 +7,10 @@ using System.Threading.Tasks;
using Dalamud.Interface.Colors;
using Dalamud.Interface.Windowing;
using ImGuiNET;
using MareSynchronos.Managers;
using MareSynchronos.Utils;
using MareSynchronos.Localization;
using Dalamud.Utility;
using MareSynchronos.FileCacheDB;
namespace MareSynchronos.UI
{
@@ -18,7 +18,7 @@ namespace MareSynchronos.UI
{
private readonly UiShared _uiShared;
private readonly Configuration _pluginConfiguration;
private readonly FileCacheManager _fileCacheManager;
private readonly PeriodicFileScanner _fileCacheManager;
private readonly WindowSystem _windowSystem;
private bool _readFirstPage;
@@ -53,7 +53,7 @@ namespace MareSynchronos.UI
}
public IntroUi(WindowSystem windowSystem, UiShared uiShared, Configuration pluginConfiguration,
FileCacheManager fileCacheManager) : base("Mare Synchronos Setup")
PeriodicFileScanner fileCacheManager) : base("Mare Synchronos Setup")
{
Logger.Verbose("Creating " + nameof(IntroUi));
@@ -226,7 +226,7 @@ namespace MareSynchronos.UI
if (ImGui.Button("Start Scan##startScan"))
{
_fileCacheManager.StartInitialScan();
_fileCacheManager.InvokeScan();
}
}
else

4
MareSynchronos/UI/SettingsUi.cs
View File

@@ -586,6 +586,8 @@ namespace MareSynchronos.UI
private void DrawFileCacheSettings()
{
_uiShared.DrawFileScanState();
_uiShared.DrawParallelScansSetting();
_uiShared.DrawTimeSpanBetweenScansSetting();
_uiShared.DrawCacheDirectorySetting();
ImGui.Text($"Local cache size: {UiShared.ByteToString(_uiShared.FileCacheSize)}");
ImGui.SameLine();
@@ -597,6 +599,8 @@ namespace MareSynchronos.UI
{
File.Delete(file);
}
_uiShared.RecalculateFileCacheSize();
});
}
}

53
MareSynchronos/UI/UIShared.cs
View File

@@ -11,6 +11,7 @@ using Dalamud.Interface.ImGuiFileDialog;
using Dalamud.Plugin;
using Dalamud.Utility;
using ImGuiNET;
using MareSynchronos.FileCacheDB;
using MareSynchronos.Localization;
using MareSynchronos.Managers;
using MareSynchronos.Utils;
@@ -25,13 +26,13 @@ namespace MareSynchronos.UI
private readonly IpcManager _ipcManager;
private readonly ApiController _apiController;
private readonly FileCacheManager _fileCacheManager;
private readonly PeriodicFileScanner _cacheScanner;
private readonly FileDialogManager _fileDialogManager;
private readonly Configuration _pluginConfiguration;
private readonly DalamudUtil _dalamudUtil;
private readonly DalamudPluginInterface _pluginInterface;
private readonly Dalamud.Localization _localization;
public long FileCacheSize => _fileCacheManager.FileCacheSize;
public long FileCacheSize => _cacheScanner.FileCacheSize;
public string PlayerName => _dalamudUtil.PlayerName;
public bool HasValidPenumbraModPath => !(_ipcManager.PenumbraModDirectory() ?? string.Empty).IsNullOrEmpty() && Directory.Exists(_ipcManager.PenumbraModDirectory());
public bool EditTrackerPosition { get; set; }
@@ -42,11 +43,12 @@ namespace MareSynchronos.UI
public ApiController ApiController => _apiController;
public UiShared(IpcManager ipcManager, ApiController apiController, FileCacheManager fileCacheManager, FileDialogManager fileDialogManager, Configuration pluginConfiguration, DalamudUtil dalamudUtil, DalamudPluginInterface pluginInterface, Dalamud.Localization localization)
public UiShared(IpcManager ipcManager, ApiController apiController, PeriodicFileScanner cacheScanner, FileDialogManager fileDialogManager,
Configuration pluginConfiguration, DalamudUtil dalamudUtil, DalamudPluginInterface pluginInterface, Dalamud.Localization localization)
{
_ipcManager = ipcManager;
_apiController = apiController;
_fileCacheManager = fileCacheManager;
_cacheScanner = cacheScanner;
_fileDialogManager = fileDialogManager;
_pluginConfiguration = pluginConfiguration;
_dalamudUtil = dalamudUtil;
@@ -144,19 +146,23 @@ namespace MareSynchronos.UI
public void DrawFileScanState()
{
ImGui.Text("File Scanner Status");
if (_fileCacheManager.IsScanRunning)
ImGui.SameLine();
if (_cacheScanner.IsScanRunning)
{
ImGui.Text("Scan is running");
ImGui.Text("Current Progress:");
ImGui.SameLine();
ImGui.Text(_fileCacheManager.TotalFiles <= 0
ImGui.Text(_cacheScanner.TotalFiles <= 1
? "Collecting files"
: $"Processing {_fileCacheManager.CurrentFileProgress} / {_fileCacheManager.TotalFiles} files");
: $"Processing {_cacheScanner.CurrentFileProgress} / {_cacheScanner.TotalFiles} files");
}
else if (_pluginConfiguration.FileScanPaused)
{
ImGui.Text("File scanner is paused");
}
else
{
ImGui.Text("Watching Penumbra Directory: " + _fileCacheManager.WatchedPenumbraDirectory);
ImGui.Text("Watching Cache Directory: " + _fileCacheManager.WatchedCacheDirectory);
ImGui.Text("Next scan in " + _cacheScanner.TimeUntilNextScan);
}
}
@@ -444,7 +450,7 @@ namespace MareSynchronos.UI
{
_pluginConfiguration.CacheFolder = path;
_pluginConfiguration.Save();
_fileCacheManager.StartWatchers();
_cacheScanner.StartWatchers();
}
});
}
@@ -474,6 +480,7 @@ namespace MareSynchronos.UI
_pluginConfiguration.MaxLocalCacheInGiB = maxCacheSize;
_pluginConfiguration.Save();
}
DrawHelpText("The cache is automatically governed by Mare. It will clear itself automatically once it reaches the set capacity by removing the oldest unused files. You typically do not need to clear it yourself.");
}
private bool _isDirectoryWritable = false;
@@ -503,14 +510,38 @@ namespace MareSynchronos.UI
}
}
public void RecalculateFileCacheSize()
{
_cacheScanner.InvokeScan();
}
public void DrawParallelScansSetting()
{
var parallelScans = _pluginConfiguration.MaxParallelScan;
if (ImGui.SliderInt("Parallel File Scans##parallelism", ref parallelScans, 1, 20))
if (ImGui.SliderInt("File scan parallelism##parallelism", ref parallelScans, 1, 20))
{
_pluginConfiguration.MaxParallelScan = parallelScans;
_pluginConfiguration.Save();
}
DrawHelpText("Decrease to lessen load of file scans. File scans will take longer to execute with less parallel threads.");
}
public void DrawTimeSpanBetweenScansSetting()
{
var timeSpan = _pluginConfiguration.TimeSpanBetweenScansInSeconds;
if (ImGui.SliderInt("Seconds between scans##timespan", ref timeSpan, 20, 60))
{
_pluginConfiguration.TimeSpanBetweenScansInSeconds = timeSpan;
_pluginConfiguration.Save();
}
DrawHelpText("This is the time in seconds between file scans. Increase it to reduce system load. A too high setting can cause issues when manually fumbling about in the cache or Penumbra mods folders.");
var isPaused = _pluginConfiguration.FileScanPaused;
if (ImGui.Checkbox("Pause periodic file scan##filescanpause", ref isPaused))
{
_pluginConfiguration.FileScanPaused = isPaused;
_pluginConfiguration.Save();
}
DrawHelpText("This allows you to stop the periodic scans of your Penumbra and Mare cache directories. This setting will automatically revert itself on restart of Mare.");
}
public void Dispose()

50
MareSynchronos/WebAPI/ApIController.Functions.Files.cs
View File

@@ -5,7 +5,6 @@ using System.Linq;
using System.Net;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using LZ4;
@@ -59,7 +58,13 @@ namespace MareSynchronos.WebAPI
string fileName = Path.GetTempFileName();
ct.Register(wc.CancelAsync);
try
{
await wc.DownloadFileTaskAsync(downloadUri, fileName);
}
catch { }
CurrentDownloads[downloadId].Single(f => f.Hash == hash).Transferred = CurrentDownloads[downloadId].Single(f => f.Hash == hash).Total;
@@ -71,6 +76,7 @@ namespace MareSynchronos.WebAPI
public async Task DownloadFiles(int currentDownloadId, List<FileReplacementDto> fileReplacementDto, CancellationToken ct)
{
DownloadStarted?.Invoke();
Logger.Debug("Downloading files (Download ID " + currentDownloadId + ")");
List<DownloadFileDto> downloadFileInfoFromService = new List<DownloadFileDto>();
@@ -89,23 +95,29 @@ namespace MareSynchronos.WebAPI
}
}
foreach (var file in CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred))
await Parallel.ForEachAsync(CurrentDownloads[currentDownloadId].Where(f => f.CanBeTransferred), new ParallelOptions()
{
MaxDegreeOfParallelism = 5,
CancellationToken = ct
},
async (file, token) =>
{
var hash = file.Hash;
var tempFile = await DownloadFile(currentDownloadId, file.Hash, file.DownloadUri, ct);
if (ct.IsCancellationRequested)
var tempFile = await DownloadFile(currentDownloadId, file.Hash, file.DownloadUri, token);
if (token.IsCancellationRequested)
{
File.Delete(tempFile);
Logger.Debug("Detected cancellation, removing " + currentDownloadId);
DownloadFinished?.Invoke();
CancelDownload(currentDownloadId);
break;
return;
}
var tempFileData = await File.ReadAllBytesAsync(tempFile, ct);
var tempFileData = await File.ReadAllBytesAsync(tempFile, token);
var extractedFile = LZ4Codec.Unwrap(tempFileData);
File.Delete(tempFile);
var filePath = Path.Combine(_pluginConfiguration.CacheFolder, file.Hash);
await File.WriteAllBytesAsync(filePath, extractedFile, ct);
await File.WriteAllBytesAsync(filePath, extractedFile, token);
var fi = new FileInfo(filePath);
Func<DateTime> RandomDayFunc()
{
@@ -118,26 +130,20 @@ namespace MareSynchronos.WebAPI
fi.CreationTime = RandomDayFunc().Invoke();
fi.LastAccessTime = RandomDayFunc().Invoke();
fi.LastWriteTime = RandomDayFunc().Invoke();
}
var allFilesInDb = false;
while (!allFilesInDb && !ct.IsCancellationRequested)
try
{
await using (var db = new FileCacheContext())
_ = _fileDbManager.CreateFileCacheEntity(filePath);
}
catch (Exception ex)
{
var fileCount = CurrentDownloads[currentDownloadId]
.Where(c => c.CanBeTransferred)
.Count(h => db.FileCaches.Any(f => f.Hash == h.Hash));
var totalFiles = CurrentDownloads[currentDownloadId].Count(c => c.CanBeTransferred);
Logger.Debug("Waiting for files to be in the DB, added " + fileCount + " of " + totalFiles);
allFilesInDb = fileCount == totalFiles;
}
await Task.Delay(250, ct);
Logger.Warn("Issue adding file to the DB");
Logger.Warn(ex.Message);
Logger.Warn(ex.StackTrace);
}
});
Logger.Debug("Download complete, removing " + currentDownloadId);
DownloadFinished?.Invoke();
CancelDownload(currentDownloadId);
}

12
MareSynchronos/WebAPI/ApiController.Connectivity.cs
View File

@@ -6,6 +6,7 @@ using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using MareSynchronos.API;
using MareSynchronos.Managers;
using MareSynchronos.Utils;
using MareSynchronos.WebAPI.Utils;
using Microsoft.AspNetCore.Http.Connections;
@@ -35,7 +36,7 @@ namespace MareSynchronos.WebAPI
private readonly Configuration _pluginConfiguration;
private readonly DalamudUtil _dalamudUtil;
private readonly FileDbManager _fileDbManager;
private CancellationTokenSource _connectionCancellationTokenSource;
private HubConnection? _mareHub;
@@ -48,12 +49,13 @@ namespace MareSynchronos.WebAPI
public bool IsAdmin => _connectionDto?.IsAdmin ?? false;
public ApiController(Configuration pluginConfiguration, DalamudUtil dalamudUtil)
public ApiController(Configuration pluginConfiguration, DalamudUtil dalamudUtil, FileDbManager fileDbManager)
{
Logger.Verbose("Creating " + nameof(ApiController));
_pluginConfiguration = pluginConfiguration;
_dalamudUtil = dalamudUtil;
_fileDbManager = fileDbManager;
_connectionCancellationTokenSource = new CancellationTokenSource();
_dalamudUtil.LogIn += DalamudUtilOnLogIn;
_dalamudUtil.LogOut += DalamudUtilOnLogOut;
@@ -80,8 +82,6 @@ namespace MareSynchronos.WebAPI
public event EventHandler<CharacterReceivedEventArgs>? CharacterReceived;
public event VoidDelegate? RegisterFinalized;
public event VoidDelegate? Connected;
public event VoidDelegate? Disconnected;
@@ -93,6 +93,8 @@ namespace MareSynchronos.WebAPI
public event SimpleStringDelegate? PairedWithOther;
public event SimpleStringDelegate? UnpairedFromOther;
public event VoidDelegate? DownloadStarted;
public event VoidDelegate? DownloadFinished;
public ConcurrentDictionary<int, List<DownloadFileTransfer>> CurrentDownloads { get; } = new();
@@ -313,7 +315,7 @@ namespace MareSynchronos.WebAPI
.WithAutomaticReconnect(new ForeverRetryPolicy())
.ConfigureLogging(a => {
a.ClearProviders().AddProvider(new DalamudLoggingProvider());
a.SetMinimumLevel(LogLevel.Trace);
a.SetMinimumLevel(LogLevel.Warning);
})
.Build();
}