t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more refactoring, rework FileCacheManager

Browse Source
This commit is contained in:
Stanley Dimant
2022-06-24 11:22:46 +02:00
parent 2dcd02d170
commit 6ee159f716
11 changed files with 172 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
MareSynchronos/Factories/CharacterDataFactory.cs
View File

@@ -14,35 +14,30 @@ namespace MareSynchronos.Factories
{
public class CharacterDataFactory
{
private readonly ClientState _clientState;
private readonly DalamudUtil _dalamudUtil;
private readonly IpcManager _ipcManager;
private readonly FileReplacementFactory _factory;
public CharacterDataFactory(ClientState clientState, IpcManager ipcManager, FileReplacementFactory factory)
public CharacterDataFactory(DalamudUtil dalamudUtil, IpcManager ipcManager, FileReplacementFactory factory)
{
Logger.Debug("Creating " + nameof(CharacterDataFactory));
_clientState = clientState;
_dalamudUtil = dalamudUtil;
_ipcManager = ipcManager;
_factory = factory;
}
private string GetPlayerName()
{
return _clientState.LocalPlayer!.Name.ToString();
}
public unsafe CharacterData BuildCharacterData()
{
Stopwatch st = Stopwatch.StartNew();
var cache = new CharacterData();
while (_clientState.LocalPlayer == null)
while (!_dalamudUtil.IsPlayerPresent)
{
Logger.Debug("Character is null but it shouldn't be, waiting");
Thread.Sleep(50);
}
var model = (CharacterBase*)((Character*)_clientState.LocalPlayer!.Address)->GameObject.GetDrawObject();
var model = (CharacterBase*)((Character*)_dalamudUtil.PlayerPointer)->GameObject.GetDrawObject();
for (var idx = 0; idx < model->SlotCount; ++idx)
{
var mdl = (RenderModel*)model->ModelArray[idx];
@@ -54,7 +49,7 @@ namespace MareSynchronos.Factories
var mdlPath = new Utf8String(mdl->ResourceHandle->FileName()).ToString();
FileReplacement cachedMdlResource = _factory.Create();
cachedMdlResource.GamePaths = _ipcManager.PenumbraReverseResolvePath(mdlPath, GetPlayerName());
cachedMdlResource.GamePaths = _ipcManager.PenumbraReverseResolvePath(mdlPath, _dalamudUtil.PlayerName);
cachedMdlResource.SetResolvedPath(mdlPath);
//PluginLog.Verbose("Resolving for model " + mdlPath);
@@ -69,7 +64,7 @@ namespace MareSynchronos.Factories
var mtrlPath = new Utf8String(mtrl->ResourceHandle->FileName()).ToString().Split("|")[2];
//PluginLog.Verbose("Resolving for material " + mtrlPath);
var cachedMtrlResource = _factory.Create();
cachedMtrlResource.GamePaths = _ipcManager.PenumbraReverseResolvePath(mtrlPath, GetPlayerName());
cachedMtrlResource.GamePaths = _ipcManager.PenumbraReverseResolvePath(mtrlPath, _dalamudUtil.PlayerName);
cachedMtrlResource.SetResolvedPath(mtrlPath);
cache.AddAssociatedResource(cachedMtrlResource, cachedMdlResource, null!);
@@ -82,12 +77,12 @@ namespace MareSynchronos.Factories
var cachedTexResource = _factory.Create();
cachedTexResource.GamePaths = new[] { texPath };
cachedTexResource.SetResolvedPath(_ipcManager.PenumbraResolvePath(texPath, GetPlayerName())!);
cachedTexResource.SetResolvedPath(_ipcManager.PenumbraResolvePath(texPath, _dalamudUtil.PlayerName)!);
if (!cachedTexResource.HasFileReplacement)
{
// try resolving tex with -- in name instead
texPath = texPath.Insert(texPath.LastIndexOf('/') + 1, "--");
var reResolvedPath = _ipcManager.PenumbraResolvePath(texPath, GetPlayerName())!;
var reResolvedPath = _ipcManager.PenumbraResolvePath(texPath, _dalamudUtil.PlayerName)!;
if (reResolvedPath != texPath)
{
cachedTexResource.GamePaths = new[] { texPath };
@@ -100,8 +95,8 @@ namespace MareSynchronos.Factories
}
cache.GlamourerString = _ipcManager.GlamourerGetCharacterCustomization()!;
cache.ManipulationString = _ipcManager.PenumbraGetMetaManipulations(_clientState.LocalPlayer!.Name.ToString());
cache.JobId = _clientState.LocalPlayer!.ClassJob.Id;
cache.ManipulationString = _ipcManager.PenumbraGetMetaManipulations(_dalamudUtil.PlayerName);
cache.JobId = _dalamudUtil.PlayerJobId;
st.Stop();
Logger.Debug("Building Character Data took " + st.Elapsed);

20
MareSynchronos/Factories/FileCacheFactory.cs
View File

@@ -1,4 +1,5 @@
using System.IO;
using System.Threading;
using MareSynchronos.FileCacheDB;
using MareSynchronos.Utils;
@@ -9,6 +10,11 @@ namespace MareSynchronos.Factories
public FileCache Create(string file)
{
FileInfo fileInfo = new(file);
while (IsFileLocked(fileInfo))
{
Thread.Sleep(100);
Logger.Debug("File is locked, waiting for release: " + fileInfo.FullName);
}
var sha1Hash = Crypto.GetFileHash(fileInfo.FullName);
return new FileCache()
{
@@ -24,5 +30,19 @@ namespace MareSynchronos.Factories
cache.Hash = Crypto.GetFileHash(cache.Filepath);
cache.LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString();
}
private bool IsFileLocked(FileInfo file)
{
try
{
using var fs = file.Open(FileMode.Open, FileAccess.ReadWrite, FileShare.None);
}
catch
{
return true;
}
return false;
}
}
}

4
MareSynchronos/Managers/CharacterManager.cs
View File

@@ -75,7 +75,7 @@ namespace MareSynchronos.Managers
Task.WaitAll(apiTask);
_characterCacheManager.AddInitialPairs(apiTask.Result);
_ipcManager.PenumbraRedrawEvent += IpcManager_PenumbraRedrawEvent;
}
@@ -142,7 +142,7 @@ namespace MareSynchronos.Managers
Logger.Debug("Not sending data, already sent");
return;
}
await _apiController.SendCharacterData(cacheDto, _dalamudUtil.GetLocalPlayers().Select(d => d.Key).ToList());
_ = _apiController.SendCharacterData(cacheDto, _dalamudUtil.GetLocalPlayers().Select(d => d.Key).ToList());
_lastSentHash = cacheDto.Hash;
});
}

167
MareSynchronos/Managers/FileCacheManager.cs
View File

@@ -1,7 +1,6 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
@@ -15,14 +14,14 @@ namespace MareSynchronos.Managers
{
internal class FileCacheManager : IDisposable
{
private const int MinutesForScan = 10;
private readonly FileCacheFactory _fileCacheFactory;
private readonly IpcManager _ipcManager;
private readonly Configuration _pluginConfiguration;
private CancellationTokenSource? _scanCancellationTokenSource;
private System.Timers.Timer? _scanScheduler;
private Task? _scanTask;
private Stopwatch? _timerStopWatch;
private FileSystemWatcher? _penumbraDirWatcher;
private FileSystemWatcher? _cacheDirWatcher;
public FileCacheManager(FileCacheFactory fileCacheFactory, IpcManager ipcManager, Configuration pluginConfiguration)
{
Logger.Debug("Creating " + nameof(FileCacheManager));
@@ -41,17 +40,82 @@ namespace MareSynchronos.Managers
}
}
private void OnCreated(object sender, FileSystemEventArgs e)
{
var fi = new FileInfo(e.FullPath);
using var db = new FileCacheContext();
if (fi.Extension.ToLower() is not ".mdl" or ".tex" or ".mtrl")
{
// this is most likely a folder
Logger.Debug("Folder added: " + e.FullPath);
var newFiles = Directory.EnumerateFiles(e.FullPath, "*.*", SearchOption.AllDirectories)
.Where(f => f.EndsWith(".tex", StringComparison.OrdinalIgnoreCase) ||
f.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase) ||
f.EndsWith(".mtrl", StringComparison.OrdinalIgnoreCase)).ToList();
foreach (var file in newFiles)
{
Logger.Debug("Adding " + file);
db.Add(_fileCacheFactory.Create(file));
}
}
else
{
Logger.Debug("File created: " + e.FullPath);
var createdFileCache = _fileCacheFactory.Create(fi.FullName);
db.Add(createdFileCache);
}
db.SaveChanges();
}
private void OnDeleted(object sender, FileSystemEventArgs e)
{
var fi = new FileInfo(e.FullPath);
using var db = new FileCacheContext();
if (fi.Extension.ToLower() is not ".mdl" or ".tex" or ".mtrl")
{
// this is most likely a folder
var filesToRemove = db.FileCaches.Where(f => f.Filepath.StartsWith(e.FullPath.ToLower())).ToList();
Logger.Debug($"Folder deleted: {e.FullPath}, removing {filesToRemove.Count} files");
db.RemoveRange(filesToRemove);
}
else
{
Logger.Debug("File deleted: " + e.FullPath);
var fileInDb = db.FileCaches.SingleOrDefault(f => f.Filepath == fi.FullName.ToLower());
if (fileInDb == null) return;
db.Remove(fileInDb);
}
db.SaveChanges();
}
private void OnModified(object sender, FileSystemEventArgs e)
{
Logger.Debug("OnModified: " + e.FullPath);
var fi = new FileInfo(e.FullPath);
if (fi.Extension.ToLower() is not ".mdl" or ".tex" or ".mtrl") return;
Logger.Debug("File changed: " + e.FullPath);
using var db = new FileCacheContext();
var modifiedFile = _fileCacheFactory.Create(fi.FullName);
var fileInDb = db.FileCaches.SingleOrDefault(f => f.Filepath == fi.FullName.ToLower() || modifiedFile.Hash == f.Hash);
if (fileInDb == null) return;
db.Remove(fileInDb);
db.Add(modifiedFile);
db.SaveChanges();
}
public long CurrentFileProgress { get; private set; }
public bool IsScanRunning => !_scanTask?.IsCompleted ?? false;
public TimeSpan TimeToNextScan => TimeSpan.FromMinutes(MinutesForScan).Subtract(_timerStopWatch?.Elapsed ?? TimeSpan.FromMinutes(MinutesForScan));
public long TotalFiles { get; private set; }
public string WatchedPenumbraDirectory => (!_penumbraDirWatcher?.EnableRaisingEvents ?? false) ? "Not watched" : _penumbraDirWatcher!.Path;
public string WatchedCacheDirectory => (!_cacheDirWatcher?.EnableRaisingEvents ?? false) ? "Not watched" : _cacheDirWatcher!.Path;
public void Dispose()
{
Logger.Debug("Disposing " + nameof(FileCacheManager));
_scanScheduler?.Stop();
_cacheDirWatcher?.Dispose();
_penumbraDirWatcher?.Dispose();
_scanCancellationTokenSource?.Cancel();
}
@@ -65,21 +129,20 @@ namespace MareSynchronos.Managers
{
_scanCancellationTokenSource = new CancellationTokenSource();
var penumbraDir = _ipcManager.PenumbraModDirectory()!;
Logger.Debug("Getting files from " + penumbraDir);
Logger.Debug("Getting files from " + penumbraDir + " and " + _pluginConfiguration.CacheFolder);
var scannedFiles = new ConcurrentDictionary<string, bool>(
Directory.EnumerateFiles(penumbraDir, "*.*", SearchOption.AllDirectories)
.Select(s => s.ToLowerInvariant())
.Where(f => f.Contains(@"\chara\") && (f.EndsWith(".tex") || f.EndsWith(".mdl") || f.EndsWith(".mtrl")))
.Where(f => f.Contains(@"\chara\"))
.Concat(Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.AllDirectories)
.Select(s => s.ToLowerInvariant()))
.Where(f => (f.EndsWith(".tex", StringComparison.OrdinalIgnoreCase) || f.EndsWith(".mdl", StringComparison.OrdinalIgnoreCase) || f.EndsWith(".mtrl", StringComparison.OrdinalIgnoreCase)))
.Select(p => new KeyValuePair<string, bool>(p, false)));
List<FileCache> fileCaches;
await using (FileCacheContext db = new())
{
fileCaches = db.FileCaches.ToList();
}
await using FileCacheContext db = new();
List<FileCache> fileCaches = db.FileCaches.ToList();
TotalFiles = scannedFiles.Count;
var fileCachesToUpdate = new ConcurrentBag<FileCache>();
var fileCachesToDelete = new ConcurrentBag<FileCache>();
var fileCachesToAdd = new ConcurrentBag<FileCache>();
@@ -105,8 +168,8 @@ namespace MareSynchronos.Managers
}
FileInfo fileInfo = new(cache.Filepath);
if (fileInfo.LastWriteTimeUtc.Ticks == long.Parse(cache.LastModifiedDate)) return;
_fileCacheFactory.UpdateFileCache(cache);
fileCachesToUpdate.Add(cache);
fileCachesToAdd.Add(_fileCacheFactory.Create(cache.Filepath));
fileCachesToDelete.Add(cache);
}
var files = CurrentFileProgress;
@@ -131,15 +194,18 @@ namespace MareSynchronos.Managers
CurrentFileProgress = files;
});
await using (FileCacheContext db = new())
if (fileCachesToAdd.Any() || fileCachesToDelete.Any())
{
if (fileCachesToAdd.Any() || fileCachesToUpdate.Any() || fileCachesToDelete.Any())
Logger.Debug("Found " + fileCachesToAdd.Count + " additions and " + fileCachesToDelete.Count + " deletions");
try
{
db.FileCaches.AddRange(fileCachesToAdd);
db.FileCaches.UpdateRange(fileCachesToUpdate);
db.FileCaches.RemoveRange(fileCachesToDelete);
await db.SaveChangesAsync(ct);
db.FileCaches.AddRange(fileCachesToAdd);
db.SaveChanges();
}
catch (Exception ex)
{
PluginLog.Error(ex, ex.Message);
}
}
@@ -151,40 +217,41 @@ namespace MareSynchronos.Managers
{
_pluginConfiguration.InitialScanComplete = true;
_pluginConfiguration.Save();
_timerStopWatch = Stopwatch.StartNew();
StartScheduler();
}
else if (_timerStopWatch == null)
{
StartScheduler();
_timerStopWatch = Stopwatch.StartNew();
}
StartWatchers();
}
private void StartScheduler()
public void StartWatchers()
{
Logger.Debug("Scheduling next scan for in " + MinutesForScan + " minutes");
_scanScheduler = new System.Timers.Timer(TimeSpan.FromMinutes(MinutesForScan).TotalMilliseconds)
{
AutoReset = false,
Enabled = false,
};
_scanScheduler.AutoReset = true;
_scanScheduler.Elapsed += (_, _) =>
{
_timerStopWatch?.Stop();
if (_scanTask?.IsCompleted ?? false)
{
PluginLog.Warning("Scanning task is still running, not re-initiating.");
return;
}
Logger.Debug("Starting File System Watchers");
_penumbraDirWatcher?.Dispose();
_cacheDirWatcher?.Dispose();
Logger.Debug("Initiating periodic scan for mod changes");
Task.Run(() => _scanTask = StartFileScan(_scanCancellationTokenSource!.Token));
_timerStopWatch = Stopwatch.StartNew();
_penumbraDirWatcher = new FileSystemWatcher(_ipcManager.PenumbraModDirectory()!)
{
EnableRaisingEvents = true,
Filter = "*.*",
IncludeSubdirectories = true,
};
_penumbraDirWatcher.NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.Size;
_penumbraDirWatcher.Created += OnCreated;
_penumbraDirWatcher.Deleted += OnDeleted;
_penumbraDirWatcher.Changed += OnModified;
_penumbraDirWatcher.Error += (sender, args) => PluginLog.Error(args.GetException(), "Error in Penumbra Dir Watcher");
_scanScheduler.Start();
_cacheDirWatcher = new FileSystemWatcher(_pluginConfiguration.CacheFolder)
{
EnableRaisingEvents = true,
Filter = "*.*",
IncludeSubdirectories = true,
};
_cacheDirWatcher.NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.Size;
_cacheDirWatcher.Created += OnCreated;
_cacheDirWatcher.Deleted += OnDeleted;
_cacheDirWatcher.Changed += OnModified;
_cacheDirWatcher.Error +=
(sender, args) => PluginLog.Error(args.GetException(), "Error in Cache Dir Watcher");
}
}
}

2
MareSynchronos/Models/CharacterData.cs
View File

@@ -30,7 +30,7 @@ namespace MareSynchronos.Models
public bool IsReady => FileReplacements.All(f => f.Computed);
[JsonProperty]
public uint JobId { get; set; } = 0;
public int JobId { get; set; } = 0;
public string ManipulationString { get; set; } = string.Empty;

12
MareSynchronos/Models/FileReplacement.cs
View File

@@ -15,18 +15,18 @@ namespace MareSynchronos.Models
[JsonObject(MemberSerialization.OptIn)]
public class FileReplacement
{
private readonly string penumbraDirectory;
private readonly string _penumbraDirectory;
private Task? computationTask = null;
private Task? _computationTask = null;
public FileReplacement(string penumbraDirectory)
{
this.penumbraDirectory = penumbraDirectory;
this._penumbraDirectory = penumbraDirectory;
}
public List<FileReplacement> Associated { get; set; } = new List<FileReplacement>();
public bool Computed => (computationTask == null || (computationTask?.IsCompleted ?? true)) && Associated.All(f => f.Computed);
public bool Computed => (_computationTask == null || (_computationTask?.IsCompleted ?? true)) && Associated.All(f => f.Computed);
[JsonProperty]
public string[] GamePaths { get; set; } = Array.Empty<string>();
@@ -74,10 +74,10 @@ namespace MareSynchronos.Models
public void SetResolvedPath(string path)
{
ResolvedPath = path.ToLower().Replace('/', '\\').Replace(penumbraDirectory, "").Replace('\\', '/');
ResolvedPath = path.ToLower().Replace('/', '\\').Replace(_penumbraDirectory, "").Replace('\\', '/');
if (!HasFileReplacement) return;
computationTask = Task.Run(() =>
_computationTask = Task.Run(() =>
{
FileCache? fileCache;
using (FileCacheContext db = new())

2
MareSynchronos/Plugin.cs
View File

@@ -145,7 +145,7 @@ namespace MareSynchronos
try
{
var characterCacheFactory =
new CharacterDataFactory(_clientState, _ipcManager, new FileReplacementFactory(_ipcManager));
new CharacterDataFactory(_dalamudUtil, _ipcManager, new FileReplacementFactory(_ipcManager));
_characterManager = new CharacterManager(_apiController, _objectTable, _ipcManager,
characterCacheFactory, _characterCacheManager, _dalamudUtil, _playerWatcher);
_characterManager.StartWatchingPlayer();

3
MareSynchronos/UI/PluginUI.cs
View File

@@ -242,8 +242,9 @@ namespace MareSynchronos.UI
{
if (_apiController.PairedClients.All(w => w.OtherUID != tempNameUID))
{
var nameToSend = tempNameUID;
tempNameUID = string.Empty;
_ = _apiController.SendPairedClientAddition(tempNameUID);
_ = _apiController.SendPairedClientAddition(nameToSend);
}
}
ImGui.PopFont();

11
MareSynchronos/UI/UIShared.cs
View File

@@ -1,4 +1,5 @@
using System.IO;
using System;
using System.IO;
using System.Numerics;
using System.Threading.Tasks;
using Dalamud.Interface.Colors;
@@ -59,13 +60,10 @@ namespace MareSynchronos.UI
? "Collecting files"
: $"Processing {_fileCacheManager.CurrentFileProgress} / {_fileCacheManager.TotalFiles} files");
}
else if (_fileCacheManager.TimeToNextScan.TotalSeconds == 0)
{
ImGui.Text("Scan not started");
}
else
{
ImGui.Text("Next scan in " + _fileCacheManager.TimeToNextScan.ToString(@"mm\:ss") + " minutes");
ImGui.Text("Watching Penumbra Directory: " + _fileCacheManager.WatchedPenumbraDirectory);
ImGui.Text("Watching Cache Directory: " + _fileCacheManager.WatchedCacheDirectory);
}
}
@@ -156,6 +154,7 @@ namespace MareSynchronos.UI
if (!string.IsNullOrEmpty(_pluginConfiguration.CacheFolder) && Directory.Exists(_pluginConfiguration.CacheFolder))
{
_pluginConfiguration.Save();
_fileCacheManager.StartWatchers();
}
}

5
MareSynchronos/Utils/DalamudUtil.cs
View File

@@ -23,6 +23,10 @@ namespace MareSynchronos.Utils
public string PlayerName => _clientState.LocalPlayer!.Name.ToString();
public int PlayerJobId => (int)_clientState.LocalPlayer!.ClassJob.Id;
public IntPtr PlayerPointer => _clientState.LocalPlayer!.Address;
public string PlayerNameHashed => Crypto.GetHash256(PlayerName + _clientState.LocalPlayer!.HomeWorld.Id);
public Dictionary<string, PlayerCharacter> GetLocalPlayers()
@@ -44,6 +48,7 @@ namespace MareSynchronos.Utils
{
var obj = (GameObject*)characterAddress;
// ReSharper disable once LoopVariableIsNeverChangedInsideLoop
while ((obj->RenderFlags & 0b100000000000) == 0b100000000000) // 0b100000000000 is "still rendering" or something
{
Logger.Debug("Waiting for character to finish drawing");

14
MareSynchronos/WebAPI/ApiController.cs
View File

@@ -87,7 +87,7 @@ namespace MareSynchronos.WebAPI
public void Dispose()
{
Logger.Debug("Disposing " + nameof(ApiController));
_cts?.Cancel();
_ = DisposeHubConnections();
}
@@ -124,19 +124,9 @@ namespace MareSynchronos.WebAPI
var hash = file.Hash;
var data = await DownloadFile(hash);
var extractedFile = LZ4Codec.Unwrap(data);
var ext = file.GamePaths.First().Split(".", StringSplitOptions.None).Last();
var ext = file.GamePaths.First().Split(".").Last();
var filePath = Path.Combine(_pluginConfiguration.CacheFolder, file.Hash + "." + ext);
await File.WriteAllBytesAsync(filePath, extractedFile);
await using (var db = new FileCacheContext())
{
db.Add(new FileCache
{
Filepath = filePath.ToLower(),
Hash = file.Hash,
LastModifiedDate = DateTime.Now.Ticks.ToString(),
});
await db.SaveChangesAsync();
}
Logger.Debug("File downloaded to " + filePath);
}