From 3b9c4856275f41f76edda24186be6b95d8a542ca Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Fri, 10 Jun 2022 01:41:47 +0200 Subject: [PATCH] add initial file scan and rescanning algorithms to hash all penumbra files --- MareSynchronos/Configuration.cs | 2 +- MareSynchronos/Factories/FileCacheFactory.cs | 41 ++++++ MareSynchronos/FileCacheDB/FileCache.cs | 14 +++ .../FileCacheDB/FileCacheContext.cs | 47 +++++++ MareSynchronos/MareSynchronos.csproj | 10 ++ MareSynchronos/Plugin.cs | 118 +++++++++++++++++- MareSynchronos/PluginUI.cs | 14 +-- MareSynchronos/filecache.db | Bin 12288 -> 20480 bytes 8 files changed, 231 insertions(+), 15 deletions(-) create mode 100644 MareSynchronos/Factories/FileCacheFactory.cs create mode 100644 MareSynchronos/FileCacheDB/FileCache.cs create mode 100644 MareSynchronos/FileCacheDB/FileCacheContext.cs diff --git a/MareSynchronos/Configuration.cs b/MareSynchronos/Configuration.cs index f71a1b9..a76485e 100644 --- a/MareSynchronos/Configuration.cs +++ b/MareSynchronos/Configuration.cs @@ -9,7 +9,7 @@ namespace SamplePlugin { public int Version { get; set; } = 0; - public bool SomePropertyToBeSavedAndWithADefault { get; set; } = true; + public string PenumbraFolder { get; set; } = string.Empty; // the below exist just to make saving less cumbersome diff --git a/MareSynchronos/Factories/FileCacheFactory.cs b/MareSynchronos/Factories/FileCacheFactory.cs new file mode 100644 index 0000000..2e9820f --- /dev/null +++ b/MareSynchronos/Factories/FileCacheFactory.cs @@ -0,0 +1,41 @@ +using System; +using System.IO; +using MareSynchronos.FileCacheDB; +using System.Security.Cryptography; + + +namespace MareSynchronos.Factories +{ + public class FileCacheFactory + { + public FileCacheFactory() + { + + } + + public FileCache Create(string file) + { + FileInfo fileInfo = new(file); + string sha1Hash = GetHash(fileInfo.FullName); + return new FileCache() + { + Filepath = fileInfo.FullName, + Hash = sha1Hash, + LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString(), + }; + } + + public void UpdateFileCache(FileCache cache) + { + FileInfo fileInfo = new(cache.Filepath); + cache.Hash = GetHash(cache.Filepath); + cache.LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString(); + } + + private string GetHash(string filePath) + { + using SHA1CryptoServiceProvider cryptoProvider = new(); + return BitConverter.ToString(cryptoProvider.ComputeHash(File.ReadAllBytes(filePath))); + } + } +} diff --git a/MareSynchronos/FileCacheDB/FileCache.cs b/MareSynchronos/FileCacheDB/FileCache.cs new file mode 100644 index 0000000..0ce783e --- /dev/null +++ b/MareSynchronos/FileCacheDB/FileCache.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; + +#nullable disable + +namespace MareSynchronos.FileCacheDB +{ + public partial class FileCache + { + public string Hash { get; set; } + public string Filepath { get; set; } + public string LastModifiedDate { get; set; } + } +} diff --git a/MareSynchronos/FileCacheDB/FileCacheContext.cs b/MareSynchronos/FileCacheDB/FileCacheContext.cs new file mode 100644 index 0000000..7813da8 --- /dev/null +++ b/MareSynchronos/FileCacheDB/FileCacheContext.cs @@ -0,0 +1,47 @@ +using System; +using System.IO; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; + +#nullable disable + +namespace MareSynchronos.FileCacheDB +{ + public partial class FileCacheContext : DbContext + { + public FileCacheContext() + { + Database.EnsureCreated(); + } + + public FileCacheContext(DbContextOptions options) + : base(options) + { + } + + public virtual DbSet FileCaches { get; set; } + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + { + if (!optionsBuilder.IsConfigured) + { + string dbPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "XIVLauncher", "pluginConfigs", "FileCacheDebug.db"); + optionsBuilder.UseSqlite("Data Source=" + dbPath); + } + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.HasKey(e => new { e.Hash, e.Filepath }); + + entity.ToTable("FileCache"); + }); + + OnModelCreatingPartial(modelBuilder); + } + + partial void OnModelCreatingPartial(ModelBuilder modelBuilder); + } +} diff --git a/MareSynchronos/MareSynchronos.csproj b/MareSynchronos/MareSynchronos.csproj index bf057d7..675cea6 100644 --- a/MareSynchronos/MareSynchronos.csproj +++ b/MareSynchronos/MareSynchronos.csproj @@ -26,6 +26,10 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + @@ -61,4 +65,10 @@ + + + Always + + + diff --git a/MareSynchronos/Plugin.cs b/MareSynchronos/Plugin.cs index 4383b7a..d7366a6 100644 --- a/MareSynchronos/Plugin.cs +++ b/MareSynchronos/Plugin.cs @@ -1,9 +1,16 @@ using Dalamud.Game.Command; using Dalamud.IoC; +using Dalamud.Logging; using Dalamud.Plugin; +using MareSynchronos.FileCacheDB; +using MareSynchronos.Factories; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; using System.IO; -using System.Reflection; -//using System.Data.SQLite; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; namespace SamplePlugin { @@ -17,6 +24,9 @@ namespace SamplePlugin private CommandManager CommandManager { get; init; } private Configuration Configuration { get; init; } private PluginUI PluginUi { get; init; } + private FileCacheFactory FileCacheFactory { get; init; } + + private CancellationTokenSource cts; public Plugin( [RequiredVersion("1.0")] DalamudPluginInterface pluginInterface, @@ -33,9 +43,11 @@ namespace SamplePlugin this.CommandManager.AddHandler(commandName, new CommandInfo(OnCommand) { - HelpMessage = "A useful message to display in /xlhelp" + HelpMessage = "pass 'scan' to initialize or rescan files into the database" }); + FileCacheFactory = new FileCacheFactory(); + this.PluginInterface.UiBuilder.Draw += DrawUI; this.PluginInterface.UiBuilder.OpenConfigUi += DrawConfigUI; } @@ -48,8 +60,104 @@ namespace SamplePlugin private void OnCommand(string command, string args) { - //using var connection = new SQLiteConnection("Data Source=penumbracache.db"); - //connection.Open(); + if (args == "stop") + { + cts?.Cancel(); + return; + } + + if (args == "scan") + { + cts = new CancellationTokenSource(); + + Task.Run(() => StartScan(), cts.Token); + } + } + + private void StartScan() + { + Stopwatch st = Stopwatch.StartNew(); + + string penumbraDir = Configuration.PenumbraFolder; + PluginLog.Debug("Getting files from " + penumbraDir); + ConcurrentDictionary charaFiles = new ConcurrentDictionary( + Directory.GetFiles(penumbraDir, "*.*", SearchOption.AllDirectories) + .Select(s => s.ToLowerInvariant()) + .Where(f => !f.EndsWith(".json")) + .Where(f => f.Contains(@"\chara\")) + .Select(p => new KeyValuePair(p, false))); + int count = 0; + using FileCacheContext db = new(); + var fileCaches = db.FileCaches.ToList(); + + var fileCachesToUpdate = new ConcurrentBag(); + var fileCachesToDelete = new ConcurrentBag(); + var fileCachesToAdd = new ConcurrentBag(); + + // scan files from database + Parallel.ForEach(fileCaches, new ParallelOptions() + { + CancellationToken = cts.Token, + MaxDegreeOfParallelism = 10 + }, + cache => + { + count = Interlocked.Increment(ref count); + PluginLog.Debug($"[{count}/{fileCaches.Count}] Checking: {cache.Filepath}"); + + if (!File.Exists(cache.Filepath)) + { + PluginLog.Debug("File was not found anymore: " + cache.Filepath); + fileCachesToDelete.Add(cache); + } + else + { + charaFiles[cache.Filepath] = true; + + FileInfo fileInfo = new(cache.Filepath); + if (fileInfo.LastWriteTimeUtc.Ticks != long.Parse(cache.LastModifiedDate)) + { + PluginLog.Debug("File was modified since last time: " + cache.Filepath + "; " + cache.LastModifiedDate + " / " + fileInfo.LastWriteTimeUtc.Ticks); + FileCacheFactory.UpdateFileCache(cache); + fileCachesToUpdate.Add(cache); + } + } + }); + + // scan new files + count = 0; + Parallel.ForEach(charaFiles.Where(c => c.Value == false), new ParallelOptions() + { + CancellationToken = cts.Token, + MaxDegreeOfParallelism = 10 + }, + file => + { + count = Interlocked.Increment(ref count); + PluginLog.Debug($"[{count}/{charaFiles.Count()}] Hashing: {file.Key}"); + + fileCachesToAdd.Add(FileCacheFactory.Create(file.Key)); + }); + + st.Stop(); + + if (cts.Token.IsCancellationRequested) return; + + PluginLog.Debug("Scanning complete, total elapsed time: " + st.Elapsed.ToString()); + + if (fileCachesToAdd.Any() || fileCachesToUpdate.Any() || fileCachesToDelete.Any()) + { + PluginLog.Debug("Writing files to database…"); + + db.FileCaches.AddRange(fileCachesToAdd); + db.FileCaches.UpdateRange(fileCachesToUpdate); + db.FileCaches.RemoveRange(fileCachesToDelete); + + db.SaveChanges(); + PluginLog.Debug("Database has been written."); + } + + cts = new CancellationTokenSource(); } private void DrawUI() diff --git a/MareSynchronos/PluginUI.cs b/MareSynchronos/PluginUI.cs index a0b3baa..1e3ca93 100644 --- a/MareSynchronos/PluginUI.cs +++ b/MareSynchronos/PluginUI.cs @@ -59,8 +59,6 @@ namespace SamplePlugin ImGui.SetNextWindowSizeConstraints(new Vector2(375, 330), new Vector2(float.MaxValue, float.MaxValue)); if (ImGui.Begin("My Amazing Window", ref this.visible, ImGuiWindowFlags.NoScrollbar | ImGuiWindowFlags.NoScrollWithMouse)) { - ImGui.Text($"The random config bool is {this.configuration.SomePropertyToBeSavedAndWithADefault}"); - if (ImGui.Button("Show Settings")) { SettingsVisible = true; @@ -78,16 +76,14 @@ namespace SamplePlugin return; } - ImGui.SetNextWindowSize(new Vector2(232, 75), ImGuiCond.Always); - if (ImGui.Begin("A Wonderful Configuration Window", ref this.settingsVisible, + ImGui.SetNextWindowSize(new Vector2(500, 75), ImGuiCond.Always); + if (ImGui.Begin("QUALITY UI DEVELOPMENT", ref this.settingsVisible, ImGuiWindowFlags.NoResize | ImGuiWindowFlags.NoCollapse | ImGuiWindowFlags.NoScrollbar | ImGuiWindowFlags.NoScrollWithMouse)) { // can't ref a property, so use a local copy - var configValue = this.configuration.SomePropertyToBeSavedAndWithADefault; - if (ImGui.Checkbox("Random Config Bool", ref configValue)) - { - this.configuration.SomePropertyToBeSavedAndWithADefault = configValue; - // can save immediately on change, if you don't want to provide a "Save and Close" button + string penumbraFolder = configuration.PenumbraFolder; + if(ImGui.InputText("Penumbra mod folder", ref penumbraFolder, 255)) { + this.configuration.PenumbraFolder = penumbraFolder; this.configuration.Save(); } } diff --git a/MareSynchronos/filecache.db b/MareSynchronos/filecache.db index 90c8852490ff6b5da1d9527c8e4e26166376fcf5..aae39a7d29e62f11419faced6f8ae939f06f2df5 100644 GIT binary patch delta 375 zcmZojXjs5FL0XWVfq{V)ikW~k=R_T2QFaDB|5{%D9}GUiLL9UJ=t_mTJPCl**N+2O61r07vCAZAfoD@SPPEWrO zS9jMS9hi_2vQR*fr>|pBq=L6=qy|if4uoN(q{*cTv{Tbsl#wAbFU2jjqPQ?8vm`Y> zv9u%~D3Dqa53()Z5Ja*9Rc;hlRb_0F2Rg4&osnH!UY@Z@7VK8H%$!u`#N>=rAY-yV zzbG?^_Tmrm2RaJH1s;jT8A>4gb5aWuOEN$%Fl1sAca;ab0E+|QCdV5BnVUcI^D8V8 Y5SSRi&dUf4P!=F&24beof&y>&0e&cF9RL6T delta 184 zcmZozz}S#5L0XWRfq?;pVSss}jxkVB&%c(J{|5sTKOX}>AOHT%f&%mSY#Q|#*~L{= z8Jpxw5|eULLsBbBN{dpR6O%Jg*i6nru8twD3L%b8KCTK%5M`77_(Rkc{QN@{{6c+v zbhtPJf;@d4gCZ5YT_ZJ=JQ9mDlr*_C&6(K5y;Uc_=93aam>q8jF=F#aetrc2v5PSd