add initial file scan and rescanning algorithms to hash all penumbra files

MareSynchronos/Configuration.cs

@@ -9,7 +9,7 @@ namespace SamplePlugin
     {
         public int Version { get; set; } = 0;
 
-        public bool SomePropertyToBeSavedAndWithADefault { get; set; } = true;
+        public string PenumbraFolder { get; set; } = string.Empty;
 
         // the below exist just to make saving less cumbersome
 

MareSynchronos/Factories/FileCacheFactory.cs

@@ -0,0 +1,41 @@
+using System;
+using System.IO;
+using MareSynchronos.FileCacheDB;
+using System.Security.Cryptography;
+
+
+namespace MareSynchronos.Factories
+{
+    public class FileCacheFactory
+    {
+        public FileCacheFactory()
+        {
+
+        }
+
+        public FileCache Create(string file)
+        {
+            FileInfo fileInfo = new(file);
+            string sha1Hash = GetHash(fileInfo.FullName);
+            return new FileCache()
+            {
+                Filepath = fileInfo.FullName,
+                Hash = sha1Hash,
+                LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString(),
+            };
+        }
+
+        public void UpdateFileCache(FileCache cache)
+        {
+            FileInfo fileInfo = new(cache.Filepath);
+            cache.Hash = GetHash(cache.Filepath);
+            cache.LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString();
+        }
+
+        private string GetHash(string filePath)
+        {
+            using SHA1CryptoServiceProvider cryptoProvider = new();
+            return BitConverter.ToString(cryptoProvider.ComputeHash(File.ReadAllBytes(filePath)));
+        }
+    }
+}

MareSynchronos/FileCacheDB/FileCache.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+
+#nullable disable
+
+namespace MareSynchronos.FileCacheDB
+{
+    public partial class FileCache
+    {
+        public string Hash { get; set; }
+        public string Filepath { get; set; }
+        public string LastModifiedDate { get; set; }
+    }
+}

MareSynchronos/FileCacheDB/FileCacheContext.cs

@@ -0,0 +1,47 @@
+using System;
+using System.IO;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata;
+
+#nullable disable
+
+namespace MareSynchronos.FileCacheDB
+{
+    public partial class FileCacheContext : DbContext
+    {
+        public FileCacheContext()
+        {
+            Database.EnsureCreated();
+        }
+
+        public FileCacheContext(DbContextOptions<FileCacheContext> options)
+            : base(options)
+        {
+        }
+
+        public virtual DbSet<FileCache> FileCaches { get; set; }
+
+        protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
+        {
+            if (!optionsBuilder.IsConfigured)
+            {
+                string dbPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "XIVLauncher", "pluginConfigs", "FileCacheDebug.db");
+                optionsBuilder.UseSqlite("Data Source=" + dbPath);
+            }
+        }
+
+        protected override void OnModelCreating(ModelBuilder modelBuilder)
+        {
+            modelBuilder.Entity<FileCache>(entity =>
+            {
+                entity.HasKey(e => new { e.Hash, e.Filepath });
+
+                entity.ToTable("FileCache");
+            });
+
+            OnModelCreatingPartial(modelBuilder);
+        }
+
+        partial void OnModelCreatingPartial(ModelBuilder modelBuilder);
+    }
+}

MareSynchronos/MareSynchronos.csproj

@@ -26,6 +26,10 @@
 
   <ItemGroup>
     <PackageReference Include="DalamudPackager" Version="2.1.7" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="5.0.17">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.17" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite.Design" Version="2.0.0-preview1-final" />
   </ItemGroup>
@@ -61,4 +65,10 @@
     </Reference>
   </ItemGroup>
 
+  <ItemGroup>
+    <None Update="FileCache.db">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
 </Project>

MareSynchronos/Plugin.cs

@@ -1,9 +1,16 @@
 using Dalamud.Game.Command;
 using Dalamud.IoC;
+using Dalamud.Logging;
 using Dalamud.Plugin;
+using MareSynchronos.FileCacheDB;
+using MareSynchronos.Factories;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
-using System.Reflection;
+using System.Linq;
-//using System.Data.SQLite;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace SamplePlugin
 {
@@ -17,6 +24,9 @@ namespace SamplePlugin
         private CommandManager CommandManager { get; init; }
         private Configuration Configuration { get; init; }
         private PluginUI PluginUi { get; init; }
+        private FileCacheFactory FileCacheFactory { get; init; }
+
+        private CancellationTokenSource cts;
 
         public Plugin(
             [RequiredVersion("1.0")] DalamudPluginInterface pluginInterface,
@@ -33,9 +43,11 @@ namespace SamplePlugin
 
             this.CommandManager.AddHandler(commandName, new CommandInfo(OnCommand)
             {
-                HelpMessage = "A useful message to display in /xlhelp"
+                HelpMessage = "pass 'scan' to initialize or rescan files into the database"
             });
 
+            FileCacheFactory = new FileCacheFactory();
+
             this.PluginInterface.UiBuilder.Draw += DrawUI;
             this.PluginInterface.UiBuilder.OpenConfigUi += DrawConfigUI;
         }
@@ -48,8 +60,104 @@ namespace SamplePlugin
 
         private void OnCommand(string command, string args)
         {
-            //using var connection = new SQLiteConnection("Data Source=penumbracache.db");
+            if (args == "stop")
-            //connection.Open();
+            {
+                cts?.Cancel();
+                return;
+            }
+
+            if (args == "scan")
+            {
+                cts = new CancellationTokenSource();
+
+                Task.Run(() => StartScan(), cts.Token);
+            }
+        }
+
+        private void StartScan()
+        {
+            Stopwatch st = Stopwatch.StartNew();
+
+            string penumbraDir = Configuration.PenumbraFolder;
+            PluginLog.Debug("Getting files from " + penumbraDir);
+            ConcurrentDictionary<string, bool> charaFiles = new ConcurrentDictionary<string, bool>(
+                Directory.GetFiles(penumbraDir, "*.*", SearchOption.AllDirectories)
+                                .Select(s => s.ToLowerInvariant())
+                                .Where(f => !f.EndsWith(".json"))
+                                .Where(f => f.Contains(@"\chara\"))
+                                .Select(p => new KeyValuePair<string, bool>(p, false)));
+            int count = 0;
+            using FileCacheContext db = new();
+            var fileCaches = db.FileCaches.ToList();
+
+            var fileCachesToUpdate = new ConcurrentBag<FileCache>();
+            var fileCachesToDelete = new ConcurrentBag<FileCache>();
+            var fileCachesToAdd = new ConcurrentBag<FileCache>();
+
+            // scan files from database
+            Parallel.ForEach(fileCaches, new ParallelOptions()
+            {
+                CancellationToken = cts.Token,
+                MaxDegreeOfParallelism = 10
+            },
+            cache =>
+            {
+                count = Interlocked.Increment(ref count);
+                PluginLog.Debug($"[{count}/{fileCaches.Count}] Checking: {cache.Filepath}");
+
+                if (!File.Exists(cache.Filepath))
+                {
+                    PluginLog.Debug("File was not found anymore: " + cache.Filepath);
+                    fileCachesToDelete.Add(cache);
+                }
+                else
+                {
+                    charaFiles[cache.Filepath] = true;
+
+                    FileInfo fileInfo = new(cache.Filepath);
+                    if (fileInfo.LastWriteTimeUtc.Ticks != long.Parse(cache.LastModifiedDate))
+                    {
+                        PluginLog.Debug("File was modified since last time: " + cache.Filepath + "; " + cache.LastModifiedDate + " / " + fileInfo.LastWriteTimeUtc.Ticks);
+                        FileCacheFactory.UpdateFileCache(cache);
+                        fileCachesToUpdate.Add(cache);
+                    }
+                }
+            });
+
+            // scan new files
+            count = 0;
+            Parallel.ForEach(charaFiles.Where(c => c.Value == false), new ParallelOptions()
+            {
+                CancellationToken = cts.Token,
+                MaxDegreeOfParallelism = 10
+            },
+            file =>
+            {
+                count = Interlocked.Increment(ref count);
+                PluginLog.Debug($"[{count}/{charaFiles.Count()}] Hashing: {file.Key}");
+
+                fileCachesToAdd.Add(FileCacheFactory.Create(file.Key));
+            });
+
+            st.Stop();
+
+            if (cts.Token.IsCancellationRequested) return;
+
+            PluginLog.Debug("Scanning complete, total elapsed time: " + st.Elapsed.ToString());
+
+            if (fileCachesToAdd.Any() || fileCachesToUpdate.Any() || fileCachesToDelete.Any())
+            {
+                PluginLog.Debug("Writing files to database…");
+
+                db.FileCaches.AddRange(fileCachesToAdd);
+                db.FileCaches.UpdateRange(fileCachesToUpdate);
+                db.FileCaches.RemoveRange(fileCachesToDelete);
+
+                db.SaveChanges();
+                PluginLog.Debug("Database has been written.");
+            }
+
+            cts = new CancellationTokenSource();
         }
 
         private void DrawUI()

MareSynchronos/PluginUI.cs

@@ -59,8 +59,6 @@ namespace SamplePlugin
             ImGui.SetNextWindowSizeConstraints(new Vector2(375, 330), new Vector2(float.MaxValue, float.MaxValue));
             if (ImGui.Begin("My Amazing Window", ref this.visible, ImGuiWindowFlags.NoScrollbar | ImGuiWindowFlags.NoScrollWithMouse))
             {
-                ImGui.Text($"The random config bool is {this.configuration.SomePropertyToBeSavedAndWithADefault}");
-
                 if (ImGui.Button("Show Settings"))
                 {
                     SettingsVisible = true;
@@ -78,16 +76,14 @@ namespace SamplePlugin
                 return;
             }
 
-            ImGui.SetNextWindowSize(new Vector2(232, 75), ImGuiCond.Always);
+            ImGui.SetNextWindowSize(new Vector2(500, 75), ImGuiCond.Always);
-            if (ImGui.Begin("A Wonderful Configuration Window", ref this.settingsVisible,
+            if (ImGui.Begin("QUALITY UI DEVELOPMENT", ref this.settingsVisible,
                 ImGuiWindowFlags.NoResize | ImGuiWindowFlags.NoCollapse | ImGuiWindowFlags.NoScrollbar | ImGuiWindowFlags.NoScrollWithMouse))
             {
                 // can't ref a property, so use a local copy
-                var configValue = this.configuration.SomePropertyToBeSavedAndWithADefault;
+                string penumbraFolder = configuration.PenumbraFolder;
-                if (ImGui.Checkbox("Random Config Bool", ref configValue))
+                if(ImGui.InputText("Penumbra mod folder", ref penumbraFolder, 255)) {
-                {
+                    this.configuration.PenumbraFolder = penumbraFolder;
-                    this.configuration.SomePropertyToBeSavedAndWithADefault = configValue;
-                    // can save immediately on change, if you don't want to provide a "Save and Close" button
                     this.configuration.Save();
                 }
             }