diff --git a/MareSynchronos/Configuration.cs b/MareSynchronos/Configuration.cs index f71a1b9..a76485e 100644 --- a/MareSynchronos/Configuration.cs +++ b/MareSynchronos/Configuration.cs @@ -9,7 +9,7 @@ namespace SamplePlugin { public int Version { get; set; } = 0; - public bool SomePropertyToBeSavedAndWithADefault { get; set; } = true; + public string PenumbraFolder { get; set; } = string.Empty; // the below exist just to make saving less cumbersome diff --git a/MareSynchronos/Factories/FileCacheFactory.cs b/MareSynchronos/Factories/FileCacheFactory.cs new file mode 100644 index 0000000..2e9820f --- /dev/null +++ b/MareSynchronos/Factories/FileCacheFactory.cs @@ -0,0 +1,41 @@ +using System; +using System.IO; +using MareSynchronos.FileCacheDB; +using System.Security.Cryptography; + + +namespace MareSynchronos.Factories +{ + public class FileCacheFactory + { + public FileCacheFactory() + { + + } + + public FileCache Create(string file) + { + FileInfo fileInfo = new(file); + string sha1Hash = GetHash(fileInfo.FullName); + return new FileCache() + { + Filepath = fileInfo.FullName, + Hash = sha1Hash, + LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString(), + }; + } + + public void UpdateFileCache(FileCache cache) + { + FileInfo fileInfo = new(cache.Filepath); + cache.Hash = GetHash(cache.Filepath); + cache.LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString(); + } + + private string GetHash(string filePath) + { + using SHA1CryptoServiceProvider cryptoProvider = new(); + return BitConverter.ToString(cryptoProvider.ComputeHash(File.ReadAllBytes(filePath))); + } + } +} diff --git a/MareSynchronos/FileCacheDB/FileCache.cs b/MareSynchronos/FileCacheDB/FileCache.cs new file mode 100644 index 0000000..0ce783e --- /dev/null +++ b/MareSynchronos/FileCacheDB/FileCache.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; + +#nullable disable + +namespace MareSynchronos.FileCacheDB +{ + public partial class FileCache + { + public string Hash { get; set; } + public string Filepath { get; set; } + public string LastModifiedDate { get; set; } + } +} diff --git a/MareSynchronos/FileCacheDB/FileCacheContext.cs b/MareSynchronos/FileCacheDB/FileCacheContext.cs new file mode 100644 index 0000000..7813da8 --- /dev/null +++ b/MareSynchronos/FileCacheDB/FileCacheContext.cs @@ -0,0 +1,47 @@ +using System; +using System.IO; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; + +#nullable disable + +namespace MareSynchronos.FileCacheDB +{ + public partial class FileCacheContext : DbContext + { + public FileCacheContext() + { + Database.EnsureCreated(); + } + + public FileCacheContext(DbContextOptions options) + : base(options) + { + } + + public virtual DbSet FileCaches { get; set; } + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + { + if (!optionsBuilder.IsConfigured) + { + string dbPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "XIVLauncher", "pluginConfigs", "FileCacheDebug.db"); + optionsBuilder.UseSqlite("Data Source=" + dbPath); + } + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.HasKey(e => new { e.Hash, e.Filepath }); + + entity.ToTable("FileCache"); + }); + + OnModelCreatingPartial(modelBuilder); + } + + partial void OnModelCreatingPartial(ModelBuilder modelBuilder); + } +} diff --git a/MareSynchronos/MareSynchronos.csproj b/MareSynchronos/MareSynchronos.csproj index bf057d7..675cea6 100644 --- a/MareSynchronos/MareSynchronos.csproj +++ b/MareSynchronos/MareSynchronos.csproj @@ -26,6 +26,10 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + @@ -61,4 +65,10 @@ + + + Always + + + diff --git a/MareSynchronos/Plugin.cs b/MareSynchronos/Plugin.cs index 4383b7a..d7366a6 100644 --- a/MareSynchronos/Plugin.cs +++ b/MareSynchronos/Plugin.cs @@ -1,9 +1,16 @@ using Dalamud.Game.Command; using Dalamud.IoC; +using Dalamud.Logging; using Dalamud.Plugin; +using MareSynchronos.FileCacheDB; +using MareSynchronos.Factories; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; using System.IO; -using System.Reflection; -//using System.Data.SQLite; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; namespace SamplePlugin { @@ -17,6 +24,9 @@ namespace SamplePlugin private CommandManager CommandManager { get; init; } private Configuration Configuration { get; init; } private PluginUI PluginUi { get; init; } + private FileCacheFactory FileCacheFactory { get; init; } + + private CancellationTokenSource cts; public Plugin( [RequiredVersion("1.0")] DalamudPluginInterface pluginInterface, @@ -33,9 +43,11 @@ namespace SamplePlugin this.CommandManager.AddHandler(commandName, new CommandInfo(OnCommand) { - HelpMessage = "A useful message to display in /xlhelp" + HelpMessage = "pass 'scan' to initialize or rescan files into the database" }); + FileCacheFactory = new FileCacheFactory(); + this.PluginInterface.UiBuilder.Draw += DrawUI; this.PluginInterface.UiBuilder.OpenConfigUi += DrawConfigUI; } @@ -48,8 +60,104 @@ namespace SamplePlugin private void OnCommand(string command, string args) { - //using var connection = new SQLiteConnection("Data Source=penumbracache.db"); - //connection.Open(); + if (args == "stop") + { + cts?.Cancel(); + return; + } + + if (args == "scan") + { + cts = new CancellationTokenSource(); + + Task.Run(() => StartScan(), cts.Token); + } + } + + private void StartScan() + { + Stopwatch st = Stopwatch.StartNew(); + + string penumbraDir = Configuration.PenumbraFolder; + PluginLog.Debug("Getting files from " + penumbraDir); + ConcurrentDictionary charaFiles = new ConcurrentDictionary( + Directory.GetFiles(penumbraDir, "*.*", SearchOption.AllDirectories) + .Select(s => s.ToLowerInvariant()) + .Where(f => !f.EndsWith(".json")) + .Where(f => f.Contains(@"\chara\")) + .Select(p => new KeyValuePair(p, false))); + int count = 0; + using FileCacheContext db = new(); + var fileCaches = db.FileCaches.ToList(); + + var fileCachesToUpdate = new ConcurrentBag(); + var fileCachesToDelete = new ConcurrentBag(); + var fileCachesToAdd = new ConcurrentBag(); + + // scan files from database + Parallel.ForEach(fileCaches, new ParallelOptions() + { + CancellationToken = cts.Token, + MaxDegreeOfParallelism = 10 + }, + cache => + { + count = Interlocked.Increment(ref count); + PluginLog.Debug($"[{count}/{fileCaches.Count}] Checking: {cache.Filepath}"); + + if (!File.Exists(cache.Filepath)) + { + PluginLog.Debug("File was not found anymore: " + cache.Filepath); + fileCachesToDelete.Add(cache); + } + else + { + charaFiles[cache.Filepath] = true; + + FileInfo fileInfo = new(cache.Filepath); + if (fileInfo.LastWriteTimeUtc.Ticks != long.Parse(cache.LastModifiedDate)) + { + PluginLog.Debug("File was modified since last time: " + cache.Filepath + "; " + cache.LastModifiedDate + " / " + fileInfo.LastWriteTimeUtc.Ticks); + FileCacheFactory.UpdateFileCache(cache); + fileCachesToUpdate.Add(cache); + } + } + }); + + // scan new files + count = 0; + Parallel.ForEach(charaFiles.Where(c => c.Value == false), new ParallelOptions() + { + CancellationToken = cts.Token, + MaxDegreeOfParallelism = 10 + }, + file => + { + count = Interlocked.Increment(ref count); + PluginLog.Debug($"[{count}/{charaFiles.Count()}] Hashing: {file.Key}"); + + fileCachesToAdd.Add(FileCacheFactory.Create(file.Key)); + }); + + st.Stop(); + + if (cts.Token.IsCancellationRequested) return; + + PluginLog.Debug("Scanning complete, total elapsed time: " + st.Elapsed.ToString()); + + if (fileCachesToAdd.Any() || fileCachesToUpdate.Any() || fileCachesToDelete.Any()) + { + PluginLog.Debug("Writing files to database…"); + + db.FileCaches.AddRange(fileCachesToAdd); + db.FileCaches.UpdateRange(fileCachesToUpdate); + db.FileCaches.RemoveRange(fileCachesToDelete); + + db.SaveChanges(); + PluginLog.Debug("Database has been written."); + } + + cts = new CancellationTokenSource(); } private void DrawUI() diff --git a/MareSynchronos/PluginUI.cs b/MareSynchronos/PluginUI.cs index a0b3baa..1e3ca93 100644 --- a/MareSynchronos/PluginUI.cs +++ b/MareSynchronos/PluginUI.cs @@ -59,8 +59,6 @@ namespace SamplePlugin ImGui.SetNextWindowSizeConstraints(new Vector2(375, 330), new Vector2(float.MaxValue, float.MaxValue)); if (ImGui.Begin("My Amazing Window", ref this.visible, ImGuiWindowFlags.NoScrollbar | ImGuiWindowFlags.NoScrollWithMouse)) { - ImGui.Text($"The random config bool is {this.configuration.SomePropertyToBeSavedAndWithADefault}"); - if (ImGui.Button("Show Settings")) { SettingsVisible = true; @@ -78,16 +76,14 @@ namespace SamplePlugin return; } - ImGui.SetNextWindowSize(new Vector2(232, 75), ImGuiCond.Always); - if (ImGui.Begin("A Wonderful Configuration Window", ref this.settingsVisible, + ImGui.SetNextWindowSize(new Vector2(500, 75), ImGuiCond.Always); + if (ImGui.Begin("QUALITY UI DEVELOPMENT", ref this.settingsVisible, ImGuiWindowFlags.NoResize | ImGuiWindowFlags.NoCollapse | ImGuiWindowFlags.NoScrollbar | ImGuiWindowFlags.NoScrollWithMouse)) { // can't ref a property, so use a local copy - var configValue = this.configuration.SomePropertyToBeSavedAndWithADefault; - if (ImGui.Checkbox("Random Config Bool", ref configValue)) - { - this.configuration.SomePropertyToBeSavedAndWithADefault = configValue; - // can save immediately on change, if you don't want to provide a "Save and Close" button + string penumbraFolder = configuration.PenumbraFolder; + if(ImGui.InputText("Penumbra mod folder", ref penumbraFolder, 255)) { + this.configuration.PenumbraFolder = penumbraFolder; this.configuration.Save(); } } diff --git a/MareSynchronos/filecache.db b/MareSynchronos/filecache.db index 90c8852..aae39a7 100644 Binary files a/MareSynchronos/filecache.db and b/MareSynchronos/filecache.db differ