t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add initial file scan and rescanning algorithms to hash all penumbra files

Browse Source
This commit is contained in:
Stanley Dimant
2022-06-10 01:41:47 +02:00
parent 3f737df71f
commit 3b9c485627
8 changed files with 231 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
MareSynchronos/Configuration.cs
View File

@@ -9,7 +9,7 @@ namespace SamplePlugin
{
public int Version { get; set; } = 0;
public bool SomePropertyToBeSavedAndWithADefault { get; set; } = true;
public string PenumbraFolder { get; set; } = string.Empty;
// the below exist just to make saving less cumbersome

41
MareSynchronos/Factories/FileCacheFactory.cs Normal file
View File

@@ -0,0 +1,41 @@
using System;
using System.IO;
using MareSynchronos.FileCacheDB;
using System.Security.Cryptography;
namespace MareSynchronos.Factories
{
public class FileCacheFactory
{
public FileCacheFactory()
{
}
public FileCache Create(string file)
{
FileInfo fileInfo = new(file);
string sha1Hash = GetHash(fileInfo.FullName);
return new FileCache()
{
Filepath = fileInfo.FullName,
Hash = sha1Hash,
LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString(),
};
}
public void UpdateFileCache(FileCache cache)
{
FileInfo fileInfo = new(cache.Filepath);
cache.Hash = GetHash(cache.Filepath);
cache.LastModifiedDate = fileInfo.LastWriteTimeUtc.Ticks.ToString();
}
private string GetHash(string filePath)
{
using SHA1CryptoServiceProvider cryptoProvider = new();
return BitConverter.ToString(cryptoProvider.ComputeHash(File.ReadAllBytes(filePath)));
}
}
}

14
MareSynchronos/FileCacheDB/FileCache.cs Normal file
View File

@@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
#nullable disable
namespace MareSynchronos.FileCacheDB
{
public partial class FileCache
{
public string Hash { get; set; }
public string Filepath { get; set; }
public string LastModifiedDate { get; set; }
}
}

47
MareSynchronos/FileCacheDB/FileCacheContext.cs Normal file
View File

@@ -0,0 +1,47 @@
using System;
using System.IO;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata;
#nullable disable
namespace MareSynchronos.FileCacheDB
{
public partial class FileCacheContext : DbContext
{
public FileCacheContext()
{
Database.EnsureCreated();
}
public FileCacheContext(DbContextOptions<FileCacheContext> options)
: base(options)
{
}
public virtual DbSet<FileCache> FileCaches { get; set; }
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
if (!optionsBuilder.IsConfigured)
{
string dbPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "XIVLauncher", "pluginConfigs", "FileCacheDebug.db");
optionsBuilder.UseSqlite("Data Source=" + dbPath);
}
}
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.Entity<FileCache>(entity =>
{
entity.HasKey(e => new { e.Hash, e.Filepath });
entity.ToTable("FileCache");
});
OnModelCreatingPartial(modelBuilder);
}
partial void OnModelCreatingPartial(ModelBuilder modelBuilder);
}
}

10
MareSynchronos/MareSynchronos.csproj
View File

@@ -26,6 +26,10 @@
<ItemGroup>
<PackageReference Include="DalamudPackager" Version="2.1.7" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="5.0.17">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.17" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite.Design" Version="2.0.0-preview1-final" />
</ItemGroup>
@@ -61,4 +65,10 @@
</Reference>
</ItemGroup>
<ItemGroup>
<None Update="FileCache.db">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

118
MareSynchronos/Plugin.cs
View File

@@ -1,9 +1,16 @@
using Dalamud.Game.Command;
using Dalamud.IoC;
using Dalamud.Logging;
using Dalamud.Plugin;
using MareSynchronos.FileCacheDB;
using MareSynchronos.Factories;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Reflection;
//using System.Data.SQLite;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace SamplePlugin
{
@@ -17,6 +24,9 @@ namespace SamplePlugin
private CommandManager CommandManager { get; init; }
private Configuration Configuration { get; init; }
private PluginUI PluginUi { get; init; }
private FileCacheFactory FileCacheFactory { get; init; }
private CancellationTokenSource cts;
public Plugin(
[RequiredVersion("1.0")] DalamudPluginInterface pluginInterface,
@@ -33,9 +43,11 @@ namespace SamplePlugin
this.CommandManager.AddHandler(commandName, new CommandInfo(OnCommand)
{
HelpMessage = "A useful message to display in /xlhelp"
HelpMessage = "pass 'scan' to initialize or rescan files into the database"
});
FileCacheFactory = new FileCacheFactory();
this.PluginInterface.UiBuilder.Draw += DrawUI;
this.PluginInterface.UiBuilder.OpenConfigUi += DrawConfigUI;
}
@@ -48,8 +60,104 @@ namespace SamplePlugin
private void OnCommand(string command, string args)
{
//using var connection = new SQLiteConnection("Data Source=penumbracache.db");
//connection.Open();
if (args == "stop")
{
cts?.Cancel();
return;
}
if (args == "scan")
{
cts = new CancellationTokenSource();
Task.Run(() => StartScan(), cts.Token);
}
}
private void StartScan()
{
Stopwatch st = Stopwatch.StartNew();
string penumbraDir = Configuration.PenumbraFolder;
PluginLog.Debug("Getting files from " + penumbraDir);
ConcurrentDictionary<string, bool> charaFiles = new ConcurrentDictionary<string, bool>(
Directory.GetFiles(penumbraDir, "*.*", SearchOption.AllDirectories)
.Select(s => s.ToLowerInvariant())
.Where(f => !f.EndsWith(".json"))
.Where(f => f.Contains(@"\chara\"))
.Select(p => new KeyValuePair<string, bool>(p, false)));
int count = 0;
using FileCacheContext db = new();
var fileCaches = db.FileCaches.ToList();
var fileCachesToUpdate = new ConcurrentBag<FileCache>();
var fileCachesToDelete = new ConcurrentBag<FileCache>();
var fileCachesToAdd = new ConcurrentBag<FileCache>();
// scan files from database
Parallel.ForEach(fileCaches, new ParallelOptions()
{
CancellationToken = cts.Token,
MaxDegreeOfParallelism = 10
},
cache =>
{
count = Interlocked.Increment(ref count);
PluginLog.Debug($"[{count}/{fileCaches.Count}] Checking: {cache.Filepath}");
if (!File.Exists(cache.Filepath))
{
PluginLog.Debug("File was not found anymore: " + cache.Filepath);
fileCachesToDelete.Add(cache);
}
else
{
charaFiles[cache.Filepath] = true;
FileInfo fileInfo = new(cache.Filepath);
if (fileInfo.LastWriteTimeUtc.Ticks != long.Parse(cache.LastModifiedDate))
{
PluginLog.Debug("File was modified since last time: " + cache.Filepath + "; " + cache.LastModifiedDate + " / " + fileInfo.LastWriteTimeUtc.Ticks);
FileCacheFactory.UpdateFileCache(cache);
fileCachesToUpdate.Add(cache);
}
}
});
// scan new files
count = 0;
Parallel.ForEach(charaFiles.Where(c => c.Value == false), new ParallelOptions()
{
CancellationToken = cts.Token,
MaxDegreeOfParallelism = 10
},
file =>
{
count = Interlocked.Increment(ref count);
PluginLog.Debug($"[{count}/{charaFiles.Count()}] Hashing: {file.Key}");
fileCachesToAdd.Add(FileCacheFactory.Create(file.Key));
});
st.Stop();
if (cts.Token.IsCancellationRequested) return;
PluginLog.Debug("Scanning complete, total elapsed time: " + st.Elapsed.ToString());
if (fileCachesToAdd.Any() || fileCachesToUpdate.Any() || fileCachesToDelete.Any())
{
PluginLog.Debug("Writing files to database…");
db.FileCaches.AddRange(fileCachesToAdd);
db.FileCaches.UpdateRange(fileCachesToUpdate);
db.FileCaches.RemoveRange(fileCachesToDelete);
db.SaveChanges();
PluginLog.Debug("Database has been written.");
}
cts = new CancellationTokenSource();
}
private void DrawUI()

14
MareSynchronos/PluginUI.cs
View File

@@ -59,8 +59,6 @@ namespace SamplePlugin
ImGui.SetNextWindowSizeConstraints(new Vector2(375, 330), new Vector2(float.MaxValue, float.MaxValue));
if (ImGui.Begin("My Amazing Window", ref this.visible, ImGuiWindowFlags.NoScrollbar | ImGuiWindowFlags.NoScrollWithMouse))
{
ImGui.Text($"The random config bool is {this.configuration.SomePropertyToBeSavedAndWithADefault}");
if (ImGui.Button("Show Settings"))
{
SettingsVisible = true;
@@ -78,16 +76,14 @@ namespace SamplePlugin
return;
}
ImGui.SetNextWindowSize(new Vector2(232, 75), ImGuiCond.Always);
if (ImGui.Begin("A Wonderful Configuration Window", ref this.settingsVisible,
ImGui.SetNextWindowSize(new Vector2(500, 75), ImGuiCond.Always);
if (ImGui.Begin("QUALITY UI DEVELOPMENT", ref this.settingsVisible,
ImGuiWindowFlags.NoResize | ImGuiWindowFlags.NoCollapse | ImGuiWindowFlags.NoScrollbar | ImGuiWindowFlags.NoScrollWithMouse))
{
// can't ref a property, so use a local copy
var configValue = this.configuration.SomePropertyToBeSavedAndWithADefault;
if (ImGui.Checkbox("Random Config Bool", ref configValue))
{
this.configuration.SomePropertyToBeSavedAndWithADefault = configValue;
// can save immediately on change, if you don't want to provide a "Save and Close" button
string penumbraFolder = configuration.PenumbraFolder;
if(ImGui.InputText("Penumbra mod folder", ref penumbraFolder, 255)) {
this.configuration.PenumbraFolder = penumbraFolder;
this.configuration.Save();
}
}

BIN
MareSynchronos/filecache.db
View File

Binary file not shown.