t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

performance optimizations

Browse Source
This commit is contained in:
Stanley Dimant
2022-09-26 00:16:05 +02:00
parent afb1cf1429
commit a75ff627b8
8 changed files with 45 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
MareSynchronos/Configuration.cs
View File

@@ -57,26 +57,12 @@ namespace MareSynchronos
public Dictionary<string, string> ClientSecret { get; set; } = new();
public Dictionary<string, string> CustomServerList { get; set; } = new();
public int MaxLocalCacheInGiB { get; set; } = 20;
public bool ReverseUserSort { get; set; } = true;
public int TimeSpanBetweenScansInSeconds { get; set; } = 30;
public bool FileScanPaused { get; set; } = false;
public bool InitialScanComplete { get; set; } = false;
public int MaxParallelScan
{
get => _maxParallelScan;
set
{
_maxParallelScan = value switch
{
< 0 => 1,
> 20 => 10,
_ => value
};
}
}
public bool FullPause { get; set; } = false;
public Dictionary<string, Dictionary<string, string>> UidServerComments { get; set; } = new();

21
MareSynchronos/FileCacheDB/FileCacheContext.cs
View File

@@ -1,5 +1,4 @@
using System.IO;
using MareSynchronos.Utils;
using Microsoft.EntityFrameworkCore;
#nullable disable
@@ -12,22 +11,6 @@ namespace MareSynchronos.FileCacheDB
public FileCacheContext()
{
DbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "FileCache.db");
string oldDbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "FileCacheDebug.db");
if (!Directory.Exists(Plugin.PluginInterface.ConfigDirectory.FullName))
{
Directory.CreateDirectory(Plugin.PluginInterface.ConfigDirectory.FullName);
}
var veryOldDbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "..", "FileCacheDebug.db");
if (File.Exists(veryOldDbPath))
{
Logger.Debug("Migrated old path to new path");
File.Move(veryOldDbPath, oldDbPath, true);
}
if (File.Exists(oldDbPath))
{
File.Move(oldDbPath, DbPath, true);
}
Database.EnsureCreated();
}
@@ -42,7 +25,7 @@ namespace MareSynchronos.FileCacheDB
{
if (!optionsBuilder.IsConfigured)
{
optionsBuilder.UseSqlite("Data Source=" + DbPath);
optionsBuilder.UseSqlite("Data Source=" + DbPath+";Cache=Shared");
}
}
@@ -53,8 +36,6 @@ namespace MareSynchronos.FileCacheDB
entity.HasKey(e => new { e.Hash, e.Filepath });
entity.ToTable("FileCache");
entity.Property(c => c.Version).HasDefaultValue(0).IsRowVersion();
});
OnModelCreatingPartial(modelBuilder);

7
MareSynchronos/FileCacheDB/FileDbManager.cs
View File

@@ -115,13 +115,10 @@ public class FileDbManager
private FileCache? GetValidatedFileCache(FileCacheEntity e)
{
var fileCache = new FileCache(e);
var resulingFileCache = MigrateLegacy(fileCache);
if (resulingFileCache == null) return null;
resulingFileCache = ReplacePathPrefixes(resulingFileCache);
resulingFileCache = Validate(resulingFileCache);
return resulingFileCache;
}
@@ -178,11 +175,11 @@ public class FileDbManager
private FileCache ReplacePathPrefixes(FileCache fileCache)
{
if (fileCache.OriginalFilepath.Contains(PenumbraPrefix))
if (fileCache.OriginalFilepath.StartsWith(PenumbraPrefix))
{
fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory()));
}
else if (fileCache.OriginalFilepath.Contains(CachePrefix))
else if (fileCache.OriginalFilepath.StartsWith(CachePrefix))
{
fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(CachePrefix, _configuration.CacheFolder));
}

84
MareSynchronos/FileCacheDB/PeriodicFileScanner.cs
View File

@@ -1,5 +1,4 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@@ -84,7 +83,7 @@ public class PeriodicFileScanner : IDisposable
if (!_pluginConfiguration.FileScanPaused || isForced)
{
isForced = false;
await PeriodicFileScan(token);
PeriodicFileScan(token);
}
_timeUntilNextScan = TimeSpan.FromSeconds(timeBetweenScans);
while (_timeUntilNextScan.TotalSeconds >= 0)
@@ -130,7 +129,7 @@ public class PeriodicFileScanner : IDisposable
return true;
}
private async Task PeriodicFileScan(CancellationToken ct)
private void PeriodicFileScan(CancellationToken ct)
{
TotalFiles = 1;
var penumbraDir = _ipcManager.PenumbraModDirectory();
@@ -153,68 +152,69 @@ public class PeriodicFileScanner : IDisposable
Logger.Debug("Getting files from " + penumbraDir + " and " + _pluginConfiguration.CacheFolder);
string[] ext = { ".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp" };
var scannedFiles = new ConcurrentDictionary<string, bool>(
Directory.EnumerateFiles(penumbraDir, "*.*", SearchOption.AllDirectories)
.Select(s => new FileInfo(s))
.Where(f => ext.Contains(f.Extension) && !f.FullName.Contains(@"\bg\") && !f.FullName.Contains(@"\bgcommon\") && !f.FullName.Contains(@"\ui\"))
.Select(f => f.FullName.ToLowerInvariant())
.Concat(Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.AllDirectories)
var penumbraFiles = Directory.EnumerateDirectories(penumbraDir)
.SelectMany(d => Directory.EnumerateFiles(d, "*.*", SearchOption.AllDirectories)
.Select(s => new FileInfo(s))
.Where(f => ext.Contains(f.Extension) && !f.FullName.Contains(@"\bg\") && !f.FullName.Contains(@"\bgcommon\") && !f.FullName.Contains(@"\ui\"))
.Select(f => f.FullName.ToLowerInvariant())).ToList();
var cacheFiles = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.TopDirectoryOnly)
.Where(f => new FileInfo(f).Name.Length == 40)
.Select(s => s.ToLowerInvariant()))
.Select(p => new KeyValuePair<string, bool>(p, false)).ToList());
.Select(s => s.ToLowerInvariant());
var scannedFiles = new Dictionary<string, bool>(penumbraFiles.Concat(cacheFiles).Select(c => new KeyValuePair<string, bool>(c, false)));
List<FileCacheEntity> fileDbEntries;
using (var db = new FileCacheContext())
{
fileDbEntries = await db.FileCaches.ToListAsync(cancellationToken: ct);
fileDbEntries = db.FileCaches.AsNoTracking().ToList();
}
TotalFiles = scannedFiles.Count;
Logger.Debug("Database contains " + fileDbEntries.Count + " files, local system contains " + TotalFiles);
// scan files from database
Parallel.ForEach(fileDbEntries.ToList(), new ParallelOptions()
var cpuCount = (int)(Environment.ProcessorCount / 2.0f);
foreach (var chunk in fileDbEntries.Chunk(cpuCount))
{
MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
CancellationToken = ct,
},
dbEntry =>
{
if (ct.IsCancellationRequested) return;
try
Task[] tasks = chunk.Select(c => Task.Run(() =>
{
var file = _fileDbManager.ValidateFileCache(dbEntry);
if (file != null && scannedFiles.ContainsKey(file.Filepath))
var file = _fileDbManager.ValidateFileCache(c);
if (file != null)
{
scannedFiles[file.Filepath] = true;
}
}
catch (Exception ex)
{
Logger.Warn(ex.Message);
Logger.Warn(ex.StackTrace);
}
Interlocked.Increment(ref currentFileProgress);
});
Interlocked.Increment(ref currentFileProgress);
})).ToArray();
Task.WaitAll(tasks, ct);
Thread.Sleep(3);
if (ct.IsCancellationRequested) return;
}
Logger.Debug("Scanner validated existing db files");
if (ct.IsCancellationRequested) return;
// scan new files
Parallel.ForEach(scannedFiles.Where(c => c.Value == false), new ParallelOptions()
foreach (var chunk in scannedFiles.Where(c => c.Value == false).Chunk(cpuCount))
{
MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
CancellationToken = ct
},
file =>
{
if (ct.IsCancellationRequested) return;
Task[] tasks = chunk.Select(c => Task.Run(() =>
{
var entry = _fileDbManager.CreateFileEntry(c.Key);
if (entry == null) _ = _fileDbManager.CreateCacheEntry(c.Key);
var entry = _fileDbManager.CreateFileEntry(file.Key);
if (entry == null) _ = _fileDbManager.CreateCacheEntry(file.Key);
Interlocked.Increment(ref currentFileProgress);
});
Interlocked.Increment(ref currentFileProgress);
})).ToArray();
Task.WaitAll(tasks, ct);
Thread.Sleep(3);
if (ct.IsCancellationRequested) return;
}
Logger.Debug("Scanner added new files to db");

7
MareSynchronos/Plugin.cs
View File

@@ -23,7 +23,6 @@ namespace MareSynchronos
private const string CommandName = "/mare";
private readonly ApiController _apiController;
private readonly CommandManager _commandManager;
private readonly Framework _framework;
private readonly Configuration _configuration;
private readonly PeriodicFileScanner _fileCacheManager;
private readonly IntroUi _introUi;
@@ -49,7 +48,6 @@ namespace MareSynchronos
Logger.Debug("Launching " + Name);
PluginInterface = pluginInterface;
_commandManager = commandManager;
_framework = framework;
_configuration = PluginInterface.GetPluginConfig() as Configuration ?? new Configuration();
_configuration.Initialize(PluginInterface);
_configuration.Migrate();
@@ -65,11 +63,6 @@ namespace MareSynchronos
_dalamudUtil = new DalamudUtil(clientState, objectTable, framework, condition);
_ipcManager = new IpcManager(PluginInterface, _dalamudUtil);
// Compatibility for FileSystemWatchers under OSX
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
Environment.SetEnvironmentVariable("MONO_MANAGED_WATCHER", "enabled");
_fileDialogManager = new FileDialogManager();
_fileDbManager = new FileDbManager(_ipcManager, _configuration);
_apiController = new ApiController(_configuration, _dalamudUtil, _fileDbManager);

4
MareSynchronos/UI/IntroUI.cs
View File

@@ -220,10 +220,6 @@ namespace MareSynchronos.UI
if (!_fileCacheManager.IsScanRunning && !string.IsNullOrEmpty(_pluginConfiguration.CacheFolder) && _uiShared.HasValidPenumbraModPath && Directory.Exists(_pluginConfiguration.CacheFolder))
{
UiShared.TextWrapped("You can adjust how many parallel threads will be used for scanning. Mind that ultimately it will depend on the amount of mods, your disk speed and your CPU. " +
"More is not necessarily better, the default of 10 should be fine for most cases.");
_uiShared.DrawParallelScansSetting();
if (ImGui.Button("Start Scan##startScan"))
{
_fileCacheManager.InvokeScan(true);

1
MareSynchronos/UI/SettingsUi.cs
View File

@@ -586,7 +586,6 @@ namespace MareSynchronos.UI
private void DrawFileCacheSettings()
{
_uiShared.DrawFileScanState();
_uiShared.DrawParallelScansSetting();
_uiShared.DrawTimeSpanBetweenScansSetting();
_uiShared.DrawCacheDirectorySetting();
ImGui.Text($"Local cache size: {UiShared.ByteToString(_uiShared.FileCacheSize)}");

11
MareSynchronos/UI/UIShared.cs
View File

@@ -520,17 +520,6 @@ namespace MareSynchronos.UI
_cacheScanner.InvokeScan(true);
}
public void DrawParallelScansSetting()
{
var parallelScans = _pluginConfiguration.MaxParallelScan;
if (ImGui.SliderInt("File scan parallelism##parallelism", ref parallelScans, 1, 20))
{
_pluginConfiguration.MaxParallelScan = parallelScans;
_pluginConfiguration.Save();
}
DrawHelpText("Decrease to lessen load of file scans. File scans will take longer to execute with less parallel threads.");
}
public void DrawTimeSpanBetweenScansSetting()
{
var timeSpan = _pluginConfiguration.TimeSpanBetweenScansInSeconds;