t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

performance optimizations

Browse Source
This commit is contained in:
Stanley Dimant
2022-09-26 00:16:05 +02:00
parent afb1cf1429
commit a75ff627b8
8 changed files with 45 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
MareSynchronos/FileCacheDB/FileCacheContext.cs
View File

@@ -1,5 +1,4 @@
using System.IO;
using MareSynchronos.Utils;
using Microsoft.EntityFrameworkCore;
#nullable disable
@@ -12,22 +11,6 @@ namespace MareSynchronos.FileCacheDB
public FileCacheContext()
{
DbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "FileCache.db");
string oldDbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "FileCacheDebug.db");
if (!Directory.Exists(Plugin.PluginInterface.ConfigDirectory.FullName))
{
Directory.CreateDirectory(Plugin.PluginInterface.ConfigDirectory.FullName);
}
var veryOldDbPath = Path.Combine(Plugin.PluginInterface.ConfigDirectory.FullName, "..", "FileCacheDebug.db");
if (File.Exists(veryOldDbPath))
{
Logger.Debug("Migrated old path to new path");
File.Move(veryOldDbPath, oldDbPath, true);
}
if (File.Exists(oldDbPath))
{
File.Move(oldDbPath, DbPath, true);
}
Database.EnsureCreated();
}
@@ -42,7 +25,7 @@ namespace MareSynchronos.FileCacheDB
{
if (!optionsBuilder.IsConfigured)
{
optionsBuilder.UseSqlite("Data Source=" + DbPath);
optionsBuilder.UseSqlite("Data Source=" + DbPath+";Cache=Shared");
}
}
@@ -53,8 +36,6 @@ namespace MareSynchronos.FileCacheDB
entity.HasKey(e => new { e.Hash, e.Filepath });
entity.ToTable("FileCache");
entity.Property(c => c.Version).HasDefaultValue(0).IsRowVersion();
});
OnModelCreatingPartial(modelBuilder);

7
MareSynchronos/FileCacheDB/FileDbManager.cs
View File

@@ -115,13 +115,10 @@ public class FileDbManager
private FileCache? GetValidatedFileCache(FileCacheEntity e)
{
var fileCache = new FileCache(e);
var resulingFileCache = MigrateLegacy(fileCache);
if (resulingFileCache == null) return null;
resulingFileCache = ReplacePathPrefixes(resulingFileCache);
resulingFileCache = Validate(resulingFileCache);
return resulingFileCache;
}
@@ -178,11 +175,11 @@ public class FileDbManager
private FileCache ReplacePathPrefixes(FileCache fileCache)
{
if (fileCache.OriginalFilepath.Contains(PenumbraPrefix))
if (fileCache.OriginalFilepath.StartsWith(PenumbraPrefix))
{
fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory()));
}
else if (fileCache.OriginalFilepath.Contains(CachePrefix))
else if (fileCache.OriginalFilepath.StartsWith(CachePrefix))
{
fileCache.SetResolvedFilePath(fileCache.OriginalFilepath.Replace(CachePrefix, _configuration.CacheFolder));
}

84
MareSynchronos/FileCacheDB/PeriodicFileScanner.cs
View File

@@ -1,5 +1,4 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@@ -84,7 +83,7 @@ public class PeriodicFileScanner : IDisposable
if (!_pluginConfiguration.FileScanPaused || isForced)
{
isForced = false;
await PeriodicFileScan(token);
PeriodicFileScan(token);
}
_timeUntilNextScan = TimeSpan.FromSeconds(timeBetweenScans);
while (_timeUntilNextScan.TotalSeconds >= 0)
@@ -130,7 +129,7 @@ public class PeriodicFileScanner : IDisposable
return true;
}
private async Task PeriodicFileScan(CancellationToken ct)
private void PeriodicFileScan(CancellationToken ct)
{
TotalFiles = 1;
var penumbraDir = _ipcManager.PenumbraModDirectory();
@@ -153,68 +152,69 @@ public class PeriodicFileScanner : IDisposable
Logger.Debug("Getting files from " + penumbraDir + " and " + _pluginConfiguration.CacheFolder);
string[] ext = { ".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp" };
var scannedFiles = new ConcurrentDictionary<string, bool>(
Directory.EnumerateFiles(penumbraDir, "*.*", SearchOption.AllDirectories)
.Select(s => new FileInfo(s))
.Where(f => ext.Contains(f.Extension) && !f.FullName.Contains(@"\bg\") && !f.FullName.Contains(@"\bgcommon\") && !f.FullName.Contains(@"\ui\"))
.Select(f => f.FullName.ToLowerInvariant())
.Concat(Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.AllDirectories)
var penumbraFiles = Directory.EnumerateDirectories(penumbraDir)
.SelectMany(d => Directory.EnumerateFiles(d, "*.*", SearchOption.AllDirectories)
.Select(s => new FileInfo(s))
.Where(f => ext.Contains(f.Extension) && !f.FullName.Contains(@"\bg\") && !f.FullName.Contains(@"\bgcommon\") && !f.FullName.Contains(@"\ui\"))
.Select(f => f.FullName.ToLowerInvariant())).ToList();
var cacheFiles = Directory.EnumerateFiles(_pluginConfiguration.CacheFolder, "*.*", SearchOption.TopDirectoryOnly)
.Where(f => new FileInfo(f).Name.Length == 40)
.Select(s => s.ToLowerInvariant()))
.Select(p => new KeyValuePair<string, bool>(p, false)).ToList());
.Select(s => s.ToLowerInvariant());
var scannedFiles = new Dictionary<string, bool>(penumbraFiles.Concat(cacheFiles).Select(c => new KeyValuePair<string, bool>(c, false)));
List<FileCacheEntity> fileDbEntries;
using (var db = new FileCacheContext())
{
fileDbEntries = await db.FileCaches.ToListAsync(cancellationToken: ct);
fileDbEntries = db.FileCaches.AsNoTracking().ToList();
}
TotalFiles = scannedFiles.Count;
Logger.Debug("Database contains " + fileDbEntries.Count + " files, local system contains " + TotalFiles);
// scan files from database
Parallel.ForEach(fileDbEntries.ToList(), new ParallelOptions()
var cpuCount = (int)(Environment.ProcessorCount / 2.0f);
foreach (var chunk in fileDbEntries.Chunk(cpuCount))
{
MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
CancellationToken = ct,
},
dbEntry =>
{
if (ct.IsCancellationRequested) return;
try
Task[] tasks = chunk.Select(c => Task.Run(() =>
{
var file = _fileDbManager.ValidateFileCache(dbEntry);
if (file != null && scannedFiles.ContainsKey(file.Filepath))
var file = _fileDbManager.ValidateFileCache(c);
if (file != null)
{
scannedFiles[file.Filepath] = true;
}
}
catch (Exception ex)
{
Logger.Warn(ex.Message);
Logger.Warn(ex.StackTrace);
}
Interlocked.Increment(ref currentFileProgress);
});
Interlocked.Increment(ref currentFileProgress);
})).ToArray();
Task.WaitAll(tasks, ct);
Thread.Sleep(3);
if (ct.IsCancellationRequested) return;
}
Logger.Debug("Scanner validated existing db files");
if (ct.IsCancellationRequested) return;
// scan new files
Parallel.ForEach(scannedFiles.Where(c => c.Value == false), new ParallelOptions()
foreach (var chunk in scannedFiles.Where(c => c.Value == false).Chunk(cpuCount))
{
MaxDegreeOfParallelism = _pluginConfiguration.MaxParallelScan,
CancellationToken = ct
},
file =>
{
if (ct.IsCancellationRequested) return;
Task[] tasks = chunk.Select(c => Task.Run(() =>
{
var entry = _fileDbManager.CreateFileEntry(c.Key);
if (entry == null) _ = _fileDbManager.CreateCacheEntry(c.Key);
var entry = _fileDbManager.CreateFileEntry(file.Key);
if (entry == null) _ = _fileDbManager.CreateCacheEntry(file.Key);
Interlocked.Increment(ref currentFileProgress);
});
Interlocked.Increment(ref currentFileProgress);
})).ToArray();
Task.WaitAll(tasks, ct);
Thread.Sleep(3);
if (ct.IsCancellationRequested) return;
}
Logger.Debug("Scanner added new files to db");