t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

stop scan on character data creation, downloads, gpose, zone switch, update outside of read loop, adjust verify

Browse Source
This commit is contained in:
Stanley Dimant
2022-09-28 00:16:14 +02:00
parent 1241e9c14e
commit 287c1b0eff
7 changed files with 107 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
MareSynchronos/FileCacheDB/FileDbManager.cs
View File

@@ -8,6 +8,13 @@ using System.Linq;
namespace MareSynchronos.Managers;
public enum FileState
{
Valid,
RequireUpdate,
RequireDeletion
}
public class FileDbManager
{
private const string PenumbraPrefix = "{penumbra}";
@@ -45,9 +52,19 @@ public class FileDbManager
return GetValidatedFileCache(new FileCache(matchingEntries.First()));
}
public FileCache? ValidateFileCacheEntity(string hash, string path, string lastModifiedDate)
public (FileState, string) ValidateFileCacheEntity(string hash, string path, string lastModifiedDate)
{
return GetValidatedFileCache(new FileCache(hash, path, lastModifiedDate), false);
var fileCache = new FileCache(hash, path, lastModifiedDate);
if (!fileCache.OriginalFilepath.StartsWith(PenumbraPrefix + "\\") && !fileCache.OriginalFilepath.StartsWith(CachePrefix))
return (FileState.RequireUpdate, path);
fileCache = ReplacePathPrefixes(fileCache);
FileInfo fi = new FileInfo(fileCache.Filepath);
if (!fi.Exists)
return (FileState.RequireDeletion, fileCache.Filepath);
if (fi.LastWriteTimeUtc.Ticks != fileCache.LastModifiedDateTicks)
return (FileState.RequireUpdate, fileCache.Filepath);
return (FileState.Valid, fileCache.Filepath);
}
public FileCache? GetFileCacheByPath(string path)
@@ -61,6 +78,7 @@ public class FileDbManager
if (matchingEntries == null)
{
Logger.Debug("Found no entries for " + cleanedPath);
return CreateFileEntry(path);
}
@@ -104,7 +122,8 @@ public class FileDbManager
}
catch (Exception ex)
{
Logger.Warn("Could not add " + fileInfo.FullName);
Logger.Warn("Could not add " + fileInfo.FullName ?? String.Empty);
Logger.Warn(ex.Message);
}
}
var result = GetFileCacheByPath(prefixedPath);
@@ -112,25 +131,22 @@ public class FileDbManager
return result;
}
private FileCache? GetValidatedFileCache(FileCache fileCache, bool removeOnNonExistence = true)
private FileCache? GetValidatedFileCache(FileCache fileCache)
{
var resulingFileCache = MigrateLegacy(fileCache);
if (resulingFileCache == null) return null;
resulingFileCache = ReplacePathPrefixes(resulingFileCache);
resulingFileCache = Validate(resulingFileCache, removeOnNonExistence);
resulingFileCache = Validate(resulingFileCache);
return resulingFileCache;
}
private FileCache? Validate(FileCache fileCache, bool removeOnNonExistence = true)
private FileCache? Validate(FileCache fileCache)
{
var file = new FileInfo(fileCache.Filepath);
if (!file.Exists)
{
if (removeOnNonExistence)
{
DeleteFromDatabase(new[] { fileCache });
}
DeleteFromDatabase(new[] { fileCache });
return null;
}
@@ -143,7 +159,7 @@ public class FileDbManager
return fileCache;
}
private FileCache? MigrateLegacy(FileCache fileCache, bool removeOnNonExistence = true)
private FileCache? MigrateLegacy(FileCache fileCache)
{
if (fileCache.OriginalFilepath.StartsWith(PenumbraPrefix + "\\") || fileCache.OriginalFilepath.StartsWith(CachePrefix)) return fileCache;
@@ -168,10 +184,7 @@ public class FileDbManager
}
else
{
if (removeOnNonExistence)
{
DeleteFromDatabase(new[] { fileCache });
}
DeleteFromDatabase(new[] { fileCache });
return null;
}
@@ -214,7 +227,9 @@ public class FileDbManager
}
catch (Exception ex)
{
Logger.Warn("Error updating file hash (" + ex.Message + "), returning currently existing");
Logger.Warn("Error updating file hash (" + ex.Message + "), returning currently existing" ?? string.Empty);
Logger.Warn(ex.InnerException?.Message ?? string.Empty);
Logger.Warn(ex.StackTrace ?? string.Empty);
using var db = new FileCacheContext();
var cache = db.FileCaches.First(f => f.Filepath == markedForUpdate.OriginalFilepath);
markedForUpdate.UpdateFileCache(cache);

75
MareSynchronos/FileCacheDB/PeriodicFileScanner.cs
View File

@@ -1,6 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
@@ -18,9 +17,11 @@ public class PeriodicFileScanner : IDisposable
private readonly Configuration _pluginConfiguration;
private readonly FileDbManager _fileDbManager;
private readonly ApiController _apiController;
private readonly DalamudUtil _dalamudUtil;
private int haltScanRequests = 0;
private CancellationTokenSource? _scanCancellationTokenSource;
private Task? _fileScannerTask = null;
public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileDbManager fileDbManager, ApiController apiController)
public PeriodicFileScanner(IpcManager ipcManager, Configuration pluginConfiguration, FileDbManager fileDbManager, ApiController apiController, DalamudUtil dalamudUtil)
{
Logger.Verbose("Creating " + nameof(PeriodicFileScanner));
@@ -28,27 +29,34 @@ public class PeriodicFileScanner : IDisposable
_pluginConfiguration = pluginConfiguration;
_fileDbManager = fileDbManager;
_apiController = apiController;
_dalamudUtil = dalamudUtil;
_ipcManager.PenumbraInitialized += StartScan;
if (!string.IsNullOrEmpty(_ipcManager.PenumbraModDirectory()))
{
StartScan();
}
_apiController.DownloadStarted += _apiController_DownloadStarted;
_apiController.DownloadFinished += _apiController_DownloadFinished;
_apiController.DownloadStarted += HaltScan;
_apiController.DownloadFinished += ResumeScan;
_dalamudUtil.ZoneSwitchStart += HaltScan;
_dalamudUtil.ZoneSwitchEnd += ResumeScan;
}
private void _apiController_DownloadFinished()
public void ResumeScan()
{
if (fileScanWasRunning)
Interlocked.Decrement(ref haltScanRequests);
if (fileScanWasRunning && haltScanRequests == 0)
{
fileScanWasRunning = false;
InvokeScan(true);
}
}
private void _apiController_DownloadStarted()
public void HaltScan()
{
if (IsScanRunning)
Interlocked.Increment(ref haltScanRequests);
if (IsScanRunning && haltScanRequests >= 0)
{
_scanCancellationTokenSource?.Cancel();
fileScanWasRunning = true;
@@ -74,8 +82,10 @@ public class PeriodicFileScanner : IDisposable
Logger.Verbose("Disposing " + nameof(PeriodicFileScanner));
_ipcManager.PenumbraInitialized -= StartScan;
_apiController.DownloadStarted -= _apiController_DownloadStarted;
_apiController.DownloadFinished -= _apiController_DownloadFinished;
_apiController.DownloadStarted -= HaltScan;
_apiController.DownloadFinished -= ResumeScan;
_dalamudUtil.ZoneSwitchStart -= HaltScan;
_dalamudUtil.ZoneSwitchEnd -= ResumeScan;
_scanCancellationTokenSource?.Cancel();
}
@@ -91,6 +101,11 @@ public class PeriodicFileScanner : IDisposable
{
while (!token.IsCancellationRequested)
{
while (haltScanRequests > 0)
{
await Task.Delay(TimeSpan.FromSeconds(1));
}
isForced |= RecalculateFileCacheSize();
if (!_pluginConfiguration.FileScanPaused || isForced)
{
@@ -98,6 +113,8 @@ public class PeriodicFileScanner : IDisposable
TotalFiles = 0;
currentFileProgress = 0;
PeriodicFileScan(token);
TotalFiles = 0;
currentFileProgress = 0;
}
_timeUntilNextScan = TimeSpan.FromSeconds(timeBetweenScans);
while (_timeUntilNextScan.TotalSeconds >= 0)
@@ -182,6 +199,7 @@ public class PeriodicFileScanner : IDisposable
Task[] dbTasks = Enumerable.Range(0, cpuCount).Select(c => Task.CompletedTask).ToArray();
ConcurrentBag<Tuple<string, string>> entitiesToRemove = new();
ConcurrentBag<string> entitiesToUpdate = new();
try
{
using var ctx = new FileCacheContext();
@@ -200,14 +218,19 @@ public class PeriodicFileScanner : IDisposable
{
try
{
var file = _fileDbManager.ValidateFileCacheEntity(hash, filePath, date);
if (file != null)
var fileState = _fileDbManager.ValidateFileCacheEntity(hash, filePath, date);
switch (fileState.Item1)
{
scannedFiles[file.Filepath] = true;
}
else
{
entitiesToRemove.Add(new Tuple<string, string>(hash, filePath));
case FileState.Valid:
scannedFiles[fileState.Item2] = true;
break;
case FileState.RequireDeletion:
entitiesToRemove.Add(new Tuple<string, string>(hash, filePath));
break;
case FileState.RequireUpdate:
scannedFiles[fileState.Item2] = true;
entitiesToUpdate.Add(fileState.Item2);
break;
}
}
catch (Exception ex)
@@ -225,6 +248,10 @@ public class PeriodicFileScanner : IDisposable
if (ct.IsCancellationRequested) return;
}
}
catch (OperationCanceledException)
{
return;
}
catch (Exception ex)
{
Logger.Warn("Error during enumerating FileCaches: " + ex.Message);
@@ -243,8 +270,22 @@ public class PeriodicFileScanner : IDisposable
db.FileCaches.Remove(toRemove);
}
}
if (entitiesToUpdate.Any())
{
foreach (var entry in entitiesToUpdate)
{
Logger.Debug("Updating " + entry);
_fileDbManager.GetFileCacheByPath(entry);
}
}
if (entitiesToUpdate.Any() || entitiesToRemove.Any())
{
db.SaveChanges();
}
}
catch (Exception ex)
{