t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make storage size calculation asynchronous and running in parallel

Browse Source
This commit is contained in:
rootdarkarchon
2024-02-15 02:38:41 +01:00
committed by Loporrit
parent 50990542fd
commit e1ca5dd6f8
9 changed files with 105 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
MareSynchronos/FileCache/CacheMonitor.cs
View File

@@ -18,6 +18,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
private readonly PerformanceCollectorService _performanceCollector;
private long _currentFileProgress = 0;
private CancellationTokenSource _scanCancellationTokenSource = new();
private readonly CancellationTokenSource _periodicCalculationTokenSource = new();
private readonly string[] _allowedExtensions = [".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".pbd", ".scd", ".skp", ".shpk"];
public CacheMonitor(ILogger<CacheMonitor> logger, IpcManager ipcManager, MareConfigService configService,
@@ -57,6 +58,25 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
{
StartMareWatcher(configService.Current.CacheFolder);
}
var token = _periodicCalculationTokenSource.Token;
_ = Task.Run(async () =>
{
Logger.LogInformation("Starting Periodic Storage Directory Calculation Task");
var token = _periodicCalculationTokenSource.Token;
while (!token.IsCancellationRequested)
{
try
{
RecalculateFileCacheSize(token);
}
catch
{
// ignore
}
await Task.Delay(TimeSpan.FromMinutes(1), token).ConfigureAwait(false);
}
}, token);
}
public long CurrentFileProgress => _currentFileProgress;
@@ -86,17 +106,21 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
PenumbraWatcher = null;
}
public bool StorageisNTFS { get; private set; } = false;
public void StartMareWatcher(string? marePath)
{
MareWatcher?.Dispose();
if (string.IsNullOrEmpty(marePath))
if (string.IsNullOrEmpty(marePath) || !Directory.Exists(marePath))
{
MareWatcher = null;
Logger.LogWarning("Mare file path is not set, cannot start the FSW for Mare.");
return;
}
RecalculateFileCacheSize();
DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
StorageisNTFS = string.Equals("NTFS", di.DriveFormat, StringComparison.OrdinalIgnoreCase);
Logger.LogInformation("Mare Storage is on NTFS drive: {isNtfs}", StorageisNTFS);
Logger.LogDebug("Initializing Mare FSW on {path}", marePath);
MareWatcher = new()
@@ -248,9 +272,6 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
}
}
_ = RecalculateFileCacheSize();
if (changes.Any(c => c.Value.ChangeType == WatcherChangeTypes.Deleted))
{
lock (_fileDbManager)
@@ -356,26 +377,43 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
}, token);
}
public bool RecalculateFileCacheSize()
public void RecalculateFileCacheSize(CancellationToken token)
{
FileCacheSize = Directory.EnumerateFiles(_configService.Current.CacheFolder).Sum(f =>
if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder))
{
try
{
return _fileCompactor.GetFileSizeOnDisk(f);
}
catch
{
return 0;
}
});
FileCacheSize = 0;
return;
}
DriveInfo di = new DriveInfo(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
FileCacheDriveFree = di.AvailableFreeSpace;
FileCacheSize = -1;
DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
try
{
FileCacheDriveFree = di.AvailableFreeSpace;
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder);
}
FileCacheSize = Directory.EnumerateFiles(_configService.Current.CacheFolder)
.AsParallel().Sum(f =>
{
token.ThrowIfCancellationRequested();
try
{
return _fileCompactor.GetFileSizeOnDisk(f, StorageisNTFS);
}
catch
{
return 0;
}
});
var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d);
if (FileCacheSize < maxCacheInBytes) return false;
if (FileCacheSize < maxCacheInBytes) return;
var allFiles = Directory.EnumerateFiles(_configService.Current.CacheFolder)
.Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList();
@@ -387,8 +425,6 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
File.Delete(oldestFile.FullName);
allFiles.Remove(oldestFile);
}
return true;
}
public void ResetLocks()
@@ -412,6 +448,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
MareWatcher?.Dispose();
_penumbraFswCts?.CancelDispose();
_mareFswCts?.CancelDispose();
_periodicCalculationTokenSource?.CancelDispose();
}
private void FullFileScan(CancellationToken ct)

12
MareSynchronos/FileCache/FileCacheManager.cs
View File

@@ -367,12 +367,17 @@ public sealed class FileCacheManager : IHostedService
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Starting FileCacheManager");
lock (_fileWriteLock)
{
try
{
_logger.LogInformation("Checking for {bakPath}", CsvBakPath);
if (File.Exists(CsvBakPath))
{
_logger.LogInformation("{bakPath} found, moving to {csvPath}", CsvBakPath, _csvPath);
File.Move(CsvBakPath, _csvPath, overwrite: true);
}
}
@@ -393,6 +398,8 @@ public sealed class FileCacheManager : IHostedService
if (File.Exists(_csvPath))
{
_logger.LogInformation("{csvPath} found, parsing", _csvPath);
bool success = false;
string[] entries = [];
int attempts = 0;
@@ -400,6 +407,7 @@ public sealed class FileCacheManager : IHostedService
{
try
{
_logger.LogInformation("Attempting to read {csvPath}", _csvPath);
entries = File.ReadAllLines(_csvPath);
success = true;
}
@@ -416,6 +424,8 @@ public sealed class FileCacheManager : IHostedService
_logger.LogWarning("Could not load entries from {path}, continuing with empty file cache", _csvPath);
}
_logger.LogInformation("Found {amount} files in {path}", entries.Length, _csvPath);
Dictionary<string, bool> processedFiles = new(StringComparer.OrdinalIgnoreCase);
foreach (var entry in entries)
{
@@ -462,6 +472,8 @@ public sealed class FileCacheManager : IHostedService
}
}
_logger.LogInformation("Started FileCacheManager");
return Task.CompletedTask;
}

14
MareSynchronos/FileCache/FileCompactor.cs
View File

@@ -62,9 +62,11 @@ public sealed class FileCompactor
MassCompactRunning = false;
}
public long GetFileSizeOnDisk(string filePath)
public long GetFileSizeOnDisk(string filePath, bool? isNTFS = null)
{
if (Dalamud.Utility.Util.IsWine()) return new FileInfo(filePath).Length;
bool ntfs = isNTFS ?? string.Equals(new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName).DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase);
if (Dalamud.Utility.Util.IsWine() || !ntfs) return new FileInfo(filePath).Length;
var clusterSize = GetClusterSize(filePath);
if (clusterSize == -1) return new FileInfo(filePath).Length;
@@ -105,6 +107,14 @@ public sealed class FileCompactor
private void CompactFile(string filePath)
{
var fs = new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName);
bool isNTFS = string.Equals(fs.DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase);
if (!isNTFS)
{
_logger.LogWarning("Drive for file {file} is not NTFS", filePath);
return;
}
var oldSize = new FileInfo(filePath).Length;
var clusterSize = GetClusterSize(filePath);