add file compactor
This commit is contained in:
rootdarkarchon
212
MareSynchronos/FileCache/FileCompactor.cs
Normal file
212
MareSynchronos/FileCache/FileCompactor.cs
Normal file
@@ -0,0 +1,212 @@
|
||||
using MareSynchronos.MareConfiguration;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace MareSynchronos.FileCache;
|
||||
|
||||
public sealed class FileCompactor
|
||||
{
|
||||
public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U;
|
||||
public const ulong WOF_PROVIDER_FILE = 2UL;
|
||||
|
||||
private readonly Dictionary<string, int> _clusterSizes;
|
||||
|
||||
private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo;
|
||||
private readonly ILogger<FileCompactor> _logger;
|
||||
|
||||
private readonly MareConfigService _mareConfigService;
|
||||
|
||||
public FileCompactor(ILogger<FileCompactor> logger, MareConfigService mareConfigService)
|
||||
{
|
||||
_clusterSizes = new(StringComparer.Ordinal);
|
||||
_logger = logger;
|
||||
_mareConfigService = mareConfigService;
|
||||
_efInfo = new WOF_FILE_COMPRESSION_INFO_V1
|
||||
{
|
||||
Algorithm = CompressionAlgorithm.XPRESS8K,
|
||||
Flags = 0
|
||||
};
|
||||
}
|
||||
|
||||
private enum CompressionAlgorithm
|
||||
{
|
||||
NO_COMPRESSION = -2,
|
||||
LZNT1 = -1,
|
||||
XPRESS4K = 0,
|
||||
LZX = 1,
|
||||
XPRESS8K = 2,
|
||||
XPRESS16K = 3
|
||||
}
|
||||
|
||||
public bool MassCompactRunning { get; private set; } = false;
|
||||
|
||||
public string Progress { get; private set; } = string.Empty;
|
||||
|
||||
public void CompactStorage(bool compress)
|
||||
{
|
||||
MassCompactRunning = true;
|
||||
|
||||
int currentFile = 1;
|
||||
var allFiles = Directory.EnumerateFiles(_mareConfigService.Current.CacheFolder).ToList();
|
||||
int allFilesCount = allFiles.Count;
|
||||
foreach (var file in allFiles)
|
||||
{
|
||||
Progress = $"{currentFile}/{allFilesCount}";
|
||||
if (compress)
|
||||
CompactFile(file);
|
||||
else
|
||||
DecompressFile(file);
|
||||
currentFile++;
|
||||
}
|
||||
|
||||
MassCompactRunning = false;
|
||||
}
|
||||
|
||||
public long GetFileSizeOnDisk(string filePath)
|
||||
{
|
||||
if (Dalamud.Utility.Util.IsLinux()) return new FileInfo(filePath).Length;
|
||||
|
||||
var clusterSize = GetClusterSize(filePath);
|
||||
if (clusterSize == -1) return new FileInfo(filePath).Length;
|
||||
var losize = GetCompressedFileSizeW(filePath, out uint hosize);
|
||||
var size = (long)hosize << 32 | losize;
|
||||
return ((size + clusterSize - 1) / clusterSize) * clusterSize;
|
||||
}
|
||||
|
||||
public async Task WriteAllBytesAsync(string filePath, byte[] decompressedFile, CancellationToken token)
|
||||
{
|
||||
await File.WriteAllBytesAsync(filePath, decompressedFile, token);
|
||||
|
||||
if (Dalamud.Utility.Util.IsLinux() || !_mareConfigService.Current.UseCompactor)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
CompactFile(filePath);
|
||||
}
|
||||
|
||||
[DllImport("kernel32.dll")]
|
||||
private static extern int DeviceIoControl(IntPtr hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out IntPtr lpBytesReturned, out IntPtr lpOverlapped);
|
||||
|
||||
[DllImport("kernel32.dll")]
|
||||
private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName,
|
||||
[Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)]
|
||||
private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName,
|
||||
out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters,
|
||||
out uint lpTotalNumberOfClusters);
|
||||
|
||||
[DllImport("WoFUtil.dll")]
|
||||
private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength);
|
||||
|
||||
[DllImport("WofUtil.dll")]
|
||||
private static extern int WofSetFileDataLocation(IntPtr FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length);
|
||||
|
||||
private void CompactFile(string filePath)
|
||||
{
|
||||
var oldSize = new FileInfo(filePath).Length;
|
||||
var clusterSize = GetClusterSize(filePath);
|
||||
|
||||
if (oldSize < Math.Max(clusterSize, 8 * 1024))
|
||||
{
|
||||
_logger.LogDebug("File {file} is smaller than cluster size ({size}), ignoring", filePath, clusterSize);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!IsCompactedFile(filePath))
|
||||
{
|
||||
_logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath);
|
||||
|
||||
WOFCompressFile(filePath);
|
||||
|
||||
var newSize = GetFileSizeOnDisk(filePath);
|
||||
|
||||
_logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("File {file} already compressed", filePath);
|
||||
}
|
||||
}
|
||||
|
||||
private void DecompressFile(string path)
|
||||
{
|
||||
_logger.LogDebug("Removing compression from {file}", path);
|
||||
try
|
||||
{
|
||||
using (var fs = new FileStream(path, FileMode.Open))
|
||||
{
|
||||
var hDevice = fs.SafeFileHandle.DangerousGetHandle();
|
||||
var ret = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error decompressing file {path}", path);
|
||||
}
|
||||
}
|
||||
|
||||
private int GetClusterSize(string filePath)
|
||||
{
|
||||
FileInfo fi = new(filePath);
|
||||
if (!fi.Exists) return -1;
|
||||
var root = fi.Directory?.Root.FullName.ToLower() ?? string.Empty;
|
||||
if (string.IsNullOrEmpty(root)) return -1;
|
||||
if (_clusterSizes.ContainsKey(root)) return _clusterSizes[root];
|
||||
_logger.LogDebug("Getting Cluster Size for {path}, root {root}", filePath, root);
|
||||
int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _);
|
||||
if (result == 0) return -1;
|
||||
_clusterSizes[root] = (int)(sectorsPerCluster * bytesPerSector);
|
||||
_logger.LogDebug("Determined Cluster Size for root {root}: {cluster}", root, _clusterSizes[root]);
|
||||
return _clusterSizes[root];
|
||||
}
|
||||
|
||||
private bool IsCompactedFile(string filePath)
|
||||
{
|
||||
uint buf = 8;
|
||||
_ = WofIsExternalFile(filePath, out int isExtFile, out uint _, out var info, ref buf);
|
||||
if (isExtFile == 0) return false;
|
||||
return info.Algorithm == CompressionAlgorithm.XPRESS8K;
|
||||
}
|
||||
|
||||
private void WOFCompressFile(string path)
|
||||
{
|
||||
var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo));
|
||||
Marshal.StructureToPtr(_efInfo, efInfoPtr, true);
|
||||
ulong length = (ulong)Marshal.SizeOf(_efInfo);
|
||||
try
|
||||
{
|
||||
using (var fs = new FileStream(path, FileMode.Open))
|
||||
{
|
||||
var hFile = fs.SafeFileHandle.DangerousGetHandle();
|
||||
if (fs.SafeFileHandle.IsInvalid)
|
||||
{
|
||||
_logger.LogWarning("Invalid file handle to {file}", path);
|
||||
}
|
||||
else
|
||||
{
|
||||
var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length);
|
||||
if (!(ret == 0 || ret == unchecked((int)0x80070158)))
|
||||
{
|
||||
_logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error compacting file {path}", path);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Marshal.FreeHGlobal(efInfoPtr);
|
||||
}
|
||||
}
|
||||
|
||||
private struct WOF_FILE_COMPRESSION_INFO_V1
|
||||
{
|
||||
public CompressionAlgorithm Algorithm;
|
||||
public ulong Flags;
|
||||
}
|
||||
}
|
||||
@@ -14,19 +14,22 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
|
||||
private readonly IpcManager _ipcManager;
|
||||
private readonly PerformanceCollectorService _performanceCollector;
|
||||
private readonly DalamudUtilService _dalamudUtil;
|
||||
private readonly FileCompactor _fileCompactor;
|
||||
private long _currentFileProgress = 0;
|
||||
private bool _fileScanWasRunning = false;
|
||||
private CancellationTokenSource _scanCancellationTokenSource = new();
|
||||
private TimeSpan _timeUntilNextScan = TimeSpan.Zero;
|
||||
|
||||
public PeriodicFileScanner(ILogger<PeriodicFileScanner> logger, IpcManager ipcManager, MareConfigService configService,
|
||||
FileCacheManager fileDbManager, MareMediator mediator, PerformanceCollectorService performanceCollector, DalamudUtilService dalamudUtil) : base(logger, mediator)
|
||||
FileCacheManager fileDbManager, MareMediator mediator, PerformanceCollectorService performanceCollector, DalamudUtilService dalamudUtil,
|
||||
FileCompactor fileCompactor) : base(logger, mediator)
|
||||
{
|
||||
_ipcManager = ipcManager;
|
||||
_configService = configService;
|
||||
_fileDbManager = fileDbManager;
|
||||
_performanceCollector = performanceCollector;
|
||||
_dalamudUtil = dalamudUtil;
|
||||
_fileCompactor = fileCompactor;
|
||||
Mediator.Subscribe<PenumbraInitializedMessage>(this, (_) => StartScan());
|
||||
Mediator.Subscribe<HaltScanMessage>(this, (msg) => HaltScan(msg.Source));
|
||||
Mediator.Subscribe<ResumeScanMessage>(this, (msg) => ResumeScan(msg.Source));
|
||||
@@ -108,7 +111,7 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
|
||||
{
|
||||
try
|
||||
{
|
||||
return new FileInfo(f).Length;
|
||||
return _fileCompactor.GetFileSizeOnDisk(f);
|
||||
}
|
||||
catch
|
||||
{
|
||||
@@ -126,7 +129,7 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
|
||||
while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer)
|
||||
{
|
||||
var oldestFile = allFiles[0];
|
||||
FileCacheSize -= oldestFile.Length;
|
||||
FileCacheSize -= _fileCompactor.GetFileSizeOnDisk(oldestFile.FullName);
|
||||
File.Delete(oldestFile.FullName);
|
||||
allFiles.Remove(oldestFile);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user