t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add file compactor

Browse Source
This commit is contained in:
rootdarkarchon
2023-09-14 01:17:49 +02:00
parent 9139ddae12
commit 0008428cb0
8 changed files with 285 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
MareSynchronos/FileCache/FileCompactor.cs Normal file
View File

@@ -0,0 +1,212 @@
using MareSynchronos.MareConfiguration;
using Microsoft.Extensions.Logging;
using System.Runtime.InteropServices;
namespace MareSynchronos.FileCache;
public sealed class FileCompactor
{
public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U;
public const ulong WOF_PROVIDER_FILE = 2UL;
private readonly Dictionary<string, int> _clusterSizes;
private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo;
private readonly ILogger<FileCompactor> _logger;
private readonly MareConfigService _mareConfigService;
public FileCompactor(ILogger<FileCompactor> logger, MareConfigService mareConfigService)
{
_clusterSizes = new(StringComparer.Ordinal);
_logger = logger;
_mareConfigService = mareConfigService;
_efInfo = new WOF_FILE_COMPRESSION_INFO_V1
{
Algorithm = CompressionAlgorithm.XPRESS8K,
Flags = 0
};
}
private enum CompressionAlgorithm
{
NO_COMPRESSION = -2,
LZNT1 = -1,
XPRESS4K = 0,
LZX = 1,
XPRESS8K = 2,
XPRESS16K = 3
}
public bool MassCompactRunning { get; private set; } = false;
public string Progress { get; private set; } = string.Empty;
public void CompactStorage(bool compress)
{
MassCompactRunning = true;
int currentFile = 1;
var allFiles = Directory.EnumerateFiles(_mareConfigService.Current.CacheFolder).ToList();
int allFilesCount = allFiles.Count;
foreach (var file in allFiles)
{
Progress = $"{currentFile}/{allFilesCount}";
if (compress)
CompactFile(file);
else
DecompressFile(file);
currentFile++;
}
MassCompactRunning = false;
}
public long GetFileSizeOnDisk(string filePath)
{
if (Dalamud.Utility.Util.IsLinux()) return new FileInfo(filePath).Length;
var clusterSize = GetClusterSize(filePath);
if (clusterSize == -1) return new FileInfo(filePath).Length;
var losize = GetCompressedFileSizeW(filePath, out uint hosize);
var size = (long)hosize << 32 | losize;
return ((size + clusterSize - 1) / clusterSize) * clusterSize;
}
public async Task WriteAllBytesAsync(string filePath, byte[] decompressedFile, CancellationToken token)
{
await File.WriteAllBytesAsync(filePath, decompressedFile, token);
if (Dalamud.Utility.Util.IsLinux() || !_mareConfigService.Current.UseCompactor)
{
return;
}
CompactFile(filePath);
}
[DllImport("kernel32.dll")]
private static extern int DeviceIoControl(IntPtr hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out IntPtr lpBytesReturned, out IntPtr lpOverlapped);
[DllImport("kernel32.dll")]
private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName,
[Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh);
[DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)]
private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName,
out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters,
out uint lpTotalNumberOfClusters);
[DllImport("WoFUtil.dll")]
private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength);
[DllImport("WofUtil.dll")]
private static extern int WofSetFileDataLocation(IntPtr FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length);
private void CompactFile(string filePath)
{
var oldSize = new FileInfo(filePath).Length;
var clusterSize = GetClusterSize(filePath);
if (oldSize < Math.Max(clusterSize, 8 * 1024))
{
_logger.LogDebug("File {file} is smaller than cluster size ({size}), ignoring", filePath, clusterSize);
return;
}
if (!IsCompactedFile(filePath))
{
_logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath);
WOFCompressFile(filePath);
var newSize = GetFileSizeOnDisk(filePath);
_logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize);
}
else
{
_logger.LogDebug("File {file} already compressed", filePath);
}
}
private void DecompressFile(string path)
{
_logger.LogDebug("Removing compression from {file}", path);
try
{
using (var fs = new FileStream(path, FileMode.Open))
{
var hDevice = fs.SafeFileHandle.DangerousGetHandle();
var ret = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error decompressing file {path}", path);
}
}
private int GetClusterSize(string filePath)
{
FileInfo fi = new(filePath);
if (!fi.Exists) return -1;
var root = fi.Directory?.Root.FullName.ToLower() ?? string.Empty;
if (string.IsNullOrEmpty(root)) return -1;
if (_clusterSizes.ContainsKey(root)) return _clusterSizes[root];
_logger.LogDebug("Getting Cluster Size for {path}, root {root}", filePath, root);
int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _);
if (result == 0) return -1;
_clusterSizes[root] = (int)(sectorsPerCluster * bytesPerSector);
_logger.LogDebug("Determined Cluster Size for root {root}: {cluster}", root, _clusterSizes[root]);
return _clusterSizes[root];
}
private bool IsCompactedFile(string filePath)
{
uint buf = 8;
_ = WofIsExternalFile(filePath, out int isExtFile, out uint _, out var info, ref buf);
if (isExtFile == 0) return false;
return info.Algorithm == CompressionAlgorithm.XPRESS8K;
}
private void WOFCompressFile(string path)
{
var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo));
Marshal.StructureToPtr(_efInfo, efInfoPtr, true);
ulong length = (ulong)Marshal.SizeOf(_efInfo);
try
{
using (var fs = new FileStream(path, FileMode.Open))
{
var hFile = fs.SafeFileHandle.DangerousGetHandle();
if (fs.SafeFileHandle.IsInvalid)
{
_logger.LogWarning("Invalid file handle to {file}", path);
}
else
{
var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length);
if (!(ret == 0 || ret == unchecked((int)0x80070158)))
{
_logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X"));
}
}
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error compacting file {path}", path);
}
finally
{
Marshal.FreeHGlobal(efInfoPtr);
}
}
private struct WOF_FILE_COMPRESSION_INFO_V1
{
public CompressionAlgorithm Algorithm;
public ulong Flags;
}
}

9
MareSynchronos/FileCache/PeriodicFileScanner.cs
View File

@@ -14,19 +14,22 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
private readonly IpcManager _ipcManager;
private readonly PerformanceCollectorService _performanceCollector;
private readonly DalamudUtilService _dalamudUtil;
private readonly FileCompactor _fileCompactor;
private long _currentFileProgress = 0;
private bool _fileScanWasRunning = false;
private CancellationTokenSource _scanCancellationTokenSource = new();
private TimeSpan _timeUntilNextScan = TimeSpan.Zero;
public PeriodicFileScanner(ILogger<PeriodicFileScanner> logger, IpcManager ipcManager, MareConfigService configService,
FileCacheManager fileDbManager, MareMediator mediator, PerformanceCollectorService performanceCollector, DalamudUtilService dalamudUtil) : base(logger, mediator)
FileCacheManager fileDbManager, MareMediator mediator, PerformanceCollectorService performanceCollector, DalamudUtilService dalamudUtil,
FileCompactor fileCompactor) : base(logger, mediator)
{
_ipcManager = ipcManager;
_configService = configService;
_fileDbManager = fileDbManager;
_performanceCollector = performanceCollector;
_dalamudUtil = dalamudUtil;
_fileCompactor = fileCompactor;
Mediator.Subscribe<PenumbraInitializedMessage>(this, (_) => StartScan());
Mediator.Subscribe<HaltScanMessage>(this, (msg) => HaltScan(msg.Source));
Mediator.Subscribe<ResumeScanMessage>(this, (msg) => ResumeScan(msg.Source));
@@ -108,7 +111,7 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
{
try
{
return new FileInfo(f).Length;
return _fileCompactor.GetFileSizeOnDisk(f);
}
catch
{
@@ -126,7 +129,7 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer)
{
var oldestFile = allFiles[0];
FileCacheSize -= oldestFile.Length;
FileCacheSize -= _fileCompactor.GetFileSizeOnDisk(oldestFile.FullName);
File.Delete(oldestFile.FullName);
allFiles.Remove(oldestFile);
}

1
MareSynchronos/MareConfiguration/Configurations/MareConfig.cs
View File

@@ -22,6 +22,7 @@ public class MareConfig : IMareConfiguration
public bool OpenGposeImportOnGposeStart { get; set; } = false;
public bool OpenPopupOnAdd { get; set; } = true;
public int ParallelDownloads { get; set; } = 10;
public bool UseCompactor { get; set; } = false;
public float ProfileDelay { get; set; } = 1.5f;
public bool ProfilePopoutRight { get; set; } = false;
public bool ProfilesAllowNsfw { get; set; } = false;

6
MareSynchronos/PlayerData/Factories/FileDownloadManagerFactory.cs
View File

@@ -8,21 +8,23 @@ namespace MareSynchronos.PlayerData.Factories;
public class FileDownloadManagerFactory
{
private readonly FileCacheManager _fileCacheManager;
private readonly FileCompactor _fileCompactor;
private readonly FileTransferOrchestrator _fileTransferOrchestrator;
private readonly ILoggerFactory _loggerFactory;
private readonly MareMediator _mareMediator;
public FileDownloadManagerFactory(ILoggerFactory loggerFactory, MareMediator mareMediator, FileTransferOrchestrator fileTransferOrchestrator,
FileCacheManager fileCacheManager)
FileCacheManager fileCacheManager, FileCompactor fileCompactor)
{
_loggerFactory = loggerFactory;
_mareMediator = mareMediator;
_fileTransferOrchestrator = fileTransferOrchestrator;
_fileCacheManager = fileCacheManager;
_fileCompactor = fileCompactor;
}
public FileDownloadManager Create()
{
return new FileDownloadManager(_loggerFactory.CreateLogger<FileDownloadManager>(), _mareMediator, _fileTransferOrchestrator, _fileCacheManager);
return new FileDownloadManager(_loggerFactory.CreateLogger<FileDownloadManager>(), _mareMediator, _fileTransferOrchestrator, _fileCacheManager, _fileCompactor);
}
}

1
MareSynchronos/Plugin.cs
View File

@@ -74,6 +74,7 @@ public sealed class Plugin : IDalamudPlugin
collection.AddSingleton<PairFactory>();
collection.AddSingleton<CharacterAnalyzer>();
collection.AddSingleton<PluginWarningNotificationService>();
collection.AddSingleton<FileCompactor>();
collection.AddSingleton((s) => new DalamudUtilService(s.GetRequiredService<ILogger<DalamudUtilService>>(),
clientState, objectTable, framework, gameGui, condition, gameData,
s.GetRequiredService<MareMediator>(), s.GetRequiredService<PerformanceCollectorService>()));

17
MareSynchronos/UI/IntroUI.cs
View File

@@ -47,7 +47,11 @@ public class IntroUi : WindowMediatorSubscriberBase
GetToSLocalization();
Mediator.Subscribe<SwitchToMainUiMessage>(this, (_) => IsOpen = false);
Mediator.Subscribe<SwitchToIntroUiMessage>(this, (_) => IsOpen = true);
Mediator.Subscribe<SwitchToIntroUiMessage>(this, (_) =>
{
_configService.Current.UseCompactor = !Util.IsLinux();
IsOpen = true;
});
}
public override void Draw()
@@ -170,6 +174,17 @@ public class IntroUi : WindowMediatorSubscriberBase
{
_uiShared.DrawFileScanState();
}
if (!Util.IsLinux())
{
var useFileCompactor = _configService.Current.UseCompactor;
if (ImGui.Checkbox("Use File Compactor", ref useFileCompactor))
{
_configService.Current.UseCompactor = useFileCompactor;
_configService.Save();
}
UiSharedService.ColorTextWrapped("The File Compactor can save a tremendeous amount of space on the hard disk for downloads through Mare. It will incur a minor CPU penalty on download but can speed up " +
"loading of other characters. It is recommended to keep it enabled. You can change this setting later anytime in the Mare settings.", ImGuiColors.DalamudYellow);
}
}
else if (!_uiShared.ApiController.ServerAlive)
{

42
MareSynchronos/UI/SettingsUi.cs
View File

@@ -20,6 +20,7 @@ using MareSynchronos.WebAPI.Files;
using MareSynchronos.WebAPI.Files.Models;
using MareSynchronos.PlayerData.Handlers;
using System.Collections.Concurrent;
using MareSynchronos.FileCache;
namespace MareSynchronos.UI;
@@ -29,6 +30,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
private readonly ConcurrentDictionary<GameObjectHandler, Dictionary<string, FileDownloadStatus>> _currentDownloads = new();
private readonly FileUploadManager _fileTransferManager;
private readonly FileTransferOrchestrator _fileTransferOrchestrator;
private readonly FileCompactor _fileCompactor;
private readonly MareCharaFileManager _mareCharaFileManager;
private readonly PairManager _pairManager;
private readonly PerformanceCollectorService _performanceCollector;
@@ -50,7 +52,8 @@ public class SettingsUi : WindowMediatorSubscriberBase
ServerConfigurationManager serverConfigurationManager,
MareMediator mediator, PerformanceCollectorService performanceCollector,
FileUploadManager fileTransferManager,
FileTransferOrchestrator fileTransferOrchestrator) : base(logger, mediator, "Mare Synchronos Settings")
FileTransferOrchestrator fileTransferOrchestrator,
FileCompactor fileCompactor) : base(logger, mediator, "Mare Synchronos Settings")
{
_configService = configService;
_mareCharaFileManager = mareCharaFileManager;
@@ -59,6 +62,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
_performanceCollector = performanceCollector;
_fileTransferManager = fileTransferManager;
_fileTransferOrchestrator = fileTransferOrchestrator;
_fileCompactor = fileCompactor;
_uiShared = uiShared;
SizeConstraints = new WindowSizeConstraints()
@@ -433,6 +437,42 @@ public class SettingsUi : WindowMediatorSubscriberBase
_uiShared.DrawTimeSpanBetweenScansSetting();
_uiShared.DrawCacheDirectorySetting();
ImGui.Text($"Currently utilized local storage: {UiSharedService.ByteToString(_uiShared.FileCacheSize)}");
bool isLinux = Util.IsLinux();
if (isLinux) ImGui.BeginDisabled();
bool useFileCompactor = _configService.Current.UseCompactor;
if (ImGui.Checkbox("Use file compactor", ref useFileCompactor))
{
_configService.Current.UseCompactor = useFileCompactor;
_configService.Save();
}
UiSharedService.DrawHelpText("The file compactor can massively reduce your saved files. It might incur a minor penalty on loading files on a slow CPU." + Environment.NewLine
+ "It is recommended to leave it enabled to save on space.");
ImGui.SameLine();
if (!_fileCompactor.MassCompactRunning)
{
if (UiSharedService.IconTextButton(FontAwesomeIcon.FileArchive, "Compact all files in storage"))
{
_ = Task.Run(() => _fileCompactor.CompactStorage(true));
}
UiSharedService.AttachToolTip("This will run compression on all files in your current Mare Storage." + Environment.NewLine
+ "You do not need to run this manually if you keep the file compactor enabled.");
ImGui.SameLine();
if (UiSharedService.IconTextButton(FontAwesomeIcon.File, "Decompact all files in storage"))
{
_ = Task.Run(() => _fileCompactor.CompactStorage(false));
}
UiSharedService.AttachToolTip("This will run decompression on all files in your current Mare Storage.");
}
else
{
UiSharedService.ColorText($"File compactor currently running ({_fileCompactor.Progress})", ImGuiColors.DalamudYellow);
}
if (isLinux)
{
ImGui.EndDisabled();
ImGui.Text("The file compactor is only available on Windows.");
}
ImGui.Dummy(new Vector2(10, 10));
ImGui.Text("To clear the local storage accept the following disclaimer");
ImGui.Indent();

6
MareSynchronos/WebAPI/Files/FileDownloadManager.cs
View File

@@ -17,15 +17,17 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
{
private readonly Dictionary<string, FileDownloadStatus> _downloadStatus;
private readonly FileCacheManager _fileDbManager;
private readonly FileCompactor _fileCompactor;
private readonly FileTransferOrchestrator _orchestrator;
public FileDownloadManager(ILogger<FileDownloadManager> logger, MareMediator mediator,
FileTransferOrchestrator orchestrator,
FileCacheManager fileCacheManager) : base(logger, mediator)
FileCacheManager fileCacheManager, FileCompactor fileCompactor) : base(logger, mediator)
{
_downloadStatus = new Dictionary<string, FileDownloadStatus>(StringComparer.Ordinal);
_orchestrator = orchestrator;
_fileDbManager = fileCacheManager;
_fileCompactor = fileCompactor;
}
public List<DownloadFileTransfer> CurrentDownloads { get; private set; } = new();
@@ -280,7 +282,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
var decompressedFile = LZ4Codec.Unwrap(compressedFileContent);
var filePath = _fileDbManager.GetCacheFilePath(fileHash, fileExtension);
await File.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false);
await _fileCompactor.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false);
PersistFileToStorage(fileHash, filePath);
}