diff --git a/MareSynchronos/FileCache/FileCompactor.cs b/MareSynchronos/FileCache/FileCompactor.cs new file mode 100644 index 0000000..8c3cd05 --- /dev/null +++ b/MareSynchronos/FileCache/FileCompactor.cs @@ -0,0 +1,212 @@ +using MareSynchronos.MareConfiguration; +using Microsoft.Extensions.Logging; +using System.Runtime.InteropServices; + +namespace MareSynchronos.FileCache; + +public sealed class FileCompactor +{ + public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U; + public const ulong WOF_PROVIDER_FILE = 2UL; + + private readonly Dictionary _clusterSizes; + + private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo; + private readonly ILogger _logger; + + private readonly MareConfigService _mareConfigService; + + public FileCompactor(ILogger logger, MareConfigService mareConfigService) + { + _clusterSizes = new(StringComparer.Ordinal); + _logger = logger; + _mareConfigService = mareConfigService; + _efInfo = new WOF_FILE_COMPRESSION_INFO_V1 + { + Algorithm = CompressionAlgorithm.XPRESS8K, + Flags = 0 + }; + } + + private enum CompressionAlgorithm + { + NO_COMPRESSION = -2, + LZNT1 = -1, + XPRESS4K = 0, + LZX = 1, + XPRESS8K = 2, + XPRESS16K = 3 + } + + public bool MassCompactRunning { get; private set; } = false; + + public string Progress { get; private set; } = string.Empty; + + public void CompactStorage(bool compress) + { + MassCompactRunning = true; + + int currentFile = 1; + var allFiles = Directory.EnumerateFiles(_mareConfigService.Current.CacheFolder).ToList(); + int allFilesCount = allFiles.Count; + foreach (var file in allFiles) + { + Progress = $"{currentFile}/{allFilesCount}"; + if (compress) + CompactFile(file); + else + DecompressFile(file); + currentFile++; + } + + MassCompactRunning = false; + } + + public long GetFileSizeOnDisk(string filePath) + { + if (Dalamud.Utility.Util.IsLinux()) return new FileInfo(filePath).Length; + + var clusterSize = GetClusterSize(filePath); + if (clusterSize == -1) return new FileInfo(filePath).Length; + var losize = GetCompressedFileSizeW(filePath, out uint hosize); + var size = (long)hosize << 32 | losize; + return ((size + clusterSize - 1) / clusterSize) * clusterSize; + } + + public async Task WriteAllBytesAsync(string filePath, byte[] decompressedFile, CancellationToken token) + { + await File.WriteAllBytesAsync(filePath, decompressedFile, token); + + if (Dalamud.Utility.Util.IsLinux() || !_mareConfigService.Current.UseCompactor) + { + return; + } + + CompactFile(filePath); + } + + [DllImport("kernel32.dll")] + private static extern int DeviceIoControl(IntPtr hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out IntPtr lpBytesReturned, out IntPtr lpOverlapped); + + [DllImport("kernel32.dll")] + private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, + [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh); + + [DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)] + private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, + out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters, + out uint lpTotalNumberOfClusters); + + [DllImport("WoFUtil.dll")] + private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength); + + [DllImport("WofUtil.dll")] + private static extern int WofSetFileDataLocation(IntPtr FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); + + private void CompactFile(string filePath) + { + var oldSize = new FileInfo(filePath).Length; + var clusterSize = GetClusterSize(filePath); + + if (oldSize < Math.Max(clusterSize, 8 * 1024)) + { + _logger.LogDebug("File {file} is smaller than cluster size ({size}), ignoring", filePath, clusterSize); + return; + } + + if (!IsCompactedFile(filePath)) + { + _logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath); + + WOFCompressFile(filePath); + + var newSize = GetFileSizeOnDisk(filePath); + + _logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); + } + else + { + _logger.LogDebug("File {file} already compressed", filePath); + } + } + + private void DecompressFile(string path) + { + _logger.LogDebug("Removing compression from {file}", path); + try + { + using (var fs = new FileStream(path, FileMode.Open)) + { + var hDevice = fs.SafeFileHandle.DangerousGetHandle(); + var ret = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error decompressing file {path}", path); + } + } + + private int GetClusterSize(string filePath) + { + FileInfo fi = new(filePath); + if (!fi.Exists) return -1; + var root = fi.Directory?.Root.FullName.ToLower() ?? string.Empty; + if (string.IsNullOrEmpty(root)) return -1; + if (_clusterSizes.ContainsKey(root)) return _clusterSizes[root]; + _logger.LogDebug("Getting Cluster Size for {path}, root {root}", filePath, root); + int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _); + if (result == 0) return -1; + _clusterSizes[root] = (int)(sectorsPerCluster * bytesPerSector); + _logger.LogDebug("Determined Cluster Size for root {root}: {cluster}", root, _clusterSizes[root]); + return _clusterSizes[root]; + } + + private bool IsCompactedFile(string filePath) + { + uint buf = 8; + _ = WofIsExternalFile(filePath, out int isExtFile, out uint _, out var info, ref buf); + if (isExtFile == 0) return false; + return info.Algorithm == CompressionAlgorithm.XPRESS8K; + } + + private void WOFCompressFile(string path) + { + var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo)); + Marshal.StructureToPtr(_efInfo, efInfoPtr, true); + ulong length = (ulong)Marshal.SizeOf(_efInfo); + try + { + using (var fs = new FileStream(path, FileMode.Open)) + { + var hFile = fs.SafeFileHandle.DangerousGetHandle(); + if (fs.SafeFileHandle.IsInvalid) + { + _logger.LogWarning("Invalid file handle to {file}", path); + } + else + { + var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length); + if (!(ret == 0 || ret == unchecked((int)0x80070158))) + { + _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); + } + } + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error compacting file {path}", path); + } + finally + { + Marshal.FreeHGlobal(efInfoPtr); + } + } + + private struct WOF_FILE_COMPRESSION_INFO_V1 + { + public CompressionAlgorithm Algorithm; + public ulong Flags; + } +} \ No newline at end of file diff --git a/MareSynchronos/FileCache/PeriodicFileScanner.cs b/MareSynchronos/FileCache/PeriodicFileScanner.cs index 9bd6877..5145b53 100644 --- a/MareSynchronos/FileCache/PeriodicFileScanner.cs +++ b/MareSynchronos/FileCache/PeriodicFileScanner.cs @@ -14,19 +14,22 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase private readonly IpcManager _ipcManager; private readonly PerformanceCollectorService _performanceCollector; private readonly DalamudUtilService _dalamudUtil; + private readonly FileCompactor _fileCompactor; private long _currentFileProgress = 0; private bool _fileScanWasRunning = false; private CancellationTokenSource _scanCancellationTokenSource = new(); private TimeSpan _timeUntilNextScan = TimeSpan.Zero; public PeriodicFileScanner(ILogger logger, IpcManager ipcManager, MareConfigService configService, - FileCacheManager fileDbManager, MareMediator mediator, PerformanceCollectorService performanceCollector, DalamudUtilService dalamudUtil) : base(logger, mediator) + FileCacheManager fileDbManager, MareMediator mediator, PerformanceCollectorService performanceCollector, DalamudUtilService dalamudUtil, + FileCompactor fileCompactor) : base(logger, mediator) { _ipcManager = ipcManager; _configService = configService; _fileDbManager = fileDbManager; _performanceCollector = performanceCollector; _dalamudUtil = dalamudUtil; + _fileCompactor = fileCompactor; Mediator.Subscribe(this, (_) => StartScan()); Mediator.Subscribe(this, (msg) => HaltScan(msg.Source)); Mediator.Subscribe(this, (msg) => ResumeScan(msg.Source)); @@ -108,7 +111,7 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase { try { - return new FileInfo(f).Length; + return _fileCompactor.GetFileSizeOnDisk(f); } catch { @@ -126,7 +129,7 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer) { var oldestFile = allFiles[0]; - FileCacheSize -= oldestFile.Length; + FileCacheSize -= _fileCompactor.GetFileSizeOnDisk(oldestFile.FullName); File.Delete(oldestFile.FullName); allFiles.Remove(oldestFile); } diff --git a/MareSynchronos/MareConfiguration/Configurations/MareConfig.cs b/MareSynchronos/MareConfiguration/Configurations/MareConfig.cs index 779c2ab..8e4a260 100644 --- a/MareSynchronos/MareConfiguration/Configurations/MareConfig.cs +++ b/MareSynchronos/MareConfiguration/Configurations/MareConfig.cs @@ -22,6 +22,7 @@ public class MareConfig : IMareConfiguration public bool OpenGposeImportOnGposeStart { get; set; } = false; public bool OpenPopupOnAdd { get; set; } = true; public int ParallelDownloads { get; set; } = 10; + public bool UseCompactor { get; set; } = false; public float ProfileDelay { get; set; } = 1.5f; public bool ProfilePopoutRight { get; set; } = false; public bool ProfilesAllowNsfw { get; set; } = false; diff --git a/MareSynchronos/PlayerData/Factories/FileDownloadManagerFactory.cs b/MareSynchronos/PlayerData/Factories/FileDownloadManagerFactory.cs index 70a6218..5878316 100644 --- a/MareSynchronos/PlayerData/Factories/FileDownloadManagerFactory.cs +++ b/MareSynchronos/PlayerData/Factories/FileDownloadManagerFactory.cs @@ -8,21 +8,23 @@ namespace MareSynchronos.PlayerData.Factories; public class FileDownloadManagerFactory { private readonly FileCacheManager _fileCacheManager; + private readonly FileCompactor _fileCompactor; private readonly FileTransferOrchestrator _fileTransferOrchestrator; private readonly ILoggerFactory _loggerFactory; private readonly MareMediator _mareMediator; public FileDownloadManagerFactory(ILoggerFactory loggerFactory, MareMediator mareMediator, FileTransferOrchestrator fileTransferOrchestrator, - FileCacheManager fileCacheManager) + FileCacheManager fileCacheManager, FileCompactor fileCompactor) { _loggerFactory = loggerFactory; _mareMediator = mareMediator; _fileTransferOrchestrator = fileTransferOrchestrator; _fileCacheManager = fileCacheManager; + _fileCompactor = fileCompactor; } public FileDownloadManager Create() { - return new FileDownloadManager(_loggerFactory.CreateLogger(), _mareMediator, _fileTransferOrchestrator, _fileCacheManager); + return new FileDownloadManager(_loggerFactory.CreateLogger(), _mareMediator, _fileTransferOrchestrator, _fileCacheManager, _fileCompactor); } } diff --git a/MareSynchronos/Plugin.cs b/MareSynchronos/Plugin.cs index 42b37a3..3285ede 100644 --- a/MareSynchronos/Plugin.cs +++ b/MareSynchronos/Plugin.cs @@ -74,6 +74,7 @@ public sealed class Plugin : IDalamudPlugin collection.AddSingleton(); collection.AddSingleton(); collection.AddSingleton(); + collection.AddSingleton(); collection.AddSingleton((s) => new DalamudUtilService(s.GetRequiredService>(), clientState, objectTable, framework, gameGui, condition, gameData, s.GetRequiredService(), s.GetRequiredService())); diff --git a/MareSynchronos/UI/IntroUI.cs b/MareSynchronos/UI/IntroUI.cs index 5cec40e..cdbe7f1 100644 --- a/MareSynchronos/UI/IntroUI.cs +++ b/MareSynchronos/UI/IntroUI.cs @@ -47,7 +47,11 @@ public class IntroUi : WindowMediatorSubscriberBase GetToSLocalization(); Mediator.Subscribe(this, (_) => IsOpen = false); - Mediator.Subscribe(this, (_) => IsOpen = true); + Mediator.Subscribe(this, (_) => + { + _configService.Current.UseCompactor = !Util.IsLinux(); + IsOpen = true; + }); } public override void Draw() @@ -170,6 +174,17 @@ public class IntroUi : WindowMediatorSubscriberBase { _uiShared.DrawFileScanState(); } + if (!Util.IsLinux()) + { + var useFileCompactor = _configService.Current.UseCompactor; + if (ImGui.Checkbox("Use File Compactor", ref useFileCompactor)) + { + _configService.Current.UseCompactor = useFileCompactor; + _configService.Save(); + } + UiSharedService.ColorTextWrapped("The File Compactor can save a tremendeous amount of space on the hard disk for downloads through Mare. It will incur a minor CPU penalty on download but can speed up " + + "loading of other characters. It is recommended to keep it enabled. You can change this setting later anytime in the Mare settings.", ImGuiColors.DalamudYellow); + } } else if (!_uiShared.ApiController.ServerAlive) { diff --git a/MareSynchronos/UI/SettingsUi.cs b/MareSynchronos/UI/SettingsUi.cs index efd2c24..42925f7 100644 --- a/MareSynchronos/UI/SettingsUi.cs +++ b/MareSynchronos/UI/SettingsUi.cs @@ -20,6 +20,7 @@ using MareSynchronos.WebAPI.Files; using MareSynchronos.WebAPI.Files.Models; using MareSynchronos.PlayerData.Handlers; using System.Collections.Concurrent; +using MareSynchronos.FileCache; namespace MareSynchronos.UI; @@ -29,6 +30,7 @@ public class SettingsUi : WindowMediatorSubscriberBase private readonly ConcurrentDictionary> _currentDownloads = new(); private readonly FileUploadManager _fileTransferManager; private readonly FileTransferOrchestrator _fileTransferOrchestrator; + private readonly FileCompactor _fileCompactor; private readonly MareCharaFileManager _mareCharaFileManager; private readonly PairManager _pairManager; private readonly PerformanceCollectorService _performanceCollector; @@ -50,7 +52,8 @@ public class SettingsUi : WindowMediatorSubscriberBase ServerConfigurationManager serverConfigurationManager, MareMediator mediator, PerformanceCollectorService performanceCollector, FileUploadManager fileTransferManager, - FileTransferOrchestrator fileTransferOrchestrator) : base(logger, mediator, "Mare Synchronos Settings") + FileTransferOrchestrator fileTransferOrchestrator, + FileCompactor fileCompactor) : base(logger, mediator, "Mare Synchronos Settings") { _configService = configService; _mareCharaFileManager = mareCharaFileManager; @@ -59,6 +62,7 @@ public class SettingsUi : WindowMediatorSubscriberBase _performanceCollector = performanceCollector; _fileTransferManager = fileTransferManager; _fileTransferOrchestrator = fileTransferOrchestrator; + _fileCompactor = fileCompactor; _uiShared = uiShared; SizeConstraints = new WindowSizeConstraints() @@ -433,6 +437,42 @@ public class SettingsUi : WindowMediatorSubscriberBase _uiShared.DrawTimeSpanBetweenScansSetting(); _uiShared.DrawCacheDirectorySetting(); ImGui.Text($"Currently utilized local storage: {UiSharedService.ByteToString(_uiShared.FileCacheSize)}"); + bool isLinux = Util.IsLinux(); + if (isLinux) ImGui.BeginDisabled(); + bool useFileCompactor = _configService.Current.UseCompactor; + if (ImGui.Checkbox("Use file compactor", ref useFileCompactor)) + { + _configService.Current.UseCompactor = useFileCompactor; + _configService.Save(); + } + UiSharedService.DrawHelpText("The file compactor can massively reduce your saved files. It might incur a minor penalty on loading files on a slow CPU." + Environment.NewLine + + "It is recommended to leave it enabled to save on space."); + ImGui.SameLine(); + if (!_fileCompactor.MassCompactRunning) + { + if (UiSharedService.IconTextButton(FontAwesomeIcon.FileArchive, "Compact all files in storage")) + { + _ = Task.Run(() => _fileCompactor.CompactStorage(true)); + } + UiSharedService.AttachToolTip("This will run compression on all files in your current Mare Storage." + Environment.NewLine + + "You do not need to run this manually if you keep the file compactor enabled."); + ImGui.SameLine(); + if (UiSharedService.IconTextButton(FontAwesomeIcon.File, "Decompact all files in storage")) + { + _ = Task.Run(() => _fileCompactor.CompactStorage(false)); + } + UiSharedService.AttachToolTip("This will run decompression on all files in your current Mare Storage."); + } + else + { + UiSharedService.ColorText($"File compactor currently running ({_fileCompactor.Progress})", ImGuiColors.DalamudYellow); + } + if (isLinux) + { + ImGui.EndDisabled(); + ImGui.Text("The file compactor is only available on Windows."); + } + ImGui.Dummy(new Vector2(10, 10)); ImGui.Text("To clear the local storage accept the following disclaimer"); ImGui.Indent(); diff --git a/MareSynchronos/WebAPI/Files/FileDownloadManager.cs b/MareSynchronos/WebAPI/Files/FileDownloadManager.cs index ab9deff..04fb305 100644 --- a/MareSynchronos/WebAPI/Files/FileDownloadManager.cs +++ b/MareSynchronos/WebAPI/Files/FileDownloadManager.cs @@ -17,15 +17,17 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase { private readonly Dictionary _downloadStatus; private readonly FileCacheManager _fileDbManager; + private readonly FileCompactor _fileCompactor; private readonly FileTransferOrchestrator _orchestrator; public FileDownloadManager(ILogger logger, MareMediator mediator, FileTransferOrchestrator orchestrator, - FileCacheManager fileCacheManager) : base(logger, mediator) + FileCacheManager fileCacheManager, FileCompactor fileCompactor) : base(logger, mediator) { _downloadStatus = new Dictionary(StringComparer.Ordinal); _orchestrator = orchestrator; _fileDbManager = fileCacheManager; + _fileCompactor = fileCompactor; } public List CurrentDownloads { get; private set; } = new(); @@ -280,7 +282,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase var decompressedFile = LZ4Codec.Unwrap(compressedFileContent); var filePath = _fileDbManager.GetCacheFilePath(fileHash, fileExtension); - await File.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false); + await _fileCompactor.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false); PersistFileToStorage(fileHash, filePath); }