add file compactor

MareSynchronos/FileCache/FileCompactor.cs

@@ -0,0 +1,212 @@
+using MareSynchronos.MareConfiguration;
+using Microsoft.Extensions.Logging;
+using System.Runtime.InteropServices;
+
+namespace MareSynchronos.FileCache;
+
+public sealed class FileCompactor
+{
+    public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U;
+    public const ulong WOF_PROVIDER_FILE = 2UL;
+
+    private readonly Dictionary<string, int> _clusterSizes;
+
+    private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo;
+    private readonly ILogger<FileCompactor> _logger;
+
+    private readonly MareConfigService _mareConfigService;
+
+    public FileCompactor(ILogger<FileCompactor> logger, MareConfigService mareConfigService)
+    {
+        _clusterSizes = new(StringComparer.Ordinal);
+        _logger = logger;
+        _mareConfigService = mareConfigService;
+        _efInfo = new WOF_FILE_COMPRESSION_INFO_V1
+        {
+            Algorithm = CompressionAlgorithm.XPRESS8K,
+            Flags = 0
+        };
+    }
+
+    private enum CompressionAlgorithm
+    {
+        NO_COMPRESSION = -2,
+        LZNT1 = -1,
+        XPRESS4K = 0,
+        LZX = 1,
+        XPRESS8K = 2,
+        XPRESS16K = 3
+    }
+
+    public bool MassCompactRunning { get; private set; } = false;
+
+    public string Progress { get; private set; } = string.Empty;
+
+    public void CompactStorage(bool compress)
+    {
+        MassCompactRunning = true;
+
+        int currentFile = 1;
+        var allFiles = Directory.EnumerateFiles(_mareConfigService.Current.CacheFolder).ToList();
+        int allFilesCount = allFiles.Count;
+        foreach (var file in allFiles)
+        {
+            Progress = $"{currentFile}/{allFilesCount}";
+            if (compress)
+                CompactFile(file);
+            else
+                DecompressFile(file);
+            currentFile++;
+        }
+
+        MassCompactRunning = false;
+    }
+
+    public long GetFileSizeOnDisk(string filePath)
+    {
+        if (Dalamud.Utility.Util.IsLinux()) return new FileInfo(filePath).Length;
+
+        var clusterSize = GetClusterSize(filePath);
+        if (clusterSize == -1) return new FileInfo(filePath).Length;
+        var losize = GetCompressedFileSizeW(filePath, out uint hosize);
+        var size = (long)hosize << 32 | losize;
+        return ((size + clusterSize - 1) / clusterSize) * clusterSize;
+    }
+
+    public async Task WriteAllBytesAsync(string filePath, byte[] decompressedFile, CancellationToken token)
+    {
+        await File.WriteAllBytesAsync(filePath, decompressedFile, token);
+
+        if (Dalamud.Utility.Util.IsLinux() || !_mareConfigService.Current.UseCompactor)
+        {
+            return;
+        }
+
+        CompactFile(filePath);
+    }
+
+    [DllImport("kernel32.dll")]
+    private static extern int DeviceIoControl(IntPtr hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out IntPtr lpBytesReturned, out IntPtr lpOverlapped);
+
+    [DllImport("kernel32.dll")]
+    private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName,
+                                              [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh);
+
+    [DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)]
+    private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName,
+           out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters,
+           out uint lpTotalNumberOfClusters);
+
+    [DllImport("WoFUtil.dll")]
+    private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength);
+
+    [DllImport("WofUtil.dll")]
+    private static extern int WofSetFileDataLocation(IntPtr FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length);
+
+    private void CompactFile(string filePath)
+    {
+        var oldSize = new FileInfo(filePath).Length;
+        var clusterSize = GetClusterSize(filePath);
+
+        if (oldSize < Math.Max(clusterSize, 8 * 1024))
+        {
+            _logger.LogDebug("File {file} is smaller than cluster size ({size}), ignoring", filePath, clusterSize);
+            return;
+        }
+
+        if (!IsCompactedFile(filePath))
+        {
+            _logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath);
+
+            WOFCompressFile(filePath);
+
+            var newSize = GetFileSizeOnDisk(filePath);
+
+            _logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize);
+        }
+        else
+        {
+            _logger.LogDebug("File {file} already compressed", filePath);
+        }
+    }
+
+    private void DecompressFile(string path)
+    {
+        _logger.LogDebug("Removing compression from {file}", path);
+        try
+        {
+            using (var fs = new FileStream(path, FileMode.Open))
+            {
+                var hDevice = fs.SafeFileHandle.DangerousGetHandle();
+                var ret = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _);
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error decompressing file {path}", path);
+        }
+    }
+
+    private int GetClusterSize(string filePath)
+    {
+        FileInfo fi = new(filePath);
+        if (!fi.Exists) return -1;
+        var root = fi.Directory?.Root.FullName.ToLower() ?? string.Empty;
+        if (string.IsNullOrEmpty(root)) return -1;
+        if (_clusterSizes.ContainsKey(root)) return _clusterSizes[root];
+        _logger.LogDebug("Getting Cluster Size for {path}, root {root}", filePath, root);
+        int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _);
+        if (result == 0) return -1;
+        _clusterSizes[root] = (int)(sectorsPerCluster * bytesPerSector);
+        _logger.LogDebug("Determined Cluster Size for root {root}: {cluster}", root, _clusterSizes[root]);
+        return _clusterSizes[root];
+    }
+
+    private bool IsCompactedFile(string filePath)
+    {
+        uint buf = 8;
+        _ = WofIsExternalFile(filePath, out int isExtFile, out uint _, out var info, ref buf);
+        if (isExtFile == 0) return false;
+        return info.Algorithm == CompressionAlgorithm.XPRESS8K;
+    }
+
+    private void WOFCompressFile(string path)
+    {
+        var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo));
+        Marshal.StructureToPtr(_efInfo, efInfoPtr, true);
+        ulong length = (ulong)Marshal.SizeOf(_efInfo);
+        try
+        {
+            using (var fs = new FileStream(path, FileMode.Open))
+            {
+                var hFile = fs.SafeFileHandle.DangerousGetHandle();
+                if (fs.SafeFileHandle.IsInvalid)
+                {
+                    _logger.LogWarning("Invalid file handle to {file}", path);
+                }
+                else
+                {
+                    var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length);
+                    if (!(ret == 0 || ret == unchecked((int)0x80070158)))
+                    {
+                        _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X"));
+                    }
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Error compacting file {path}", path);
+        }
+        finally
+        {
+            Marshal.FreeHGlobal(efInfoPtr);
+        }
+    }
+
+    private struct WOF_FILE_COMPRESSION_INFO_V1
+    {
+        public CompressionAlgorithm Algorithm;
+        public ulong Flags;
+    }
+}

MareSynchronos/FileCache/PeriodicFileScanner.cs

@@ -14,19 +14,22 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
     private readonly IpcManager _ipcManager;
     private readonly PerformanceCollectorService _performanceCollector;
     private readonly DalamudUtilService _dalamudUtil;
+    private readonly FileCompactor _fileCompactor;
     private long _currentFileProgress = 0;
     private bool _fileScanWasRunning = false;
     private CancellationTokenSource _scanCancellationTokenSource = new();
     private TimeSpan _timeUntilNextScan = TimeSpan.Zero;
 
     public PeriodicFileScanner(ILogger<PeriodicFileScanner> logger, IpcManager ipcManager, MareConfigService configService,
-        FileCacheManager fileDbManager, MareMediator mediator, PerformanceCollectorService performanceCollector, DalamudUtilService dalamudUtil) : base(logger, mediator)
+        FileCacheManager fileDbManager, MareMediator mediator, PerformanceCollectorService performanceCollector, DalamudUtilService dalamudUtil,
+        FileCompactor fileCompactor) : base(logger, mediator)
     {
         _ipcManager = ipcManager;
         _configService = configService;
         _fileDbManager = fileDbManager;
         _performanceCollector = performanceCollector;
         _dalamudUtil = dalamudUtil;
+        _fileCompactor = fileCompactor;
         Mediator.Subscribe<PenumbraInitializedMessage>(this, (_) => StartScan());
         Mediator.Subscribe<HaltScanMessage>(this, (msg) => HaltScan(msg.Source));
         Mediator.Subscribe<ResumeScanMessage>(this, (msg) => ResumeScan(msg.Source));
@@ -108,7 +111,7 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
         {
             try
             {
-                return new FileInfo(f).Length;
+                return _fileCompactor.GetFileSizeOnDisk(f);
             }
             catch
             {
@@ -126,7 +129,7 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase
         while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer)
         {
             var oldestFile = allFiles[0];
-            FileCacheSize -= oldestFile.Length;
+            FileCacheSize -= _fileCompactor.GetFileSizeOnDisk(oldestFile.FullName);
             File.Delete(oldestFile.FullName);
             allFiles.Remove(oldestFile);
         }

MareSynchronos/MareConfiguration/Configurations/MareConfig.cs

@@ -22,6 +22,7 @@ public class MareConfig : IMareConfiguration
     public bool OpenGposeImportOnGposeStart { get; set; } = false;
     public bool OpenPopupOnAdd { get; set; } = true;
     public int ParallelDownloads { get; set; } = 10;
+    public bool UseCompactor { get; set; } = false;
     public float ProfileDelay { get; set; } = 1.5f;
     public bool ProfilePopoutRight { get; set; } = false;
     public bool ProfilesAllowNsfw { get; set; } = false;

MareSynchronos/PlayerData/Factories/FileDownloadManagerFactory.cs

@@ -8,21 +8,23 @@ namespace MareSynchronos.PlayerData.Factories;
 public class FileDownloadManagerFactory
 {
     private readonly FileCacheManager _fileCacheManager;
+    private readonly FileCompactor _fileCompactor;
     private readonly FileTransferOrchestrator _fileTransferOrchestrator;
     private readonly ILoggerFactory _loggerFactory;
     private readonly MareMediator _mareMediator;
 
     public FileDownloadManagerFactory(ILoggerFactory loggerFactory, MareMediator mareMediator, FileTransferOrchestrator fileTransferOrchestrator,
-        FileCacheManager fileCacheManager)
+        FileCacheManager fileCacheManager, FileCompactor fileCompactor)
     {
         _loggerFactory = loggerFactory;
         _mareMediator = mareMediator;
         _fileTransferOrchestrator = fileTransferOrchestrator;
         _fileCacheManager = fileCacheManager;
+        _fileCompactor = fileCompactor;
     }
 
     public FileDownloadManager Create()
     {
-        return new FileDownloadManager(_loggerFactory.CreateLogger<FileDownloadManager>(), _mareMediator, _fileTransferOrchestrator, _fileCacheManager);
+        return new FileDownloadManager(_loggerFactory.CreateLogger<FileDownloadManager>(), _mareMediator, _fileTransferOrchestrator, _fileCacheManager, _fileCompactor);
     }
 }

MareSynchronos/Plugin.cs

@@ -74,6 +74,7 @@ public sealed class Plugin : IDalamudPlugin
             collection.AddSingleton<PairFactory>();
             collection.AddSingleton<CharacterAnalyzer>();
             collection.AddSingleton<PluginWarningNotificationService>();
+            collection.AddSingleton<FileCompactor>();
             collection.AddSingleton((s) => new DalamudUtilService(s.GetRequiredService<ILogger<DalamudUtilService>>(),
                 clientState, objectTable, framework, gameGui, condition, gameData,
                 s.GetRequiredService<MareMediator>(), s.GetRequiredService<PerformanceCollectorService>()));

MareSynchronos/UI/IntroUI.cs

@@ -47,7 +47,11 @@ public class IntroUi : WindowMediatorSubscriberBase
         GetToSLocalization();
 
         Mediator.Subscribe<SwitchToMainUiMessage>(this, (_) => IsOpen = false);
-        Mediator.Subscribe<SwitchToIntroUiMessage>(this, (_) => IsOpen = true);
+        Mediator.Subscribe<SwitchToIntroUiMessage>(this, (_) =>
+        {
+            _configService.Current.UseCompactor = !Util.IsLinux();
+            IsOpen = true;
+        });
     }
 
     public override void Draw()
@@ -170,6 +174,17 @@ public class IntroUi : WindowMediatorSubscriberBase
             {
                 _uiShared.DrawFileScanState();
             }
+            if (!Util.IsLinux())
+            {
+                var useFileCompactor = _configService.Current.UseCompactor;
+                if (ImGui.Checkbox("Use File Compactor", ref useFileCompactor))
+                {
+                    _configService.Current.UseCompactor = useFileCompactor;
+                    _configService.Save();
+                }
+                UiSharedService.ColorTextWrapped("The File Compactor can save a tremendeous amount of space on the hard disk for downloads through Mare. It will incur a minor CPU penalty on download but can speed up " +
+                    "loading of other characters. It is recommended to keep it enabled. You can change this setting later anytime in the Mare settings.", ImGuiColors.DalamudYellow);
+            }
         }
         else if (!_uiShared.ApiController.ServerAlive)
         {

MareSynchronos/UI/SettingsUi.cs

@@ -20,6 +20,7 @@ using MareSynchronos.WebAPI.Files;
 using MareSynchronos.WebAPI.Files.Models;
 using MareSynchronos.PlayerData.Handlers;
 using System.Collections.Concurrent;
+using MareSynchronos.FileCache;
 
 namespace MareSynchronos.UI;
 
@@ -29,6 +30,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
     private readonly ConcurrentDictionary<GameObjectHandler, Dictionary<string, FileDownloadStatus>> _currentDownloads = new();
     private readonly FileUploadManager _fileTransferManager;
     private readonly FileTransferOrchestrator _fileTransferOrchestrator;
+    private readonly FileCompactor _fileCompactor;
     private readonly MareCharaFileManager _mareCharaFileManager;
     private readonly PairManager _pairManager;
     private readonly PerformanceCollectorService _performanceCollector;
@@ -50,7 +52,8 @@ public class SettingsUi : WindowMediatorSubscriberBase
         ServerConfigurationManager serverConfigurationManager,
         MareMediator mediator, PerformanceCollectorService performanceCollector,
         FileUploadManager fileTransferManager,
-        FileTransferOrchestrator fileTransferOrchestrator) : base(logger, mediator, "Mare Synchronos Settings")
+        FileTransferOrchestrator fileTransferOrchestrator,
+        FileCompactor fileCompactor) : base(logger, mediator, "Mare Synchronos Settings")
     {
         _configService = configService;
         _mareCharaFileManager = mareCharaFileManager;
@@ -59,6 +62,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
         _performanceCollector = performanceCollector;
         _fileTransferManager = fileTransferManager;
         _fileTransferOrchestrator = fileTransferOrchestrator;
+        _fileCompactor = fileCompactor;
         _uiShared = uiShared;
 
         SizeConstraints = new WindowSizeConstraints()
@@ -433,6 +437,42 @@ public class SettingsUi : WindowMediatorSubscriberBase
         _uiShared.DrawTimeSpanBetweenScansSetting();
         _uiShared.DrawCacheDirectorySetting();
         ImGui.Text($"Currently utilized local storage: {UiSharedService.ByteToString(_uiShared.FileCacheSize)}");
+        bool isLinux = Util.IsLinux();
+        if (isLinux) ImGui.BeginDisabled();
+        bool useFileCompactor = _configService.Current.UseCompactor;
+        if (ImGui.Checkbox("Use file compactor", ref useFileCompactor))
+        {
+            _configService.Current.UseCompactor = useFileCompactor;
+            _configService.Save();
+        }
+        UiSharedService.DrawHelpText("The file compactor can massively reduce your saved files. It might incur a minor penalty on loading files on a slow CPU." + Environment.NewLine
+            + "It is recommended to leave it enabled to save on space.");
+        ImGui.SameLine();
+        if (!_fileCompactor.MassCompactRunning)
+        {
+            if (UiSharedService.IconTextButton(FontAwesomeIcon.FileArchive, "Compact all files in storage"))
+            {
+                _ = Task.Run(() => _fileCompactor.CompactStorage(true));
+            }
+            UiSharedService.AttachToolTip("This will run compression on all files in your current Mare Storage." + Environment.NewLine
+                + "You do not need to run this manually if you keep the file compactor enabled.");
+            ImGui.SameLine();
+            if (UiSharedService.IconTextButton(FontAwesomeIcon.File, "Decompact all files in storage"))
+            {
+                _ = Task.Run(() => _fileCompactor.CompactStorage(false));
+            }
+            UiSharedService.AttachToolTip("This will run decompression on all files in your current Mare Storage.");
+        }
+        else
+        {
+            UiSharedService.ColorText($"File compactor currently running ({_fileCompactor.Progress})", ImGuiColors.DalamudYellow);
+        }
+        if (isLinux)
+        {
+            ImGui.EndDisabled();
+            ImGui.Text("The file compactor is only available on Windows.");
+        }
+
         ImGui.Dummy(new Vector2(10, 10));
         ImGui.Text("To clear the local storage accept the following disclaimer");
         ImGui.Indent();

MareSynchronos/WebAPI/Files/FileDownloadManager.cs

@@ -17,15 +17,17 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
 {
     private readonly Dictionary<string, FileDownloadStatus> _downloadStatus;
     private readonly FileCacheManager _fileDbManager;
+    private readonly FileCompactor _fileCompactor;
     private readonly FileTransferOrchestrator _orchestrator;
 
     public FileDownloadManager(ILogger<FileDownloadManager> logger, MareMediator mediator,
         FileTransferOrchestrator orchestrator,
-        FileCacheManager fileCacheManager) : base(logger, mediator)
+        FileCacheManager fileCacheManager, FileCompactor fileCompactor) : base(logger, mediator)
     {
         _downloadStatus = new Dictionary<string, FileDownloadStatus>(StringComparer.Ordinal);
         _orchestrator = orchestrator;
         _fileDbManager = fileCacheManager;
+        _fileCompactor = fileCompactor;
     }
 
     public List<DownloadFileTransfer> CurrentDownloads { get; private set; } = new();
@@ -280,7 +282,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
 
                         var decompressedFile = LZ4Codec.Unwrap(compressedFileContent);
                         var filePath = _fileDbManager.GetCacheFilePath(fileHash, fileExtension);
-                        await File.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false);
+                        await _fileCompactor.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false);
 
                         PersistFileToStorage(fileHash, filePath);
                     }