add file storage validation

MareSynchronos/FileCache/FileCacheManager.cs

@@ -1,6 +1,7 @@
 using LZ4;
 using MareSynchronos.Interop;
 using MareSynchronos.MareConfiguration;
+using MareSynchronos.Services.Mediator;
 using MareSynchronos.Utils;
 using Microsoft.Extensions.Logging;
 using System.Collections.Concurrent;
@@ -15,17 +16,19 @@ public sealed class FileCacheManager : IDisposable
     public const string CsvSplit = "|";
     public const string PenumbraPrefix = "{penumbra}";
     private readonly MareConfigService _configService;
+    private readonly MareMediator _mareMediator;
     private readonly string _csvPath;
     private readonly ConcurrentDictionary<string, List<FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
     private readonly object _fileWriteLock = new();
     private readonly IpcManager _ipcManager;
     private readonly ILogger<FileCacheManager> _logger;
 
-    public FileCacheManager(ILogger<FileCacheManager> logger, IpcManager ipcManager, MareConfigService configService)
+    public FileCacheManager(ILogger<FileCacheManager> logger, IpcManager ipcManager, MareConfigService configService, MareMediator mareMediator)
     {
         _logger = logger;
         _ipcManager = ipcManager;
         _configService = configService;
+        _mareMediator = mareMediator;
         _csvPath = Path.Combine(configService.ConfigurationDirectory, "FileCache.csv");
 
         lock (_fileWriteLock)
@@ -172,6 +175,47 @@ public sealed class FileCacheManager : IDisposable
         return output;
     }
 
+    public Task<List<FileCacheEntity>> ValidateLocalIntegrity(IProgress<(int, int, FileCacheEntity)> progress, CancellationToken cancellationToken)
+    {
+        _mareMediator.Publish(new HaltScanMessage("IntegrityCheck"));
+        _logger.LogInformation("Validating local storage");
+        var cacheEntries = _fileCaches.SelectMany(v => v.Value).Where(v => v.IsCacheEntry).ToList();
+        List<FileCacheEntity> brokenEntities = new();
+        int i = 0;
+        foreach (var fileCache in cacheEntries)
+        {
+            _logger.LogInformation("Validating {file}", fileCache.ResolvedFilepath);
+
+            progress.Report((i, cacheEntries.Count, fileCache));
+            i++;
+            var computedHash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
+            if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal))
+            {
+                _logger.LogInformation("Failed to validate {file}, got hash {hash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
+                brokenEntities.Add(fileCache);
+            }
+
+            if (cancellationToken.IsCancellationRequested) break;
+        }
+
+        foreach (var brokenEntity in brokenEntities)
+        {
+            RemoveHashedFile(brokenEntity.Hash, brokenEntity.PrefixedFilePath);
+
+            try
+            {
+                File.Delete(brokenEntity.ResolvedFilepath);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Could not delete {file}", brokenEntity.ResolvedFilepath);
+            }
+        }
+
+        _mareMediator.Publish(new ResumeScanMessage("IntegrityCheck"));
+        return Task.FromResult(brokenEntities);
+    }
+
     public string GetCacheFilePath(string hash, string extension)
     {
         return Path.Combine(_configService.Current.CacheFolder, hash + "." + extension);

MareSynchronos/UI/SettingsUi.cs

@@ -35,6 +35,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
     private readonly FileCompactor _fileCompactor;
     private readonly FileUploadManager _fileTransferManager;
     private readonly FileTransferOrchestrator _fileTransferOrchestrator;
+    private readonly FileCacheManager _fileCacheManager;
     private readonly MareCharaFileManager _mareCharaFileManager;
     private readonly PairManager _pairManager;
     private readonly PerformanceCollectorService _performanceCollector;
@@ -49,6 +50,10 @@ public class SettingsUi : WindowMediatorSubscriberBase
     private bool _readClearCache = false;
     private bool _readExport = false;
     private bool _wasOpen = false;
+    private readonly IProgress<(int, int, FileCacheEntity)> _validationProgress;
+    private Task<List<FileCacheEntity>>? _validationTask;
+    private CancellationTokenSource? _validationCts;
+    private (int, int, FileCacheEntity) _currentProgress;
 
     public SettingsUi(ILogger<SettingsUi> logger,
         UiSharedService uiShared, MareConfigService configService,
@@ -57,6 +62,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
         MareMediator mediator, PerformanceCollectorService performanceCollector,
         FileUploadManager fileTransferManager,
         FileTransferOrchestrator fileTransferOrchestrator,
+        FileCacheManager fileCacheManager,
         FileCompactor fileCompactor, ApiController apiController) : base(logger, mediator, "Mare Synchronos Settings")
     {
         _configService = configService;
@@ -66,11 +72,13 @@ public class SettingsUi : WindowMediatorSubscriberBase
         _performanceCollector = performanceCollector;
         _fileTransferManager = fileTransferManager;
         _fileTransferOrchestrator = fileTransferOrchestrator;
+        _fileCacheManager = fileCacheManager;
         _apiController = apiController;
         _fileCompactor = fileCompactor;
         _uiShared = uiShared;
         AllowClickthrough = false;
         AllowPinning = false;
+        _validationProgress = new Progress<(int, int, FileCacheEntity)>(v => _currentProgress = v);
 
         SizeConstraints = new WindowSizeConstraints()
         {
@@ -509,6 +517,49 @@ public class SettingsUi : WindowMediatorSubscriberBase
             ImGui.EndDisabled();
             ImGui.TextUnformatted("The file compactor is only available on Windows.");
         }
+        ImGuiHelpers.ScaledDummy(new Vector2(10, 10));
+
+        ImGui.Separator();
+        UiSharedService.TextWrapped("File Storage validation can make sure that all files in your local Mare Storage are valid. " +
+            "Run the validation before you clear the Storage for no reason. " + Environment.NewLine +
+            "This operation, depending on how many files you have in your storage, can take a while and will be CPU and drive intensive.");
+        using (ImRaii.Disabled(_validationTask != null && !_validationTask.IsCompleted))
+        {
+            if (UiSharedService.NormalizedIconTextButton(FontAwesomeIcon.Check, "Start File Storage Validation"))
+            {
+                _validationCts?.Cancel();
+                _validationCts?.Dispose();
+                _validationCts = new();
+                var token = _validationCts.Token;
+                _validationTask = Task.Run(() => _fileCacheManager.ValidateLocalIntegrity(_validationProgress, token));
+            }
+        }
+        if (_validationTask != null && !_validationTask.IsCompleted)
+        {
+            ImGui.SameLine();
+            if (UiSharedService.NormalizedIconTextButton(FontAwesomeIcon.Times, "Cancel"))
+            {
+                _validationCts?.Cancel();
+            }
+        }
+
+        if (_validationTask != null)
+        {
+            using (ImRaii.PushIndent(20f))
+            {
+                if (_validationTask.IsCompleted)
+                {
+                    UiSharedService.TextWrapped($"The storage validation has completed and removed {_validationTask.Result.Count} invalid files from storage.");
+                }
+                else
+                {
+
+                    UiSharedService.TextWrapped($"Storage validation is running: {_currentProgress.Item1}/{_currentProgress.Item2}");
+                    UiSharedService.TextWrapped($"Current item: {_currentProgress.Item3.ResolvedFilepath}");
+                }
+            }
+        }
+        ImGui.Separator();
 
         ImGuiHelpers.ScaledDummy(new Vector2(10, 10));
         ImGui.TextUnformatted("To clear the local storage accept the following disclaimer");