using K4os.Compression.LZ4.Streams; using MareSynchronos.Interop; using MareSynchronos.MareConfiguration; using MareSynchronos.Services.Mediator; using MareSynchronos.Utils; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Globalization; using System.Text; namespace MareSynchronos.FileCache; public sealed class FileCacheManager : IDisposable { public const string CachePrefix = "{cache}"; public const string CsvSplit = "|"; public const string PenumbraPrefix = "{penumbra}"; private readonly MareConfigService _configService; private readonly MareMediator _mareMediator; private readonly string _csvPath; private readonly ConcurrentDictionary> _fileCaches = new(StringComparer.Ordinal); private readonly object _fileWriteLock = new(); private readonly IpcManager _ipcManager; private readonly ILogger _logger; public FileCacheManager(ILogger logger, IpcManager ipcManager, MareConfigService configService, MareMediator mareMediator) { _logger = logger; _ipcManager = ipcManager; _configService = configService; _mareMediator = mareMediator; _csvPath = Path.Combine(configService.ConfigurationDirectory, "FileCache.csv"); lock (_fileWriteLock) { try { if (File.Exists(CsvBakPath)) { File.Move(CsvBakPath, _csvPath, overwrite: true); } } catch (Exception ex) { _logger.LogWarning(ex, "Failed to move BAK to ORG, deleting BAK"); try { if (File.Exists(CsvBakPath)) File.Delete(CsvBakPath); } catch (Exception ex1) { _logger.LogWarning(ex1, "Could not delete bak file"); } } } if (File.Exists(_csvPath)) { bool success = false; string[] entries = []; int attempts = 0; while (!success && attempts < 10) { try { entries = File.ReadAllLines(_csvPath); success = true; } catch (Exception ex) { attempts++; _logger.LogWarning(ex, "Could not open {file}, trying again", _csvPath); Thread.Sleep(100); } } if (!entries.Any()) { _logger.LogWarning("Could not load entries from {path}, continuing with empty file cache", _csvPath); } Dictionary processedFiles = new(StringComparer.OrdinalIgnoreCase); foreach (var entry in entries) { var splittedEntry = entry.Split(CsvSplit, StringSplitOptions.None); try { var hash = splittedEntry[0]; if (hash.Length != 40) throw new InvalidOperationException("Expected Hash length of 40, received " + hash.Length); var path = splittedEntry[1]; var time = splittedEntry[2]; if (processedFiles.ContainsKey(path)) { _logger.LogWarning("Already processed {file}, ignoring", path); continue; } processedFiles.Add(path, value: true); long size = -1; long compressed = -1; if (splittedEntry.Length > 3) { if (long.TryParse(splittedEntry[3], CultureInfo.InvariantCulture, out long result)) { size = result; } if (long.TryParse(splittedEntry[4], CultureInfo.InvariantCulture, out long resultCompressed)) { compressed = resultCompressed; } } AddHashedFile(ReplacePathPrefixes(new FileCacheEntity(hash, path, time, size, compressed))); } catch (Exception ex) { _logger.LogWarning(ex, "Failed to initialize entry {entry}, ignoring", entry); } } if (processedFiles.Count != entries.Length) { WriteOutFullCsv(); } } } private string CsvBakPath => _csvPath + ".bak"; public FileCacheEntity? CreateCacheEntry(string path) { _logger.LogTrace("Creating cache entry for {path}", path); FileInfo fi = new(path); if (!fi.Exists) return null; var fullName = fi.FullName.ToLowerInvariant(); if (!fullName.Contains(_configService.Current.CacheFolder.ToLowerInvariant(), StringComparison.Ordinal)) return null; string prefixedPath = fullName.Replace(_configService.Current.CacheFolder.ToLowerInvariant(), CachePrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal); return CreateFileCacheEntity(fi, prefixedPath); } public FileCacheEntity? CreateFileEntry(string path) { _logger.LogTrace("Creating file entry for {path}", path); FileInfo fi = new(path); if (!fi.Exists) return null; var fullName = fi.FullName.ToLowerInvariant(); if (!fullName.Contains(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), StringComparison.Ordinal)) return null; string prefixedPath = fullName.Replace(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), PenumbraPrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal); return CreateFileCacheEntity(fi, prefixedPath); } public void Dispose() { _logger.LogTrace("Disposing {type}", GetType()); WriteOutFullCsv(); GC.SuppressFinalize(this); } public List GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v).ToList(); public List GetAllFileCachesByHash(string hash) { List output = []; if (_fileCaches.TryGetValue(hash, out var fileCacheEntities)) { foreach (var filecache in fileCacheEntities.ToList()) { var validated = GetValidatedFileCache(filecache); if (validated != null) output.Add(validated); } } return output; } public Task> ValidateLocalIntegrity(IProgress<(int, int, FileCacheEntity)> progress, CancellationToken cancellationToken) { _mareMediator.Publish(new HaltScanMessage(nameof(ValidateLocalIntegrity))); _logger.LogInformation("Validating local storage"); var cacheEntries = _fileCaches.SelectMany(v => v.Value).Where(v => v.IsCacheEntry).ToList(); List brokenEntities = new(); int i = 0; foreach (var fileCache in cacheEntries) { _logger.LogInformation("Validating {file}", fileCache.ResolvedFilepath); progress.Report((i, cacheEntries.Count, fileCache)); i++; var computedHash = Crypto.GetFileHash(fileCache.ResolvedFilepath); if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal)) { _logger.LogInformation("Failed to validate {file}, got hash {hash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash); brokenEntities.Add(fileCache); } if (cancellationToken.IsCancellationRequested) break; } foreach (var brokenEntity in brokenEntities) { RemoveHashedFile(brokenEntity.Hash, brokenEntity.PrefixedFilePath); try { File.Delete(brokenEntity.ResolvedFilepath); } catch (Exception ex) { _logger.LogWarning(ex, "Could not delete {file}", brokenEntity.ResolvedFilepath); } } _mareMediator.Publish(new ResumeScanMessage(nameof(ValidateLocalIntegrity))); return Task.FromResult(brokenEntities); } public string GetCacheFilePath(string hash, string extension) { return Path.Combine(_configService.Current.CacheFolder, hash + "." + extension); } public async Task GetCompressedFileLength(string fileHash, CancellationToken uploadToken) { var fileCache = GetFileCacheByHash(fileHash)!.ResolvedFilepath; using var fs = File.OpenRead(fileCache); var cs = new CountedStream(Stream.Null); using var encstream = LZ4Stream.Encode(cs, new LZ4EncoderSettings(){CompressionLevel=K4os.Compression.LZ4.LZ4Level.L09_HC}); await fs.CopyToAsync(encstream, uploadToken).ConfigureAwait(false); encstream.Close(); return uploadToken.IsCancellationRequested ? 0 : cs.BytesWritten; } public async Task GetCompressedFileData(string fileHash, CancellationToken uploadToken) { var fileCache = GetFileCacheByHash(fileHash)!.ResolvedFilepath; using var fs = File.OpenRead(fileCache); var ms = new MemoryStream(64 * 1024); using var encstream = LZ4Stream.Encode(ms, new LZ4EncoderSettings(){CompressionLevel=K4os.Compression.LZ4.LZ4Level.L09_HC}); await fs.CopyToAsync(encstream, uploadToken).ConfigureAwait(false); encstream.Close(); return ms.ToArray(); } public FileCacheEntity? GetFileCacheByHash(string hash) { if (_fileCaches.TryGetValue(hash, out var hashes)) { var item = hashes.FirstOrDefault(); if (item != null) return GetValidatedFileCache(item); } return null; } public FileCacheEntity? GetFileCacheByPath(string path) { var cleanedPath = path.Replace("/", "\\", StringComparison.OrdinalIgnoreCase).ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), "", StringComparison.OrdinalIgnoreCase); var entry = _fileCaches.SelectMany(v => v.Value).FirstOrDefault(f => f.ResolvedFilepath.EndsWith(cleanedPath, StringComparison.OrdinalIgnoreCase)); if (entry == null) { _logger.LogDebug("Found no entries for {path}", cleanedPath); return CreateFileEntry(path); } var validatedCacheEntry = GetValidatedFileCache(entry); return validatedCacheEntry; } public Dictionary GetFileCachesByPaths(string[] paths) { var cleanedPaths = paths.Distinct(StringComparer.OrdinalIgnoreCase).ToDictionary(p => p, p => p.Replace("/", "\\", StringComparison.OrdinalIgnoreCase) .Replace(_ipcManager.PenumbraModDirectory!, _ipcManager.PenumbraModDirectory!.EndsWith('\\') ? (PenumbraPrefix + "\\") : PenumbraPrefix, StringComparison.OrdinalIgnoreCase), StringComparer.OrdinalIgnoreCase); Dictionary result = new(StringComparer.OrdinalIgnoreCase); var dict = _fileCaches.SelectMany(f => f.Value).Where(f => f.PrefixedFilePath.StartsWith(PenumbraPrefix, StringComparison.Ordinal)) .ToDictionary(d => d.PrefixedFilePath, d => d, StringComparer.Ordinal); foreach (var entry in cleanedPaths) { if (dict.TryGetValue(entry.Value, out var entity)) { var validatedCache = GetValidatedFileCache(entity); result.Add(entry.Key, validatedCache); } else { result.Add(entry.Key, CreateFileEntry(entry.Key)); } } return result; } public void RemoveHashedFile(string hash, string prefixedFilePath) { if (_fileCaches.TryGetValue(hash, out var caches)) { caches?.RemoveAll(c => string.Equals(c.PrefixedFilePath, prefixedFilePath, StringComparison.Ordinal)); if (caches?.Count == 0) { _fileCaches.Remove(hash, out _); } } } public void UpdateHashedFile(FileCacheEntity fileCache, bool computeProperties = true) { _logger.LogTrace("Updating hash for {path}", fileCache.ResolvedFilepath); var oldHash = fileCache.Hash; var prefixedPath = fileCache.PrefixedFilePath; if (computeProperties) { var fi = new FileInfo(fileCache.ResolvedFilepath); fileCache.Size = fi.Length; fileCache.CompressedSize = null; fileCache.Hash = Crypto.GetFileHash(fileCache.ResolvedFilepath); fileCache.LastModifiedDateTicks = fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture); } RemoveHashedFile(oldHash, prefixedPath); AddHashedFile(fileCache); } public (FileState State, FileCacheEntity FileCache) ValidateFileCacheEntity(FileCacheEntity fileCache) { fileCache = ReplacePathPrefixes(fileCache); FileInfo fi = new(fileCache.ResolvedFilepath); if (!fi.Exists) { return (FileState.RequireDeletion, fileCache); } if (!string.Equals(fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), fileCache.LastModifiedDateTicks, StringComparison.Ordinal)) { return (FileState.RequireUpdate, fileCache); } return (FileState.Valid, fileCache); } public void WriteOutFullCsv() { StringBuilder sb = new(); foreach (var entry in _fileCaches.SelectMany(k => k.Value).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase)) { sb.AppendLine(entry.CsvEntry); } lock (_fileWriteLock) { if (File.Exists(_csvPath)) { File.Copy(_csvPath, CsvBakPath, overwrite: true); } try { File.WriteAllText(_csvPath, sb.ToString()); File.Delete(CsvBakPath); } catch { File.WriteAllText(CsvBakPath, sb.ToString()); } } } internal FileCacheEntity MigrateFileHashToExtension(FileCacheEntity fileCache, string ext) { try { RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath); FileInfo fileInfo = new(fileCache.ResolvedFilepath); FileInfo oldCache = fileInfo; var extensionPath = fileCache.ResolvedFilepath.ToUpper(CultureInfo.InvariantCulture) + "." + ext; File.Move(fileCache.ResolvedFilepath, extensionPath, overwrite: true); var newHashedEntity = new FileCacheEntity(fileCache.Hash, fileCache.PrefixedFilePath + "." + ext, DateTime.UtcNow.Ticks.ToString(CultureInfo.InvariantCulture)); newHashedEntity.SetResolvedFilePath(extensionPath); AddHashedFile(newHashedEntity); _logger.LogDebug("Migrated from {oldPath} to {newPath}", fileCache.ResolvedFilepath, newHashedEntity.ResolvedFilepath); return newHashedEntity; } catch (Exception ex) { AddHashedFile(fileCache); _logger.LogWarning(ex, "Failed to migrate entity {entity}", fileCache.PrefixedFilePath); return fileCache; } } private void AddHashedFile(FileCacheEntity fileCache) { if (!_fileCaches.TryGetValue(fileCache.Hash, out var entries) || entries is null) { _fileCaches[fileCache.Hash] = entries = []; } if (!entries.Exists(u => string.Equals(u.PrefixedFilePath, fileCache.PrefixedFilePath, StringComparison.OrdinalIgnoreCase))) { entries.Add(fileCache); } } private FileCacheEntity? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath, string? hash = null) { hash ??= Crypto.GetFileHash(fileInfo.FullName); var entity = new FileCacheEntity(hash, prefixedPath, fileInfo.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), fileInfo.Length); entity = ReplacePathPrefixes(entity); AddHashedFile(entity); lock (_fileWriteLock) { File.AppendAllLines(_csvPath, new[] { entity.CsvEntry }); } var result = GetFileCacheByPath(fileInfo.FullName); _logger.LogDebug("Creating file cache for {name} success: {success}", fileInfo.FullName, (result != null)); return result; } private FileCacheEntity? GetValidatedFileCache(FileCacheEntity fileCache) { var resulingFileCache = ReplacePathPrefixes(fileCache); resulingFileCache = Validate(resulingFileCache); return resulingFileCache; } private FileCacheEntity ReplacePathPrefixes(FileCacheEntity fileCache) { if (fileCache.PrefixedFilePath.StartsWith(PenumbraPrefix, StringComparison.OrdinalIgnoreCase)) { fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory, StringComparison.Ordinal)); } else if (fileCache.PrefixedFilePath.StartsWith(CachePrefix, StringComparison.OrdinalIgnoreCase)) { fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(CachePrefix, _configService.Current.CacheFolder, StringComparison.Ordinal)); } return fileCache; } private FileCacheEntity? Validate(FileCacheEntity fileCache) { var file = new FileInfo(fileCache.ResolvedFilepath); if (!file.Exists) { RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath); return null; } if (!string.Equals(file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), fileCache.LastModifiedDateTicks, StringComparison.Ordinal)) { UpdateHashedFile(fileCache); } return fileCache; } }