467 lines
18 KiB
C#
467 lines
18 KiB
C#
using K4os.Compression.LZ4.Streams;
|
|
using MareSynchronos.Interop;
|
|
using MareSynchronos.MareConfiguration;
|
|
using MareSynchronos.Services.Mediator;
|
|
using MareSynchronos.Utils;
|
|
using Microsoft.Extensions.Hosting;
|
|
using Microsoft.Extensions.Logging;
|
|
using System.Collections.Concurrent;
|
|
using System.Globalization;
|
|
using System.Text;
|
|
|
|
namespace MareSynchronos.FileCache;
|
|
|
|
public sealed class FileCacheManager : IHostedService
|
|
{
|
|
public const string CachePrefix = "{cache}";
|
|
public const string CsvSplit = "|";
|
|
public const string PenumbraPrefix = "{penumbra}";
|
|
private readonly MareConfigService _configService;
|
|
private readonly MareMediator _mareMediator;
|
|
private readonly string _csvPath;
|
|
private readonly ConcurrentDictionary<string, List<FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
|
|
private readonly object _fileWriteLock = new();
|
|
private readonly IpcManager _ipcManager;
|
|
private readonly ILogger<FileCacheManager> _logger;
|
|
|
|
public FileCacheManager(ILogger<FileCacheManager> logger, IpcManager ipcManager, MareConfigService configService, MareMediator mareMediator)
|
|
{
|
|
_logger = logger;
|
|
_ipcManager = ipcManager;
|
|
_configService = configService;
|
|
_mareMediator = mareMediator;
|
|
_csvPath = Path.Combine(configService.ConfigurationDirectory, "FileCache.csv");
|
|
}
|
|
|
|
private string CsvBakPath => _csvPath + ".bak";
|
|
|
|
public FileCacheEntity? CreateCacheEntry(string path)
|
|
{
|
|
_logger.LogTrace("Creating cache entry for {path}", path);
|
|
FileInfo fi = new(path);
|
|
if (!fi.Exists) return null;
|
|
var fullName = fi.FullName.ToLowerInvariant();
|
|
if (!fullName.Contains(_configService.Current.CacheFolder.ToLowerInvariant(), StringComparison.Ordinal)) return null;
|
|
string prefixedPath = fullName.Replace(_configService.Current.CacheFolder.ToLowerInvariant(), CachePrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
|
|
return CreateFileCacheEntity(fi, prefixedPath);
|
|
}
|
|
|
|
public FileCacheEntity? CreateFileEntry(string path)
|
|
{
|
|
_logger.LogTrace("Creating file entry for {path}", path);
|
|
FileInfo fi = new(path);
|
|
if (!fi.Exists) return null;
|
|
var fullName = fi.FullName.ToLowerInvariant();
|
|
if (!fullName.Contains(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), StringComparison.Ordinal)) return null;
|
|
string prefixedPath = fullName.Replace(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), PenumbraPrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
|
|
return CreateFileCacheEntity(fi, prefixedPath);
|
|
}
|
|
|
|
public List<FileCacheEntity> GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v).ToList();
|
|
|
|
public List<FileCacheEntity> GetAllFileCachesByHash(string hash)
|
|
{
|
|
List<FileCacheEntity> output = [];
|
|
if (_fileCaches.TryGetValue(hash, out var fileCacheEntities))
|
|
{
|
|
foreach (var filecache in fileCacheEntities.ToList())
|
|
{
|
|
var validated = GetValidatedFileCache(filecache);
|
|
if (validated != null) output.Add(validated);
|
|
}
|
|
}
|
|
|
|
return output;
|
|
}
|
|
|
|
public Task<List<FileCacheEntity>> ValidateLocalIntegrity(IProgress<(int, int, FileCacheEntity)> progress, CancellationToken cancellationToken)
|
|
{
|
|
_mareMediator.Publish(new HaltScanMessage(nameof(ValidateLocalIntegrity)));
|
|
_logger.LogInformation("Validating local storage");
|
|
var cacheEntries = _fileCaches.SelectMany(v => v.Value).Where(v => v.IsCacheEntry).ToList();
|
|
List<FileCacheEntity> brokenEntities = new();
|
|
int i = 0;
|
|
foreach (var fileCache in cacheEntries)
|
|
{
|
|
_logger.LogInformation("Validating {file}", fileCache.ResolvedFilepath);
|
|
|
|
progress.Report((i, cacheEntries.Count, fileCache));
|
|
i++;
|
|
var computedHash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
|
|
if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal))
|
|
{
|
|
_logger.LogInformation("Failed to validate {file}, got hash {hash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
|
|
brokenEntities.Add(fileCache);
|
|
}
|
|
|
|
if (cancellationToken.IsCancellationRequested) break;
|
|
}
|
|
|
|
foreach (var brokenEntity in brokenEntities)
|
|
{
|
|
RemoveHashedFile(brokenEntity.Hash, brokenEntity.PrefixedFilePath);
|
|
|
|
try
|
|
{
|
|
File.Delete(brokenEntity.ResolvedFilepath);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Could not delete {file}", brokenEntity.ResolvedFilepath);
|
|
}
|
|
}
|
|
|
|
_mareMediator.Publish(new ResumeScanMessage(nameof(ValidateLocalIntegrity)));
|
|
return Task.FromResult(brokenEntities);
|
|
}
|
|
|
|
public string GetCacheFilePath(string hash, string extension)
|
|
{
|
|
return Path.Combine(_configService.Current.CacheFolder, hash + "." + extension);
|
|
}
|
|
|
|
public async Task<long> GetCompressedFileLength(string fileHash, CancellationToken uploadToken)
|
|
{
|
|
var fileCache = GetFileCacheByHash(fileHash)!.ResolvedFilepath;
|
|
using var fs = File.OpenRead(fileCache);
|
|
var cs = new CountedStream(Stream.Null);
|
|
using var encstream = LZ4Stream.Encode(cs, new LZ4EncoderSettings(){CompressionLevel=K4os.Compression.LZ4.LZ4Level.L09_HC});
|
|
await fs.CopyToAsync(encstream, uploadToken).ConfigureAwait(false);
|
|
encstream.Close();
|
|
return uploadToken.IsCancellationRequested ? 0 : cs.BytesWritten;
|
|
}
|
|
|
|
public async Task<byte[]> GetCompressedFileData(string fileHash, CancellationToken uploadToken)
|
|
{
|
|
var fileCache = GetFileCacheByHash(fileHash)!.ResolvedFilepath;
|
|
using var fs = File.OpenRead(fileCache);
|
|
var ms = new MemoryStream(64 * 1024);
|
|
using var encstream = LZ4Stream.Encode(ms, new LZ4EncoderSettings(){CompressionLevel=K4os.Compression.LZ4.LZ4Level.L09_HC});
|
|
await fs.CopyToAsync(encstream, uploadToken).ConfigureAwait(false);
|
|
encstream.Close();
|
|
return ms.ToArray();
|
|
}
|
|
|
|
public FileCacheEntity? GetFileCacheByHash(string hash)
|
|
{
|
|
if (_fileCaches.TryGetValue(hash, out var hashes))
|
|
{
|
|
var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix) ? 0 : 1).FirstOrDefault();
|
|
if (item != null) return GetValidatedFileCache(item);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
public FileCacheEntity? GetFileCacheByPath(string path)
|
|
{
|
|
var cleanedPath = path.Replace("/", "\\", StringComparison.OrdinalIgnoreCase).ToLowerInvariant().Replace(_ipcManager.PenumbraModDirectory!.ToLowerInvariant(), "", StringComparison.OrdinalIgnoreCase);
|
|
var entry = _fileCaches.SelectMany(v => v.Value).FirstOrDefault(f => f.ResolvedFilepath.EndsWith(cleanedPath, StringComparison.OrdinalIgnoreCase));
|
|
|
|
if (entry == null)
|
|
{
|
|
_logger.LogDebug("Found no entries for {path}", cleanedPath);
|
|
return CreateFileEntry(path);
|
|
}
|
|
|
|
var validatedCacheEntry = GetValidatedFileCache(entry);
|
|
|
|
return validatedCacheEntry;
|
|
}
|
|
|
|
public Dictionary<string, FileCacheEntity?> GetFileCachesByPaths(string[] paths)
|
|
{
|
|
var cleanedPaths = paths.Distinct(StringComparer.OrdinalIgnoreCase).ToDictionary(p => p,
|
|
p => p.Replace("/", "\\", StringComparison.OrdinalIgnoreCase)
|
|
.Replace(_ipcManager.PenumbraModDirectory!,
|
|
_ipcManager.PenumbraModDirectory!.EndsWith('\\') ? (PenumbraPrefix + "\\") : PenumbraPrefix,
|
|
StringComparison.OrdinalIgnoreCase),
|
|
StringComparer.OrdinalIgnoreCase);
|
|
|
|
Dictionary<string, FileCacheEntity?> result = new(StringComparer.OrdinalIgnoreCase);
|
|
|
|
var dict = _fileCaches.SelectMany(f => f.Value).Where(f => f.PrefixedFilePath.StartsWith(PenumbraPrefix, StringComparison.Ordinal))
|
|
.ToDictionary(d => d.PrefixedFilePath, d => d, StringComparer.Ordinal);
|
|
|
|
foreach (var entry in cleanedPaths)
|
|
{
|
|
if (dict.TryGetValue(entry.Value, out var entity))
|
|
{
|
|
var validatedCache = GetValidatedFileCache(entity);
|
|
result.Add(entry.Key, validatedCache);
|
|
}
|
|
else
|
|
{
|
|
result.Add(entry.Key, CreateFileEntry(entry.Key));
|
|
}
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
public void RemoveHashedFile(string hash, string prefixedFilePath)
|
|
{
|
|
if (_fileCaches.TryGetValue(hash, out var caches))
|
|
{
|
|
caches?.RemoveAll(c => string.Equals(c.PrefixedFilePath, prefixedFilePath, StringComparison.Ordinal));
|
|
if (caches?.Count == 0)
|
|
{
|
|
_fileCaches.Remove(hash, out _);
|
|
}
|
|
}
|
|
}
|
|
|
|
public void UpdateHashedFile(FileCacheEntity fileCache, bool computeProperties = true)
|
|
{
|
|
_logger.LogTrace("Updating hash for {path}", fileCache.ResolvedFilepath);
|
|
var oldHash = fileCache.Hash;
|
|
var prefixedPath = fileCache.PrefixedFilePath;
|
|
if (computeProperties)
|
|
{
|
|
var fi = new FileInfo(fileCache.ResolvedFilepath);
|
|
fileCache.Size = fi.Length;
|
|
fileCache.CompressedSize = null;
|
|
fileCache.Hash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
|
|
fileCache.LastModifiedDateTicks = fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture);
|
|
}
|
|
RemoveHashedFile(oldHash, prefixedPath);
|
|
AddHashedFile(fileCache);
|
|
}
|
|
|
|
public (FileState State, FileCacheEntity FileCache) ValidateFileCacheEntity(FileCacheEntity fileCache)
|
|
{
|
|
fileCache = ReplacePathPrefixes(fileCache);
|
|
FileInfo fi = new(fileCache.ResolvedFilepath);
|
|
if (!fi.Exists)
|
|
{
|
|
return (FileState.RequireDeletion, fileCache);
|
|
}
|
|
if (!string.Equals(fi.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), fileCache.LastModifiedDateTicks, StringComparison.Ordinal))
|
|
{
|
|
return (FileState.RequireUpdate, fileCache);
|
|
}
|
|
|
|
return (FileState.Valid, fileCache);
|
|
}
|
|
|
|
public void WriteOutFullCsv()
|
|
{
|
|
lock (_fileWriteLock)
|
|
{
|
|
StringBuilder sb = new();
|
|
foreach (var entry in _fileCaches.SelectMany(k => k.Value).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
sb.AppendLine(entry.CsvEntry);
|
|
}
|
|
|
|
if (File.Exists(_csvPath))
|
|
{
|
|
File.Copy(_csvPath, CsvBakPath, overwrite: true);
|
|
}
|
|
|
|
try
|
|
{
|
|
File.WriteAllText(_csvPath, sb.ToString());
|
|
File.Delete(CsvBakPath);
|
|
}
|
|
catch
|
|
{
|
|
File.WriteAllText(CsvBakPath, sb.ToString());
|
|
}
|
|
}
|
|
}
|
|
|
|
internal FileCacheEntity MigrateFileHashToExtension(FileCacheEntity fileCache, string ext)
|
|
{
|
|
try
|
|
{
|
|
RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath);
|
|
FileInfo fileInfo = new(fileCache.ResolvedFilepath);
|
|
FileInfo oldCache = fileInfo;
|
|
var extensionPath = fileCache.ResolvedFilepath.ToUpper(CultureInfo.InvariantCulture) + "." + ext;
|
|
File.Move(fileCache.ResolvedFilepath, extensionPath, overwrite: true);
|
|
var newHashedEntity = new FileCacheEntity(fileCache.Hash, fileCache.PrefixedFilePath + "." + ext, DateTime.UtcNow.Ticks.ToString(CultureInfo.InvariantCulture));
|
|
newHashedEntity.SetResolvedFilePath(extensionPath);
|
|
AddHashedFile(newHashedEntity);
|
|
_logger.LogDebug("Migrated from {oldPath} to {newPath}", fileCache.ResolvedFilepath, newHashedEntity.ResolvedFilepath);
|
|
return newHashedEntity;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
AddHashedFile(fileCache);
|
|
_logger.LogWarning(ex, "Failed to migrate entity {entity}", fileCache.PrefixedFilePath);
|
|
return fileCache;
|
|
}
|
|
}
|
|
|
|
private void AddHashedFile(FileCacheEntity fileCache)
|
|
{
|
|
if (!_fileCaches.TryGetValue(fileCache.Hash, out var entries) || entries is null)
|
|
{
|
|
_fileCaches[fileCache.Hash] = entries = [];
|
|
}
|
|
|
|
if (!entries.Exists(u => string.Equals(u.PrefixedFilePath, fileCache.PrefixedFilePath, StringComparison.OrdinalIgnoreCase)))
|
|
{
|
|
entries.Add(fileCache);
|
|
}
|
|
}
|
|
|
|
private FileCacheEntity? CreateFileCacheEntity(FileInfo fileInfo, string prefixedPath, string? hash = null)
|
|
{
|
|
hash ??= Crypto.GetFileHash(fileInfo.FullName);
|
|
var entity = new FileCacheEntity(hash, prefixedPath, fileInfo.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), fileInfo.Length);
|
|
entity = ReplacePathPrefixes(entity);
|
|
AddHashedFile(entity);
|
|
lock (_fileWriteLock)
|
|
{
|
|
File.AppendAllLines(_csvPath, new[] { entity.CsvEntry });
|
|
}
|
|
var result = GetFileCacheByPath(fileInfo.FullName);
|
|
_logger.LogDebug("Creating file cache for {name} success: {success}", fileInfo.FullName, (result != null));
|
|
return result;
|
|
}
|
|
|
|
private FileCacheEntity? GetValidatedFileCache(FileCacheEntity fileCache)
|
|
{
|
|
var resulingFileCache = ReplacePathPrefixes(fileCache);
|
|
resulingFileCache = Validate(resulingFileCache);
|
|
return resulingFileCache;
|
|
}
|
|
|
|
private FileCacheEntity ReplacePathPrefixes(FileCacheEntity fileCache)
|
|
{
|
|
if (fileCache.PrefixedFilePath.StartsWith(PenumbraPrefix, StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(PenumbraPrefix, _ipcManager.PenumbraModDirectory, StringComparison.Ordinal));
|
|
}
|
|
else if (fileCache.PrefixedFilePath.StartsWith(CachePrefix, StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
fileCache.SetResolvedFilePath(fileCache.PrefixedFilePath.Replace(CachePrefix, _configService.Current.CacheFolder, StringComparison.Ordinal));
|
|
}
|
|
|
|
return fileCache;
|
|
}
|
|
|
|
private FileCacheEntity? Validate(FileCacheEntity fileCache)
|
|
{
|
|
var file = new FileInfo(fileCache.ResolvedFilepath);
|
|
if (!file.Exists)
|
|
{
|
|
RemoveHashedFile(fileCache.Hash, fileCache.PrefixedFilePath);
|
|
return null;
|
|
}
|
|
|
|
if (!string.Equals(file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture), fileCache.LastModifiedDateTicks, StringComparison.Ordinal))
|
|
{
|
|
UpdateHashedFile(fileCache);
|
|
}
|
|
|
|
return fileCache;
|
|
}
|
|
|
|
public Task StartAsync(CancellationToken cancellationToken)
|
|
{
|
|
lock (_fileWriteLock)
|
|
{
|
|
try
|
|
{
|
|
if (File.Exists(CsvBakPath))
|
|
{
|
|
File.Move(CsvBakPath, _csvPath, overwrite: true);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Failed to move BAK to ORG, deleting BAK");
|
|
try
|
|
{
|
|
if (File.Exists(CsvBakPath))
|
|
File.Delete(CsvBakPath);
|
|
}
|
|
catch (Exception ex1)
|
|
{
|
|
_logger.LogWarning(ex1, "Could not delete bak file");
|
|
}
|
|
}
|
|
}
|
|
|
|
if (File.Exists(_csvPath))
|
|
{
|
|
bool success = false;
|
|
string[] entries = [];
|
|
int attempts = 0;
|
|
while (!success && attempts < 10)
|
|
{
|
|
try
|
|
{
|
|
entries = File.ReadAllLines(_csvPath);
|
|
success = true;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
attempts++;
|
|
_logger.LogWarning(ex, "Could not open {file}, trying again", _csvPath);
|
|
Thread.Sleep(100);
|
|
}
|
|
}
|
|
|
|
if (!entries.Any())
|
|
{
|
|
_logger.LogWarning("Could not load entries from {path}, continuing with empty file cache", _csvPath);
|
|
}
|
|
|
|
Dictionary<string, bool> processedFiles = new(StringComparer.OrdinalIgnoreCase);
|
|
foreach (var entry in entries)
|
|
{
|
|
var splittedEntry = entry.Split(CsvSplit, StringSplitOptions.None);
|
|
try
|
|
{
|
|
var hash = splittedEntry[0];
|
|
if (hash.Length != 40) throw new InvalidOperationException("Expected Hash length of 40, received " + hash.Length);
|
|
var path = splittedEntry[1];
|
|
var time = splittedEntry[2];
|
|
|
|
if (processedFiles.ContainsKey(path))
|
|
{
|
|
_logger.LogWarning("Already processed {file}, ignoring", path);
|
|
continue;
|
|
}
|
|
|
|
processedFiles.Add(path, value: true);
|
|
|
|
long size = -1;
|
|
long compressed = -1;
|
|
if (splittedEntry.Length > 3)
|
|
{
|
|
if (long.TryParse(splittedEntry[3], CultureInfo.InvariantCulture, out long result))
|
|
{
|
|
size = result;
|
|
}
|
|
if (long.TryParse(splittedEntry[4], CultureInfo.InvariantCulture, out long resultCompressed))
|
|
{
|
|
compressed = resultCompressed;
|
|
}
|
|
}
|
|
AddHashedFile(ReplacePathPrefixes(new FileCacheEntity(hash, path, time, size, compressed)));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Failed to initialize entry {entry}, ignoring", entry);
|
|
}
|
|
}
|
|
|
|
if (processedFiles.Count != entries.Length)
|
|
{
|
|
WriteOutFullCsv();
|
|
}
|
|
}
|
|
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
public Task StopAsync(CancellationToken cancellationToken)
|
|
{
|
|
WriteOutFullCsv();
|
|
return Task.CompletedTask;
|
|
}
|
|
} |