From f512552fe74ea19a10dae657c69e42a88a02b219 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Sat, 6 May 2023 02:41:17 +0200 Subject: [PATCH] add migration to file extensions for mare storage --- MareSynchronos/FileCache/FileCacheManager.cs | 38 ++++++++++++++++--- .../PlayerData/Pairs/CachedPlayer.cs | 8 ++++ .../WebAPI/Files/FileDownloadManager.cs | 5 ++- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/MareSynchronos/FileCache/FileCacheManager.cs b/MareSynchronos/FileCache/FileCacheManager.cs index 808286e..ec14e6b 100644 --- a/MareSynchronos/FileCache/FileCacheManager.cs +++ b/MareSynchronos/FileCache/FileCacheManager.cs @@ -105,9 +105,9 @@ public sealed class FileCacheManager : IDisposable public List GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v).ToList(); - public string GetCacheFilePath(string hash, bool isTemporaryFile) + public string GetCacheFilePath(string hash, string extension, bool isTemporaryFile) { - return Path.Combine(_configService.Current.CacheFolder, hash + (isTemporaryFile ? ".tmp" : string.Empty)); + return Path.Combine(_configService.Current.CacheFolder, hash + "." + extension + (isTemporaryFile ? ".tmp" : string.Empty)); } public FileCacheEntity? GetFileCacheByHash(string hash) @@ -181,12 +181,13 @@ public sealed class FileCacheManager : IDisposable { sb.AppendLine(entry.CsvEntry); } - if (File.Exists(_csvPath)) - { - File.Copy(_csvPath, CsvBakPath, overwrite: true); - } lock (_fileWriteLock) { + if (File.Exists(_csvPath)) + { + File.Copy(_csvPath, CsvBakPath, overwrite: true); + } + try { File.WriteAllText(_csvPath, sb.ToString()); @@ -199,6 +200,31 @@ public sealed class FileCacheManager : IDisposable } } + internal FileCacheEntity MigrateFileHashToExtension(FileCacheEntity fileCache, string ext) + { + try + { + RemoveHashedFile(fileCache); + FileInfo oldCache = new(fileCache.ResolvedFilepath); + var extensionPath = fileCache.ResolvedFilepath.ToUpper() + "." + ext; + File.Move(fileCache.ResolvedFilepath, extensionPath, true); + var newHashedEntity = new FileCacheEntity(fileCache.Hash, fileCache.PrefixedFilePath + "." + ext, DateTime.UtcNow.Ticks.ToString()); + newHashedEntity.SetResolvedFilePath(extensionPath); + FileInfo newCache = new FileInfo(extensionPath); + newCache.LastAccessTime = oldCache.LastAccessTime; + newCache.LastWriteTime = oldCache.LastWriteTime; + AddHashedFile(newHashedEntity); + _logger.LogDebug("Migrated from {oldPath} to {newPath}", fileCache.ResolvedFilepath, newHashedEntity.ResolvedFilepath); + return newHashedEntity; + } + catch (Exception ex) + { + AddHashedFile(fileCache); + _logger.LogWarning(ex, "Failed to migrate entity {entity}", fileCache.PrefixedFilePath); + return fileCache; + } + } + private void AddHashedFile(FileCacheEntity fileCache) { if (!_fileCaches.TryGetValue(fileCache.Hash, out var entries)) diff --git a/MareSynchronos/PlayerData/Pairs/CachedPlayer.cs b/MareSynchronos/PlayerData/Pairs/CachedPlayer.cs index dd13db0..ed6332f 100644 --- a/MareSynchronos/PlayerData/Pairs/CachedPlayer.cs +++ b/MareSynchronos/PlayerData/Pairs/CachedPlayer.cs @@ -665,6 +665,7 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase List missingFiles = new(); moddedDictionary = new Dictionary(StringComparer.Ordinal); ConcurrentDictionary outputDict = new(StringComparer.Ordinal); + bool hasMigrationChanges = false; try { var replacementList = charaData.FileReplacements.SelectMany(k => k.Value.Where(v => string.IsNullOrEmpty(v.FileSwapPath))).ToList(); @@ -679,6 +680,12 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase var fileCache = _fileDbManager.GetFileCacheByHash(item.Hash); if (fileCache != null) { + if (string.IsNullOrEmpty(new FileInfo(fileCache.ResolvedFilepath).Extension)) + { + hasMigrationChanges = true; + fileCache = _fileDbManager.MigrateFileHashToExtension(fileCache, item.GamePaths[0].Split(".").Last()); + } + foreach (var gamePath in item.GamePaths) { outputDict[gamePath] = fileCache.ResolvedFilepath; @@ -706,6 +713,7 @@ public sealed class CachedPlayer : DisposableMediatorSubscriberBase { PluginLog.Error(ex, "Something went wrong during calculation replacements"); } + if (hasMigrationChanges) _fileDbManager.WriteOutFullCsv(); st.Stop(); Logger.LogDebug("ModdedPaths calculated in {time}ms, missing files: {count}, total files: {total}", st.ElapsedMilliseconds, missingFiles.Count, moddedDictionary.Keys.Count); return missingFiles; diff --git a/MareSynchronos/WebAPI/Files/FileDownloadManager.cs b/MareSynchronos/WebAPI/Files/FileDownloadManager.cs index b107343..26b7f9e 100644 --- a/MareSynchronos/WebAPI/Files/FileDownloadManager.cs +++ b/MareSynchronos/WebAPI/Files/FileDownloadManager.cs @@ -191,7 +191,8 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase foreach (var file in fileGroup) { - var tempPath = _fileDbManager.GetCacheFilePath(file.Hash, isTemporaryFile: true); + var ext = fileReplacement.First(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase)).GamePaths.First().Split(".").Last(); + var tempPath = _fileDbManager.GetCacheFilePath(file.Hash, ext, isTemporaryFile: true); Progress progress = new((bytesDownloaded) => { try @@ -235,7 +236,7 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase var tempFileData = await File.ReadAllBytesAsync(tempPath, token).ConfigureAwait(false); var extractedFile = LZ4Codec.Unwrap(tempFileData); File.Delete(tempPath); - var filePath = _fileDbManager.GetCacheFilePath(file.Hash, isTemporaryFile: false); + var filePath = _fileDbManager.GetCacheFilePath(file.Hash, ext, isTemporaryFile: false); await File.WriteAllBytesAsync(filePath, extractedFile, token).ConfigureAwait(false); var fi = new FileInfo(filePath); Func RandomDayInThePast()