t0w0bi/ClubPenguinClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rework file cache management, force usage of websocket, add uid comments per server

Browse Source
This commit is contained in:
Stanley Dimant
2022-06-28 23:45:49 +02:00
parent 3ee082d371
commit 71d48b0d8c
7 changed files with 203 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
MareSynchronos/Configuration.cs
View File

@@ -3,11 +3,40 @@ using Dalamud.Plugin;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using MareSynchronos.Utils;
using MareSynchronos.WebAPI;
using Newtonsoft.Json;
namespace MareSynchronos
{
public static class ConfigurationExtensions
{
public static bool HasValidSetup(this Configuration configuration)
{
return configuration.AcceptedAgreement && configuration.InitialScanComplete
&& !string.IsNullOrEmpty(configuration.CacheFolder)
&& Directory.Exists(configuration.CacheFolder)
&& configuration.ClientSecret.ContainsKey(configuration.ApiUri);
}
public static Dictionary<string, string> GetCurrentServerUidComments(this Configuration configuration)
{
return configuration.UidServerComments.ContainsKey(configuration.ApiUri)
? configuration.UidServerComments[configuration.ApiUri]
: new Dictionary<string, string>();
}
public static void SetCurrentServerUidComment(this Configuration configuration, string uid, string comment)
{
if (!configuration.UidServerComments.ContainsKey(configuration.ApiUri))
{
configuration.UidServerComments[configuration.ApiUri] = new Dictionary<string, string>();
}
configuration.UidServerComments[configuration.ApiUri][uid] = comment;
}
}
[Serializable]
public class Configuration : IPluginConfiguration
{
@@ -26,9 +55,6 @@ namespace MareSynchronos
public string CacheFolder { get; set; } = string.Empty;
public Dictionary<string, string> ClientSecret { get; set; } = new();
public Dictionary<string, string> CustomServerList { get; set; } = new();
[JsonIgnore]
public bool HasValidSetup => AcceptedAgreement && InitialScanComplete && !string.IsNullOrEmpty(CacheFolder) &&
Directory.Exists(CacheFolder) && ClientSecret.ContainsKey(ApiUri);
public bool InitialScanComplete { get; set; } = false;
public int MaxParallelScan
@@ -46,8 +72,10 @@ namespace MareSynchronos
}
public bool FullPause { get; set; } = false;
public Dictionary<string, Dictionary<string, string>> UidServerComments { get; set; } = new();
public Dictionary<string, string> UidComments { get; set; } = new();
public int Version { get; set; } = 0;
public int Version { get; set; } = 1;
public bool ShowTransferWindow { get; set; } = true;
@@ -61,5 +89,31 @@ namespace MareSynchronos
{
_pluginInterface!.SavePluginConfig(this);
}
public void Migrate()
{
if (Version == 0)
{
Logger.Debug("Migrating Configuration from V0 to V1");
Version = 1;
ApiUri = ApiUri.Replace("https", "wss");
foreach (var kvp in ClientSecret.ToList())
{
var newKey = kvp.Key.Replace("https", "wss");
ClientSecret.Remove(kvp.Key);
if (ClientSecret.ContainsKey(newKey))
{
ClientSecret[newKey] = kvp.Value;
}
else
{
ClientSecret.Add(newKey, kvp.Value);
}
}
UidServerComments.Add(ApiUri, UidComments.ToDictionary(k => k.Key, k => k.Value));
UidComments.Clear();
Save();
}
}
}
}

170
MareSynchronos/Managers/FileCacheManager.cs
View File

@@ -14,9 +14,13 @@ namespace MareSynchronos.Managers
public class FileCacheManager : IDisposable
{
private readonly IpcManager _ipcManager;
private readonly ConcurrentBag<string> _modifiedFiles = new();
private readonly Configuration _pluginConfiguration;
private FileSystemWatcher? _cacheDirWatcher;
private FileSystemWatcher? _penumbraDirWatcher;
private Task? _rescanTask;
private CancellationTokenSource? _rescanTaskCancellationTokenSource;
private CancellationTokenSource? _rescanTaskRunCancellationTokenSource;
private CancellationTokenSource? _scanCancellationTokenSource;
private Task? _scanTask;
public FileCacheManager(IpcManager ipcManager, Configuration pluginConfiguration)
@@ -32,26 +36,10 @@ namespace MareSynchronos.Managers
_ipcManager.PenumbraDisposed += IpcManagerOnPenumbraDisposed;
}
private void StartWatchersAndScan()
{
if (!_ipcManager.Initialized || !_pluginConfiguration.HasValidSetup) return;
Logger.Debug("Penumbra is active, configuration is valid, starting watchers and scan");
StartWatchers();
StartInitialScan();
}
private void IpcManagerOnPenumbraInitialized(object? sender, EventArgs e)
{
StartWatchersAndScan();
}
private void IpcManagerOnPenumbraDisposed(object? sender, EventArgs e)
{
StopWatchersAndScan();
}
public long CurrentFileProgress { get; private set; }
public long FileCacheSize { get; set; }
public bool IsScanRunning => !_scanTask?.IsCompleted ?? false;
public long TotalFiles { get; private set; }
@@ -83,17 +71,12 @@ namespace MareSynchronos.Managers
_ipcManager.PenumbraInitialized -= IpcManagerOnPenumbraInitialized;
_ipcManager.PenumbraDisposed -= IpcManagerOnPenumbraDisposed;
_rescanTaskCancellationTokenSource?.Cancel();
_rescanTaskRunCancellationTokenSource?.Cancel();
StopWatchersAndScan();
}
private void StopWatchersAndScan()
{
_cacheDirWatcher?.Dispose();
_penumbraDirWatcher?.Dispose();
_scanCancellationTokenSource?.Cancel();
}
public void StartInitialScan()
{
_scanCancellationTokenSource = new CancellationTokenSource();
@@ -102,8 +85,8 @@ namespace MareSynchronos.Managers
public void StartWatchers()
{
if (!_ipcManager.Initialized || !_pluginConfiguration.HasValidSetup) return;
Logger.Debug("Starting File System Watchers");
if (!_ipcManager.Initialized || !_pluginConfiguration.HasValidSetup()) return;
Logger.Verbose("Starting File System Watchers");
_penumbraDirWatcher?.Dispose();
_cacheDirWatcher?.Dispose();
@@ -113,8 +96,9 @@ namespace MareSynchronos.Managers
InternalBufferSize = 1048576
};
_penumbraDirWatcher.NotifyFilter = NotifyFilters.FileName | NotifyFilters.Size;
_penumbraDirWatcher.Deleted += OnDeleted;
_penumbraDirWatcher.Deleted += OnModified;
_penumbraDirWatcher.Changed += OnModified;
_penumbraDirWatcher.Renamed += OnModified;
_penumbraDirWatcher.Filters.Add("*.mtrl");
_penumbraDirWatcher.Filters.Add("*.mdl");
_penumbraDirWatcher.Filters.Add("*.tex");
@@ -127,8 +111,9 @@ namespace MareSynchronos.Managers
InternalBufferSize = 1048576
};
_cacheDirWatcher.NotifyFilter = NotifyFilters.FileName | NotifyFilters.Size;
_cacheDirWatcher.Deleted += OnDeleted;
_cacheDirWatcher.Deleted += OnModified;
_cacheDirWatcher.Changed += OnModified;
_cacheDirWatcher.Renamed += OnModified;
_cacheDirWatcher.Filters.Add("*.mtrl");
_cacheDirWatcher.Filters.Add("*.mdl");
_cacheDirWatcher.Filters.Add("*.tex");
@@ -139,6 +124,16 @@ namespace MareSynchronos.Managers
Task.Run(RecalculateFileCacheSize);
}
private void IpcManagerOnPenumbraDisposed(object? sender, EventArgs e)
{
StopWatchersAndScan();
}
private void IpcManagerOnPenumbraInitialized(object? sender, EventArgs e)
{
StartWatchersAndScan();
}
private bool IsFileLocked(FileInfo file)
{
try
@@ -153,62 +148,10 @@ namespace MareSynchronos.Managers
return false;
}
private void OnDeleted(object sender, FileSystemEventArgs e)
{
var fi = new FileInfo(e.FullPath);
using var db = new FileCacheContext();
var ext = fi.Extension.ToLower();
if (ext is ".mdl" or ".tex" or ".mtrl")
{
Logger.Debug("File deleted: " + e.FullPath);
var fileInDb = db.FileCaches.SingleOrDefault(f => f.Filepath == fi.FullName.ToLower());
if (fileInDb == null) return;
db.Remove(fileInDb);
}
else
{
if (fi.Extension == string.Empty)
{
// this is most likely a folder
var filesToRemove = db.FileCaches.Where(f => f.Filepath.StartsWith(e.FullPath.ToLower())).ToList();
Logger.Debug($"Folder deleted: {e.FullPath}, removing {filesToRemove.Count} files");
db.RemoveRange(filesToRemove);
}
}
db.SaveChanges();
if (e.FullPath.Contains(_pluginConfiguration.CacheFolder, StringComparison.OrdinalIgnoreCase))
{
Task.Run(RecalculateFileCacheSize);
}
}
private void OnModified(object sender, FileSystemEventArgs e)
{
var fi = new FileInfo(e.FullPath);
Logger.Debug("File changed: " + e.FullPath);
using var db = new FileCacheContext();
var modifiedFile = Create(fi.FullName);
var fileInDb = db.FileCaches.SingleOrDefault(f => f.Filepath == fi.FullName.ToLower());
if (fileInDb != null)
db.Remove(fileInDb);
else
{
var files = db.FileCaches.Where(f => f.Hash == modifiedFile.Hash);
foreach (var file in files)
{
if (!File.Exists(file.Filepath)) db.Remove(file.Filepath);
}
}
db.Add(modifiedFile);
db.SaveChanges();
if (e.FullPath.Contains(_pluginConfiguration.CacheFolder, StringComparison.OrdinalIgnoreCase))
{
Task.Run(RecalculateFileCacheSize);
}
_modifiedFiles.Add(e.FullPath);
Task.Run(() => _ = RescanTask());
}
private void RecalculateFileCacheSize()
@@ -227,6 +170,55 @@ namespace MareSynchronos.Managers
}
}
public async Task RescanTask(bool force = false)
{
_rescanTaskRunCancellationTokenSource?.Cancel();
_rescanTaskRunCancellationTokenSource = new CancellationTokenSource();
var token = _rescanTaskRunCancellationTokenSource.Token;
if(!force)
await Task.Delay(TimeSpan.FromSeconds(1), token);
while ((!_rescanTask?.IsCompleted ?? false) && !token.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(1), token);
}
if (token.IsCancellationRequested) return;
PluginLog.Debug("File changes detected, scanning the changes");
if (!_modifiedFiles.Any()) return;
_rescanTaskCancellationTokenSource = new CancellationTokenSource();
_rescanTask = Task.Run(async () =>
{
var listCopy = _modifiedFiles.ToList();
_modifiedFiles.Clear();
await using var db = new FileCacheContext();
foreach (var item in listCopy.Distinct())
{
var fi = new FileInfo(item);
if (!fi.Exists)
{
PluginLog.Verbose("Removed: " + item);
db.RemoveRange(db.FileCaches.Where(f => f.Filepath.ToLower() == item.ToLower()));
}
else
{
PluginLog.Verbose("Changed :" + item);
var fileCache = Create(item);
db.RemoveRange(db.FileCaches.Where(f => f.Hash == fileCache.Hash));
await db.AddAsync(fileCache, _rescanTaskCancellationTokenSource.Token);
}
}
await db.SaveChangesAsync(_rescanTaskCancellationTokenSource.Token);
RecalculateFileCacheSize();
}, _rescanTaskCancellationTokenSource.Token);
}
private async Task StartFileScan(CancellationToken ct)
{
_scanCancellationTokenSource = new CancellationTokenSource();
@@ -341,5 +333,17 @@ namespace MareSynchronos.Managers
_pluginConfiguration.Save();
}
}
private void StartWatchersAndScan()
{
if (!_ipcManager.Initialized || !_pluginConfiguration.HasValidSetup()) return;
Logger.Verbose("Penumbra is active, configuration is valid, starting watchers and scan");
StartWatchers();
StartInitialScan();
}
private void StopWatchersAndScan()
{
_cacheDirWatcher?.Dispose();
}
}
}

1
MareSynchronos/MareSynchronos.csproj
View File

@@ -29,7 +29,6 @@
<PackageReference Include="lz4net" Version="1.0.15.93" />
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="6.0.6" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.17" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
</ItemGroup>
<ItemGroup>

5
MareSynchronos/Plugin.cs
View File

@@ -47,6 +47,7 @@ namespace MareSynchronos
_clientState = clientState;
_configuration = PluginInterface.GetPluginConfig() as Configuration ?? new Configuration();
_configuration.Initialize(PluginInterface);
_configuration.Migrate();
_windowSystem = new WindowSystem("MareSynchronos");
@@ -123,7 +124,7 @@ namespace MareSynchronos
HelpMessage = "Opens the Mare Synchronos UI"
});
if (!_configuration.HasValidSetup)
if (!_configuration.HasValidSetup())
{
_introUi.IsOpen = true;
return;
@@ -188,7 +189,7 @@ namespace MareSynchronos
private void OpenConfigUi()
{
if (_configuration.HasValidSetup)
if (_configuration.HasValidSetup())
_pluginUi.Toggle();
else
_introUi.Toggle();

26
MareSynchronos/UI/PluginUI.cs
View File

@@ -4,6 +4,7 @@ using Dalamud.Interface.Windowing;
using ImGuiNET;
using MareSynchronos.WebAPI;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Numerics;
@@ -179,7 +180,7 @@ namespace MareSynchronos.UI
var marePaused = _configuration.FullPause;
if (_configuration.HasValidSetup)
if (_configuration.HasValidSetup())
{
if (ImGui.Checkbox("Pause Mare Synchronos", ref marePaused))
{
@@ -299,6 +300,8 @@ namespace MareSynchronos.UI
{
File.Delete(file);
}
_uiShared.ForceRescan();
});
}
ImGui.TreePop();
@@ -340,11 +343,16 @@ namespace MareSynchronos.UI
: ((item.IsPaused || item.IsPausedFromOthers) ? "Unpaired" : "Paired");
ImGui.TextColored(UiShared.GetBoolColor(item.IsSynced && !item.IsPaused && !item.IsPausedFromOthers), pairString);
ImGui.TableNextColumn();
string charComment = _configuration.UidComments.ContainsKey(item.OtherUID) ? _configuration.UidComments[item.OtherUID] : string.Empty;
string charComment = _configuration.GetCurrentServerUidComments().ContainsKey(item.OtherUID) ? _configuration.GetCurrentServerUidComments()[item.OtherUID] : string.Empty;
ImGui.SetNextItemWidth(400);
if (ImGui.InputTextWithHint("##comment" + item.OtherUID, "Add your comment here (comments will not be synced)", ref charComment, 255))
{
_configuration.UidComments[item.OtherUID] = charComment;
if (_configuration.GetCurrentServerUidComments().Count == 0)
{
_configuration.UidServerComments[_configuration.ApiUri] =
new Dictionary<string, string>();
}
_configuration.SetCurrentServerUidComment(item.OtherUID, charComment);
_configuration.Save();
}
ImGui.TableNextColumn();
@@ -362,21 +370,21 @@ namespace MareSynchronos.UI
ImGui.EndTable();
}
var pairedClientEntry = tempNameUID;
var pairedClientEntry = _tempNameUID;
ImGui.SetNextItemWidth(200);
if (ImGui.InputText("UID", ref pairedClientEntry, 20))
{
tempNameUID = pairedClientEntry;
_tempNameUID = pairedClientEntry;
}
ImGui.SameLine();
ImGui.PushFont(UiBuilder.IconFont);
if (ImGui.Button(FontAwesomeIcon.Plus.ToIconString() + "##addToPairedClients"))
{
if (_apiController.PairedClients.All(w => w.OtherUID != tempNameUID))
if (_apiController.PairedClients.All(w => w.OtherUID != _tempNameUID))
{
var nameToSend = tempNameUID;
tempNameUID = string.Empty;
var nameToSend = _tempNameUID;
_tempNameUID = string.Empty;
_ = _apiController.SendPairedClientAddition(nameToSend);
}
}
@@ -386,6 +394,6 @@ namespace MareSynchronos.UI
}
}
private string tempNameUID = string.Empty;
private string _tempNameUID = string.Empty;
}
}

20
MareSynchronos/UI/UIShared.cs
View File

@@ -54,6 +54,11 @@ namespace MareSynchronos.UI
return true;
}
public void ForceRescan()
{
Task.Run(() => _ = _fileCacheManager.RescanTask(true));
}
public void DrawFileScanState()
{
ImGui.Text("File Scanner Status");
@@ -78,10 +83,21 @@ namespace MareSynchronos.UI
var serverName = _apiController.ServerDictionary.ContainsKey(_pluginConfiguration.ApiUri)
? _apiController.ServerDictionary[_pluginConfiguration.ApiUri]
: _pluginConfiguration.ApiUri;
ImGui.Text("Service status of " + serverName);
ImGui.Text("Service status of \"" + serverName + "\":");
ImGui.SameLine();
var color = _apiController.ServerAlive ? ImGuiColors.ParsedGreen : ImGuiColors.DalamudRed;
ImGui.TextColored(color, _apiController.ServerAlive ? "Available" : "Unavailable");
if (_apiController.ServerAlive)
{
ImGui.SameLine();
ImGui.TextUnformatted("(");
ImGui.SameLine();
ImGui.TextColored(ImGuiColors.ParsedGreen, _apiController.OnlineUsers.ToString());
ImGui.SameLine();
ImGui.Text("Users Online (server-wide)");
ImGui.SameLine();
ImGui.Text(")");
}
}
public static void TextWrapped(string text)
@@ -209,7 +225,7 @@ namespace MareSynchronos.UI
ImGui.InputText("Custom Service Address", ref _customServerUri, 255);
if (ImGui.Button("Add Custom Service"))
{
if (!string.IsNullOrEmpty(_customServerUri)
if (!string.IsNullOrEmpty(_customServerUri)
&& !string.IsNullOrEmpty(_customServerName)
&& !_pluginConfiguration.CustomServerList.ContainsValue(_customServerName))
{

32
MareSynchronos/WebAPI/ApiController.cs
View File

@@ -1,5 +1,4 @@
using Dalamud.Logging;
using System;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
@@ -12,8 +11,8 @@ using LZ4;
using MareSynchronos.API;
using MareSynchronos.FileCacheDB;
using MareSynchronos.Utils;
using Microsoft.AspNetCore.Http.Connections;
using Microsoft.AspNetCore.SignalR.Client;
using Microsoft.EntityFrameworkCore.Metadata.Conventions;
namespace MareSynchronos.WebAPI
{
@@ -21,7 +20,7 @@ namespace MareSynchronos.WebAPI
{
#if DEBUG
public const string MainServer = "darkarchons Debug Server (Dev Server (CH))";
public const string MainServiceUri = "https://darkarchon.internet-box.ch:5001";
public const string MainServiceUri = "wss://darkarchon.internet-box.ch:5001";
#else
public const string MainServer = "Lunae Crescere Incipientis (Central Server EU)";
public const string MainServiceUri = "to be defined";
@@ -88,12 +87,14 @@ namespace MareSynchronos.WebAPI
public string UID { get; private set; } = string.Empty;
private string ApiUri => _pluginConfiguration.ApiUri;
public int OnlineUsers { get; private set; }
public async Task CreateConnections()
{
await StopAllConnections(_cts.Token);
_cts = new CancellationTokenSource();
var token = _cts.Token;
await StopAllConnections(token);
while (!ServerAlive && !token.IsCancellationRequested)
{
@@ -110,11 +111,14 @@ namespace MareSynchronos.WebAPI
await _userHub.StartAsync(token);
await _fileHub.StartAsync(token);
OnlineUsers = await _userHub.InvokeAsync<int>("GetOnlineUsers", token);
if (_pluginConfiguration.FullPause)
{
UID = string.Empty;
return;
}
UID = await _heartbeatHub.InvokeAsync<string>("Heartbeat", token);
if (!string.IsNullOrEmpty(UID) && !token.IsCancellationRequested) // user is authorized
{
@@ -125,6 +129,7 @@ namespace MareSynchronos.WebAPI
(s) => PairedClientOffline?.Invoke(s, EventArgs.Empty));
_userHub.On<string>("AddOnlinePairedPlayer",
(s) => PairedClientOnline?.Invoke(s, EventArgs.Empty));
_userHub.On<int>("UsersOnline", (count) => OnlineUsers = count);
PairedClients = await _userHub!.InvokeAsync<List<ClientPairDto>>("GetPairedClients", token);
@@ -161,13 +166,12 @@ namespace MareSynchronos.WebAPI
{
options.Headers.Add("Authorization", SecretKey);
}
options.Transports = HttpTransportType.WebSockets;
#if DEBUG
options.HttpMessageHandlerFactory = (message) =>
options.HttpMessageHandlerFactory = (message) => new HttpClientHandler()
{
if (message is HttpClientHandler clientHandler)
clientHandler.ServerCertificateCustomValidationCallback +=
(sender, certificate, chain, sslPolicyErrors) => true;
return message;
ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator
};
#endif
})
@@ -185,6 +189,8 @@ namespace MareSynchronos.WebAPI
private Task HeartbeatHubOnReconnected(string? arg)
{
Logger.Debug("Connection restored");
OnlineUsers = _userHub!.InvokeAsync<int>("GetOnlineUsers").Result;
UID = _heartbeatHub!.InvokeAsync<string>("Heartbeat").Result;
Connected?.Invoke(this, EventArgs.Empty);
return Task.CompletedTask;
}
@@ -198,7 +204,7 @@ namespace MareSynchronos.WebAPI
private async Task StopAllConnections(CancellationToken token)
{
if (_heartbeatHub is { State: HubConnectionState.Connected })
if (_heartbeatHub is { State: HubConnectionState.Connected or HubConnectionState.Connecting or HubConnectionState.Reconnecting })
{
await _heartbeatHub.StopAsync(token);
_heartbeatHub.Closed -= HeartbeatHubOnClosed;
@@ -207,13 +213,13 @@ namespace MareSynchronos.WebAPI
await _heartbeatHub.DisposeAsync();
}
if (_fileHub is { State: HubConnectionState.Connected })
if (_fileHub is { State: HubConnectionState.Connected or HubConnectionState.Connecting or HubConnectionState.Reconnecting })
{
await _fileHub.StopAsync(token);
await _fileHub.DisposeAsync();
}
if (_userHub is { State: HubConnectionState.Connected })
if (_userHub is { State: HubConnectionState.Connected or HubConnectionState.Connecting or HubConnectionState.Reconnecting })
{
await _userHub.StopAsync(token);
await _userHub.DisposeAsync();