From 17c69666e845730142305f0d06c5672cba2db21a Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Fri, 15 Jul 2022 02:04:04 +0200 Subject: [PATCH] add prometheus --- .../FileCleanupService.cs | 12 ++--- .../Hubs/MareHub.Files.cs | 20 ++++++-- .../MareSynchronosServer/Hubs/MareHub.User.cs | 33 ++++++++++-- .../MareSynchronosServer/Hubs/MareHub.cs | 12 +++++ .../MareSynchronosServer/MareMetrics.cs | 50 +++++++++++++++++++ .../MareSynchronosServer.csproj | 4 +- .../MareSynchronosServer/Program.cs | 6 ++- .../MareSynchronosServer/Startup.cs | 7 +++ .../MareSynchronosServer/SystemInfoService.cs | 36 ++++--------- 9 files changed, 139 insertions(+), 41 deletions(-) create mode 100644 MareSynchronosServer/MareSynchronosServer/MareMetrics.cs diff --git a/MareSynchronosServer/MareSynchronosServer/FileCleanupService.cs b/MareSynchronosServer/MareSynchronosServer/FileCleanupService.cs index 578aac8..d096db0 100644 --- a/MareSynchronosServer/MareSynchronosServer/FileCleanupService.cs +++ b/MareSynchronosServer/MareSynchronosServer/FileCleanupService.cs @@ -1,11 +1,9 @@ using System; -using System.Data; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using MareSynchronosServer.Data; -using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -52,17 +50,18 @@ namespace MareSynchronosServer var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays)); - dbContext.Database.BeginTransaction(IsolationLevel.Snapshot); - var allFiles = dbContext.Files.Where(f => f.Uploaded); + var allFiles = dbContext.Files.Where(f => f.Uploaded).ToList(); foreach (var file in allFiles) { var fileName = Path.Combine(_configuration["CacheDirectory"], file.Hash); - if (!File.Exists(fileName)) + var fi = new FileInfo(fileName); + if (!fi.Exists) { _logger.LogInformation("File does not exist anymore: " + fileName); dbContext.Files.Remove(file); - } else if (new FileInfo(fileName).LastAccessTime < prevTime) + } else if (fi.LastAccessTime < prevTime) { + MareMetrics.FilesTotalSize.Dec(fi.Length); _logger.LogInformation("File outdated: " + fileName); dbContext.Files.Remove(file); File.Delete(fileName); @@ -72,7 +71,6 @@ namespace MareSynchronosServer _logger.LogInformation($"Cleanup complete"); dbContext.SaveChanges(); - dbContext.Database.CommitTransaction(); } catch { diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs index f32a7e7..ef7e91a 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs @@ -40,7 +40,13 @@ namespace MareSynchronosServer.Hubs var ownFiles = await _dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == AuthenticatedUserId).ToListAsync(); foreach (var file in ownFiles) { - File.Delete(Path.Combine(BasePath, file.Hash)); + var fi = new FileInfo(Path.Combine(BasePath, file.Hash)); + if (fi.Exists) + { + MareMetrics.FilesTotalSize.Dec(fi.Length); + MareMetrics.FilesTotal.Dec(); + fi.Delete(); + } } _dbContext.Files.RemoveRange(ownFiles); await _dbContext.SaveChangesAsync(); @@ -59,7 +65,8 @@ namespace MareSynchronosServer.Hubs int readByteCount; var buffer = new byte[chunkSize]; - await using var fs = File.Open(Path.Combine(BasePath, hash), FileMode.Open, FileAccess.Read, FileShare.Read); + var path = Path.Combine(BasePath, hash); + await using var fs = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read); while ((readByteCount = await fs.ReadAsync(buffer, 0, chunkSize, ct)) > 0) { await Task.Delay(10, ct); @@ -67,6 +74,9 @@ namespace MareSynchronosServer.Hubs } _logger.LogInformation("User " + AuthenticatedUserId + " finished downloading file: " + hash); + + MareMetrics.UserDownloadedFiles.Inc(); + MareMetrics.UserDownloadedFilesSize.Inc(new FileInfo(path).Length); } [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AuthScheme)] @@ -114,7 +124,7 @@ namespace MareSynchronosServer.Hubs return await _dbContext.Files.AsNoTracking() .AnyAsync(f => f.Uploader.UID == userUid && !f.Uploaded); } - + [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AuthScheme)] [HubMethodName(Api.InvokeFileSendFiles)] public async Task> SendFiles(List fileListHashes) @@ -204,6 +214,10 @@ namespace MareSynchronosServer.Hubs await File.WriteAllBytesAsync(Path.Combine(BasePath, hash), uploadedFile.ToArray()); relatedFile = _dbContext.Files.Single(f => f.Hash == hash); relatedFile.Uploaded = true; + + MareMetrics.FilesTotal.Inc(); + MareMetrics.FilesTotalSize.Inc(uploadedFile.Count); + await _dbContext.SaveChangesAsync(); _logger.LogInformation("File " + hash + " added to DB"); } diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.User.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.User.cs index 739b29e..98f9102 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.User.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.User.cs @@ -24,10 +24,15 @@ namespace MareSynchronosServer.Hubs string userid = AuthenticatedUserId; var userEntry = await _dbContext.Users.SingleAsync(u => u.UID == userid); - var ownPairData = _dbContext.ClientPairs.Where(u => u.User.UID == userid); + var ownPairData = await _dbContext.ClientPairs.Where(u => u.User.UID == userid).ToListAsync(); + + MareMetrics.Pairs.Dec(ownPairData.Count); + MareMetrics.PairsPaused.Dec(ownPairData.Count(c => c.IsPaused)); + _dbContext.RemoveRange(ownPairData); await _dbContext.SaveChangesAsync(); - var otherPairData = _dbContext.ClientPairs.Include(u => u.User).Where(u => u.OtherUser.UID == userid); + var otherPairData = await _dbContext.ClientPairs.Include(u => u.User) + .Where(u => u.OtherUser.UID == userid).ToListAsync(); foreach (var pair in otherPairData) { await Clients.User(pair.User.UID) @@ -38,6 +43,10 @@ namespace MareSynchronosServer.Hubs }, userEntry.CharacterIdentification); } + MareMetrics.Pairs.Dec(otherPairData.Count()); + MareMetrics.PairsPaused.Dec(otherPairData.Count(c => c.IsPaused)); + MareMetrics.UsersRegistered.Dec(); + _dbContext.RemoveRange(otherPairData); _dbContext.Remove(userEntry); await _dbContext.SaveChangesAsync(); @@ -94,8 +103,6 @@ namespace MareSynchronosServer.Hubs }).ToList(); } - - [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AuthScheme)] [HubMethodName(Api.InvokeUserPushCharacterDataToVisibleClients)] public async Task PushCharacterDataToVisibleClients(CharacterCacheDto characterCache, List visibleCharacterIds) @@ -119,6 +126,9 @@ namespace MareSynchronosServer.Hubs await Clients.User(pairedUser.UID).SendAsync(Api.OnUserReceiveCharacterData, characterCache, user.CharacterIdentification); } + + MareMetrics.UserPushData.Inc(); + MareMetrics.UserPushDataTo.Inc(visibleCharacterIds.Count); } [HubMethodName(Api.InvokeUserRegister)] @@ -155,6 +165,8 @@ namespace MareSynchronosServer.Hubs _logger.LogInformation("User registered: " + user.UID); + MareMetrics.UsersRegistered.Inc(); + await _dbContext.SaveChangesAsync(); return computedHash; } @@ -212,6 +224,8 @@ namespace MareSynchronosServer.Hubs .SendAsync(Api.OnUserAddOnlinePairedPlayer, user.CharacterIdentification); } } + + MareMetrics.Pairs.Inc(); } [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AuthScheme)] @@ -250,6 +264,15 @@ namespace MareSynchronosServer.Hubs IsSynced = true }, user.CharacterIdentification); } + + if (isPaused) + { + MareMetrics.PairsPaused.Inc(); + } + else + { + MareMetrics.PairsPaused.Dec(); + } } [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AuthScheme)] @@ -291,6 +314,8 @@ namespace MareSynchronosServer.Hubs }, sender.CharacterIdentification); } } + + MareMetrics.Pairs.Dec(); } private ClientPair OppositeEntry(string otherUID) => diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs index 7869787..a29d729 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs @@ -30,6 +30,8 @@ namespace MareSynchronosServer.Hubs [HubMethodName(Api.InvokeHeartbeat)] public async Task Heartbeat(string? characterIdentification) { + MareMetrics.InitializedConnections.Inc(); + var userId = Context.User!.Claims.SingleOrDefault(c => c.Type == ClaimTypes.NameIdentifier)?.Value; _logger.LogInformation("Connection from " + userId + ", CI: " + characterIdentification); @@ -40,6 +42,7 @@ namespace MareSynchronosServer.Hubs if (userId != null && !isBanned && !string.IsNullOrEmpty(characterIdentification)) { + MareMetrics.AuthorizedConnections.Inc(); _logger.LogInformation("Connection from " + userId); var user = (await _dbContext.Users.SingleAsync(u => u.UID == userId)); user.CharacterIdentification = characterIdentification; @@ -52,6 +55,10 @@ namespace MareSynchronosServer.Hubs IsAdmin = user.IsAdmin }; } + else + { + MareMetrics.UnauthorizedConnections.Inc(); + } return new ConnectionDto() { @@ -70,6 +77,7 @@ namespace MareSynchronosServer.Hubs var user = await _dbContext.Users.AsNoTracking().SingleOrDefaultAsync(u => u.UID == AuthenticatedUserId); if (user != null && !string.IsNullOrEmpty(user.CharacterIdentification)) { + MareMetrics.AuthorizedConnections.Dec(); _logger.LogInformation("Disconnect from " + AuthenticatedUserId); var otherUsers = await _dbContext.ClientPairs.AsNoTracking() @@ -91,6 +99,10 @@ namespace MareSynchronosServer.Hubs await Clients.All.SendAsync("UsersOnline", await _dbContext.Users.CountAsync(u => !string.IsNullOrEmpty(u.CharacterIdentification))); } + else + { + MareMetrics.UnauthorizedConnections.Dec(); + } await base.OnDisconnectedAsync(exception); } diff --git a/MareSynchronosServer/MareSynchronosServer/MareMetrics.cs b/MareSynchronosServer/MareSynchronosServer/MareMetrics.cs new file mode 100644 index 0000000..f61ad71 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosServer/MareMetrics.cs @@ -0,0 +1,50 @@ +using System.IO; +using System.Linq; +using MareSynchronosServer.Data; +using Microsoft.Extensions.Configuration; +using Prometheus; + +namespace MareSynchronosServer +{ + public class MareMetrics + { + public static readonly Counter InitializedConnections = + Metrics.CreateCounter("mare_initialized_connections", "Initialized Connections"); + public static readonly Gauge UnauthorizedConnections = + Metrics.CreateGauge("mare_unauthorized_connections", "Unauthorized Connections"); + public static readonly Gauge AuthorizedConnections = + Metrics.CreateGauge("mare_authorized_connections", "Authorized Connections"); + + public static readonly Gauge UsersRegistered = Metrics.CreateGauge("mare_users_registered", "Total Registrations"); + + public static readonly Gauge Pairs = Metrics.CreateGauge("mare_pairs", "Total Pairs"); + public static readonly Gauge PairsPaused = Metrics.CreateGauge("mare_pairs_paused", "Total Paused Pairs"); + + public static readonly Gauge FilesTotal = Metrics.CreateGauge("mare_files", "Total uploaded files"); + public static readonly Gauge FilesTotalSize = + Metrics.CreateGauge("mare_files_size", "Total uploaded files (bytes)"); + + public static readonly Counter UserPushData = Metrics.CreateCounter("mare_user_push", "Users pushing data"); + public static readonly Counter UserPushDataTo = + Metrics.CreateCounter("mare_user_push_to", "Users Receiving Data"); + + public static readonly Counter UserDownloadedFiles = Metrics.CreateCounter("mare_user_downloaded_files", "Total Downloaded Files by Users"); + public static readonly Counter UserDownloadedFilesSize = Metrics.CreateCounter("mare_user_downloaded_files_size", "Total Downloaded Files Size by Users"); + + public static readonly Gauge + CPUUsage = Metrics.CreateGauge("mare_cpu_usage", "Total calculated CPU usage in %"); + public static readonly Gauge RAMUsage = + Metrics.CreateGauge("mare_ram_usage", "Total calculated RAM usage in bytes for Mare + MSSQL"); + public static readonly Gauge NetworkOut = Metrics.CreateGauge("mare_network_out", "Network out in byte/s"); + public static readonly Gauge NetworkIn = Metrics.CreateGauge("mare_network_in", "Network in in byte/s"); + + public static void InitializeMetrics(MareDbContext dbContext, IConfiguration configuration) + { + UsersRegistered.IncTo(dbContext.Users.Count()); + Pairs.IncTo(dbContext.ClientPairs.Count()); + PairsPaused.IncTo(dbContext.ClientPairs.Count(p => p.IsPaused)); + FilesTotal.IncTo(dbContext.Files.Count()); + FilesTotalSize.IncTo(Directory.EnumerateFiles(configuration["CacheDirectory"]).Sum(f => new FileInfo(f).Length)); + } + } +} diff --git a/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj b/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj index d24054e..aec5f4e 100644 --- a/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj +++ b/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj @@ -1,4 +1,4 @@ - + net6.0 @@ -20,6 +20,8 @@ + + diff --git a/MareSynchronosServer/MareSynchronosServer/Program.cs b/MareSynchronosServer/MareSynchronosServer/Program.cs index 4eabcee..fbbbfd2 100644 --- a/MareSynchronosServer/MareSynchronosServer/Program.cs +++ b/MareSynchronosServer/MareSynchronosServer/Program.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Hosting; using System.Linq; using MareSynchronosServer.Data; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -13,7 +14,8 @@ namespace MareSynchronosServer { public static void Main(string[] args) { - var host = CreateHostBuilder(args).Build(); + var hostBuilder = CreateHostBuilder(args); + var host = hostBuilder.Build(); using (var scope = host.Services.CreateScope()) { @@ -31,6 +33,8 @@ namespace MareSynchronosServer var looseFiles = context.Files.Where(f => f.Uploaded == false); context.RemoveRange(looseFiles); context.SaveChanges(); + + MareMetrics.InitializeMetrics(context, services.GetRequiredService()); } host.Run(); diff --git a/MareSynchronosServer/MareSynchronosServer/Startup.cs b/MareSynchronosServer/MareSynchronosServer/Startup.cs index 5d1b949..f4e42bd 100644 --- a/MareSynchronosServer/MareSynchronosServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosServer/Startup.cs @@ -12,6 +12,7 @@ using MareSynchronosServer.Hubs; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Http.Connections; using Microsoft.AspNetCore.SignalR; +using Prometheus; using WebSocketOptions = Microsoft.AspNetCore.Builder.WebSocketOptions; namespace MareSynchronosServer @@ -40,6 +41,7 @@ namespace MareSynchronosServer services.AddSingleton(); services.AddSingleton(); + services.AddScoped(_ => Configuration); services.AddDbContextPool(options => { @@ -62,6 +64,7 @@ namespace MareSynchronosServer // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { + if (env.IsDevelopment()) { app.UseDeveloperExceptionPage(); @@ -83,11 +86,13 @@ namespace MareSynchronosServer KeepAliveInterval = TimeSpan.FromSeconds(10), }; + app.UseHttpMetrics(); app.UseWebSockets(webSocketOptions); app.UseAuthentication(); app.UseAuthorization(); + app.UseEndpoints(endpoints => { endpoints.MapHub(Api.Path, options => @@ -96,6 +101,8 @@ namespace MareSynchronosServer options.TransportMaxBufferSize = 5242880; options.Transports = HttpTransportType.WebSockets; }); + + endpoints.MapMetrics(); }); } } diff --git a/MareSynchronosServer/MareSynchronosServer/SystemInfoService.cs b/MareSynchronosServer/MareSynchronosServer/SystemInfoService.cs index e45b2f5..4cba87e 100644 --- a/MareSynchronosServer/MareSynchronosServer/SystemInfoService.cs +++ b/MareSynchronosServer/MareSynchronosServer/SystemInfoService.cs @@ -86,44 +86,30 @@ public class SystemInfoService : IHostedService, IDisposable var bytesSent = endNetworkOut - networkOut; var bytesReceived = endNetworkIn - networkIn; - using var scope = _services.CreateScope(); - var dbContext = scope.ServiceProvider.GetService()!; - int uploadedFiles = 0; - var loggedInUsers = dbContext.Users.Count(u => !string.IsNullOrEmpty(u.CharacterIdentification)); - var localCacheSize = Directory.EnumerateFiles(_configuration["CacheDirectory"]) - .ToList().Sum(f => - { - uploadedFiles++; - try - { - return new FileInfo(f).Length; - } - catch - { - return 0; - } - }); + var usedRAM = Process.GetCurrentProcess().WorkingSet64 + Process.GetProcessesByName("sqlservr").FirstOrDefault()?.WorkingSet64 ?? 0; + var cpuUsage = cpuUsageTotal * 100; var totalNetworkOut = bytesSent / totalSPassed; var totalNetworkIn = bytesReceived / totalSPassed; - var cpuUsage = cpuUsageTotal * 100; - var usedRAM = Process.GetCurrentProcess().WorkingSet64 + Process.GetProcessesByName("sqlservr").FirstOrDefault()?.WorkingSet64 ?? 0; + + MareMetrics.NetworkIn.Set(totalNetworkIn); + MareMetrics.NetworkOut.Set(totalNetworkOut); + MareMetrics.CPUUsage.Set(cpuUsage); + MareMetrics.RAMUsage.Set(usedRAM); SystemInfoDto = new SystemInfoDto() { - CacheUsage = localCacheSize, + CacheUsage = 0, CpuUsage = cpuUsage, - RAMUsage = usedRAM, + RAMUsage = 0, NetworkIn = totalNetworkIn, NetworkOut = totalNetworkOut, - OnlineUsers = loggedInUsers, - UploadedFiles = uploadedFiles + OnlineUsers = (int)MareMetrics.AuthorizedConnections.Value, + UploadedFiles = 0 }; _hubContext.Clients.All.SendAsync(Api.OnUpdateSystemInfo, SystemInfoDto); - - _logger.LogInformation($"CPU:{cpuUsage:0.00}%, RAM Used:{(double)usedRAM / 1024 / 1024 / 1024:0.00}GB, Cache:{(double)localCacheSize / 1024 / 1024 / 1024:0.00}GB, Users:{loggedInUsers}, NetworkIn:{totalNetworkIn / 1024 / 1024:0.00}MB/s, NetworkOut:{totalNetworkOut / 1024 / 1024:0.00}MB/s"); } public Task StopAsync(CancellationToken cancellationToken)