idk, it's just too many changes by now

MareSynchronosServer/MareSynchronos.API/CharacterCacheDto.cs

@@ -11,6 +11,7 @@ namespace MareSynchronos.API
     {
         public List<FileReplacementDto> FileReplacements { get; set; } = new();
         public string GlamourerData { get; set; }
+        public string ManipulationData { get; set; }
         public string Hash { get; set; }
         public int JobId { get; set; }
     }
@@ -19,6 +20,5 @@ namespace MareSynchronos.API
     {
         public string[] GamePaths { get; set; } = Array.Empty<string>();
         public string Hash { get; set; }
-        public string ImcData { get; set; }
     }
 }

MareSynchronosServer/MareSynchronos.API/ClientPairDto.cs

@@ -6,5 +6,6 @@
         public bool IsPaused { get; set; }
         public bool IsSynced { get; set; }
         public bool IsPausedFromOthers { get; set; }
+        public bool IsRemoved { get; set; }
     }
 }

MareSynchronosServer/MareSynchronosServer/Data/MareDbContext.cs

@@ -15,7 +15,6 @@ namespace MareSynchronosServer.Data
         public DbSet<User> Users { get; set; }
         public DbSet<FileCache> Files { get; set; }
         public DbSet<ClientPair> ClientPairs { get; set; }
-        public DbSet<Visibility> Visibilities { get; set; }
         public DbSet<CharacterData> CharacterData { get; set; }
 
         protected override void OnModelCreating(ModelBuilder modelBuilder)
@@ -23,13 +22,11 @@ namespace MareSynchronosServer.Data
             modelBuilder.Entity<User>().ToTable("Users");
             modelBuilder.Entity<FileCache>().ToTable("FileCaches");
             modelBuilder.Entity<ClientPair>().ToTable("ClientPairs");
-            modelBuilder.Entity<Visibility>().ToTable("Visibility")
-                .HasKey(k => new { k.CID, k.OtherCID });
             modelBuilder.Entity<CharacterData>()
-                .Property(b => b.EquipmentData)
+                .Property(b => b.CharacterCache)
                 .HasConversion(
                     v => JsonConvert.SerializeObject(v),
-                    v => JsonConvert.DeserializeObject<List<FileReplacementDto>>(v));
+                    v => JsonConvert.DeserializeObject<CharacterCacheDto>(v));
             modelBuilder.Entity<CharacterData>()
                 .ToTable("CharacterData")
                 .HasKey(k => new { k.UserId, k.JobId });

MareSynchronosServer/MareSynchronosServer/FileCleanupService.cs

@@ -0,0 +1,91 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using MareSynchronosServer.Data;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace MareSynchronosServer
+{
+    public class FileCleanupService : IHostedService, IDisposable
+    {
+        private readonly ILogger<FileCleanupService> _logger;
+        private readonly IServiceProvider _services;
+        private readonly IConfiguration _configuration;
+        private Timer _timer;
+
+        public FileCleanupService(ILogger<FileCleanupService> logger, IServiceProvider services, IConfiguration configuration)
+        {
+            _logger = logger;
+            _services = services;
+            _configuration = configuration;
+        }
+
+        public Task StartAsync(CancellationToken cancellationToken)
+        {
+            _logger.LogInformation("File Cleanup Service started");
+
+            _timer = new Timer(CleanUpFiles, null, TimeSpan.Zero, TimeSpan.FromMinutes(10));
+
+            return Task.CompletedTask;
+        }
+
+        private void CleanUpFiles(object state)
+        {
+            if (!int.TryParse(_configuration["UnusedFileRetentionPeriodInDays"], out var filesOlderThanDays))
+            {
+                filesOlderThanDays = 7;
+            }
+
+            _logger.LogInformation($"Cleaning up files older than {filesOlderThanDays} days");
+
+            using var scope = _services.CreateScope();
+            var dbContext = scope.ServiceProvider.GetService<MareDbContext>();
+
+            var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(filesOlderThanDays));
+            var filesToDelete =
+                dbContext.Files.Where(f => f.LastAccessTime < prevTime);
+            dbContext.RemoveRange(filesToDelete);
+            foreach (var file in filesToDelete)
+            {
+                var fileName = Path.Combine(_configuration["CacheDirectory"], file.Hash);
+                if (File.Exists(fileName))
+                {
+                    _logger.LogInformation("Deleting: " + fileName);
+                    File.Delete(fileName);
+                }
+            }
+
+            dbContext.SaveChanges();
+
+            var allFiles = dbContext.Files;
+            foreach (var file in allFiles)
+            {
+                var fileName = Path.Combine(_configuration["CacheDirectory"], file.Hash);
+                if (!File.Exists(fileName))
+                {
+                    _logger.LogInformation("File does not exist anymore: " + fileName);
+                    dbContext.Files.Remove(file);
+                }
+            }
+
+            dbContext.SaveChanges();
+        }
+
+        public Task StopAsync(CancellationToken cancellationToken)
+        {
+            _timer?.Change(Timeout.Infinite, 0);
+
+            return Task.CompletedTask;
+        }
+
+        public void Dispose()
+        {
+            _timer?.Dispose();
+        }
+    }
+}

MareSynchronosServer/MareSynchronosServer/Hubs/BaseHub.cs

@@ -3,15 +3,18 @@ using System.Security.Claims;
 using System.Security.Cryptography;
 using MareSynchronosServer.Data;
 using Microsoft.AspNetCore.SignalR;
+using Microsoft.Extensions.Logging;
 
 namespace MareSynchronosServer.Hubs
 {
-    public abstract class BaseHub : Hub
+    public abstract class BaseHub<T> : Hub
     {
+        protected readonly ILogger<T> Logger;
         protected MareDbContext DbContext { get; init; }
 
-        protected BaseHub(MareDbContext context)
+        protected BaseHub(MareDbContext context, ILogger<T> logger)
         {
+            Logger = logger;
             DbContext = context;
         }
 

MareSynchronosServer/MareSynchronosServer/Hubs/ConnectionHub.cs

@@ -3,18 +3,23 @@ using System.Linq;
 using System.Security.Claims;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.SignalR;
+using Microsoft.Extensions.Logging;
 
 namespace MareSynchronosServer.Hubs
 {
-    public class Connection : Hub
+    public class ConnectionHub : Hub
     {
+        private readonly ILogger<ConnectionHub> _logger;
+
+        public ConnectionHub(ILogger<ConnectionHub> logger)
+        {
+            _logger = logger;
+        }
+
         public string Heartbeat()
         {
             var userId = Context.User!.Claims.SingleOrDefault(c => c.Type == ClaimTypes.NameIdentifier)?.Value;
-            if (userId != null)
-            {
-                var user = Clients.User(userId);
-            }
+            _logger.LogInformation("Heartbeat from " + (userId ?? "Unknown user"));
             return userId ?? string.Empty;
         }
     }

MareSynchronosServer/MareSynchronosServer/Hubs/FilesHub.cs

@@ -5,31 +5,36 @@ using System.Diagnostics;
 using System.Globalization;
 using System.IO;
 using System.Linq;
-using System.Security.Claims;
+using System.Runtime.CompilerServices;
 using System.Security.Cryptography;
 using System.Threading;
 using System.Threading.Channels;
 using System.Threading.Tasks;
-using MareSynchronos.API;
 using MareSynchronosServer.Authentication;
 using MareSynchronosServer.Data;
 using MareSynchronosServer.Models;
 using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.SignalR;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
 
 namespace MareSynchronosServer.Hubs
 {
-    public class Files : BaseHub
+    public class FilesHub : BaseHub<FilesHub>
     {
-        private static readonly ConcurrentDictionary<string, CancellationTokenSource> UserUploads = new();
-        public Files(MareDbContext dbContext) : base(dbContext)
+        private readonly IConfiguration _configuration;
+
+        public FilesHub(ILogger<FilesHub> logger, MareDbContext context, IConfiguration configuration) : base(context, logger)
         {
+            _configuration = configuration;
         }
 
+        private string BasePath => _configuration["CacheDirectory"];
+        
         [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AUTH_SCHEME)]
         public async Task AbortUpload()
         {
+            Logger.LogInformation("User " + AuthenticatedUserId + " aborted upload");
             var userId = AuthenticatedUserId;
             var notUploadedFiles = DbContext.Files.Where(f => !f.Uploaded && f.Uploader.UID == userId).ToList();
             DbContext.RemoveRange(notUploadedFiles);
@@ -37,13 +42,14 @@ namespace MareSynchronosServer.Hubs
         }
 
         [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AUTH_SCHEME)]
-        public async Task<List<string>> SendFiles(List<FileReplacementDto> fileList)
+        public async Task<List<string>> SendFiles(List<string> fileListHashes)
         {
-            var fileListHashes = fileList.Select(x => x.Hash).ToList();
+            Logger.LogInformation("User " + AuthenticatedUserId + " sending files");
             List<string> filesToUpload = new List<string>();
             var existingFiles = DbContext.Files.Where(f => fileListHashes.Contains(f.Hash)).ToList();
             foreach (var file in fileListHashes.Where(f => existingFiles.All(e => e.Hash != f)))
             {
+                Debug.WriteLine("File: " + file);
                 var userId = AuthenticatedUserId;
                 await DbContext.Files.AddAsync(new FileCache()
                 {
@@ -67,23 +73,24 @@ namespace MareSynchronosServer.Hubs
         }
 
         [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AUTH_SCHEME)]
-        public async Task UploadFile(string hash, ChannelReader<byte[]> stream)
+        public async Task UploadFileStreamAsync(string hash, IAsyncEnumerable<byte[]> fileContent)
         {
-            var relatedFile = DbContext.Files.SingleOrDefault(f => f.Hash == hash);
+            Logger.LogInformation("User " + AuthenticatedUserId + " uploading file: " + hash);
+
+            var relatedFile = DbContext.Files.SingleOrDefault(f => f.Hash == hash && f.Uploader.UID == AuthenticatedUserId && f.Uploaded == false);
             if (relatedFile == null) return;
-            List<byte> uploadedFile = new();
-            while (await stream.WaitToReadAsync())
+            var uploadedFile = new List<byte>();
+            await foreach (var chunk in fileContent)
             {
-                while (stream.TryRead(out var byteChunk))
-                {
-                    uploadedFile.AddRange(byteChunk);
+                uploadedFile.AddRange(chunk);
             }
-            }
-            Debug.WriteLine(DateTime.Now.ToString(CultureInfo.InvariantCulture) + ": File size of " + hash + ":" + uploadedFile.Count);
+
+            Logger.LogInformation("User " + AuthenticatedUserId + " upload finished: " + hash + ", size: " + uploadedFile.Count);
+
             try
             {
                 var decodedFile = LZ4.LZ4Codec.Unwrap(uploadedFile.ToArray());
-                using var sha1 = new SHA1CryptoServiceProvider();
+                using var sha1 = SHA1.Create();
                 var computedHash = await sha1.ComputeHashAsync(new MemoryStream(decodedFile));
                 var computedHashString = BitConverter.ToString(computedHash).Replace("-", "");
                 if (hash != computedHashString)
@@ -93,7 +100,7 @@ namespace MareSynchronosServer.Hubs
                     return;
                 }
 
-                await File.WriteAllBytesAsync(@"G:\ServerTest\" + hash, uploadedFile.ToArray());
+                await File.WriteAllBytesAsync(Path.Combine(BasePath, hash), uploadedFile.ToArray());
                 relatedFile = DbContext.Files.Single(f => f.Hash == hash);
                 relatedFile.Uploaded = true;
                 relatedFile.LastAccessTime = DateTime.Now;
@@ -111,34 +118,56 @@ namespace MareSynchronosServer.Hubs
         {
             var file = await DbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
             if (file == null) return -1;
-            return new FileInfo(@"G:\ServerTest\" + hash).Length;
+            var fileInfo = new FileInfo(Path.Combine(BasePath, hash));
+            if (fileInfo.Exists) return fileInfo.Length;
+            DbContext.Files.Remove(file);
+            await DbContext.SaveChangesAsync();
+            return -1;
+
         }
 
         [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AUTH_SCHEME)]
-        public async Task<ChannelReader<byte[]>> DownloadFile(string hash)
+        public async IAsyncEnumerable<byte[]> DownloadFileAsync(string hash, [EnumeratorCancellation] CancellationToken ct)
         {
+            Logger.LogInformation("User " + AuthenticatedUserId + " downloading file: " + hash);
+
             var file = DbContext.Files.SingleOrDefault(f => f.Hash == hash);
-            if (file == null) return null;
-            var compressedFile = await File.ReadAllBytesAsync(@"G:\ServerTest\" + hash);
+            if (file == null) yield break;
+            file.LastAccessTime = DateTime.Now;
+            DbContext.Update(file);
+            await DbContext.SaveChangesAsync(ct);
             var chunkSize = 1024 * 512; // 512kb
-            var chunks = (int)Math.Ceiling(compressedFile.Length / (double)chunkSize);
-            var channel = Channel.CreateBounded<byte[]>(chunkSize);
-            _ = Task.Run(() =>
+            int readByteCount;
+            var buffer = new byte[chunkSize];
+
+            await using var fs = File.Open(Path.Combine(BasePath, hash), FileMode.Open, FileAccess.Read);
+            while ((readByteCount = await fs.ReadAsync(buffer, 0, chunkSize, ct)) > 0)
             {
-                for (var i = 0; i < chunks; i++)
-                {
-                    channel.Writer.TryWrite(compressedFile.Skip(i * chunkSize).Take(chunkSize).ToArray());
+                await Task.Delay(10, ct);
+                yield return readByteCount == chunkSize ? buffer.ToArray() : buffer.Take(readByteCount).ToArray();
             }
 
-                channel.Writer.Complete();
-            });
+            Logger.LogInformation("User " + AuthenticatedUserId + " finished downloading file: " + hash);
+        }
         
-            return channel.Reader;
+        [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AUTH_SCHEME)]
+        public async Task DeleteAllFiles()
+        {
+            Logger.LogInformation("User " + AuthenticatedUserId + " deleted all their files");
+
+            DbContext.CharacterData.RemoveRange(DbContext.CharacterData.Where(c => c.UserId == AuthenticatedUserId));
+            await DbContext.SaveChangesAsync();
+            var ownFiles = await DbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == AuthenticatedUserId).ToListAsync();
+            foreach (var file in ownFiles)
+            {
+                File.Delete(Path.Combine(BasePath,  file.Hash));
+            }
+            DbContext.Files.RemoveRange(ownFiles);
+            await DbContext.SaveChangesAsync();
         }
 
         public override Task OnDisconnectedAsync(Exception exception)
         {
-            Debug.WriteLine("Detected disconnect from " + AuthenticatedUserId);
             var userId = AuthenticatedUserId;
             var notUploadedFiles = DbContext.Files.Where(f => !f.Uploaded && f.Uploader.UID == userId).ToList();
             DbContext.RemoveRange(notUploadedFiles);

MareSynchronosServer/MareSynchronosServer/Hubs/UserHub.cs

@@ -12,15 +12,21 @@ using MareSynchronosServer.Models;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.SignalR;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
 
 namespace MareSynchronosServer.Hubs
 {
-    public class User : BaseHub
+    public class UserHub : BaseHub<UserHub>
     {
-        public User(MareDbContext dbContext) : base(dbContext)
+        public UserHub(ILogger<UserHub> logger, MareDbContext dbContext) : base(dbContext, logger)
         {
         }
 
+        public async Task<int> GetOnlineUsers()
+        {
+            return await DbContext.Users.CountAsync(u => !string.IsNullOrEmpty(u.CharacterIdentification));
+        }
+
         public async Task<string> Register()
         {
             using var sha256 = SHA256.Create();
@@ -41,6 +47,8 @@ namespace MareSynchronosServer.Hubs
             }
             DbContext.Users.Add(user);
 
+            Logger.LogInformation("User registered: " + user.UID);
+
             await DbContext.SaveChangesAsync();
             return computedHash;
         }
@@ -54,27 +62,6 @@ namespace MareSynchronosServer.Hubs
             return AuthenticatedUserId;
         }
 
-        [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AUTH_SCHEME)]
-        public async Task SendVisibilityList(List<string> currentVisibilityList)
-        {
-            Stopwatch st = Stopwatch.StartNew();
-            var cid = DbContext.Users.Single(u => u.UID == AuthenticatedUserId).CharacterIdentification;
-            var visibilities = DbContext.Visibilities.Where(v => v.CID == cid).ToList();
-            foreach (var visibility in currentVisibilityList.Where(visibility => visibilities.All(v => v.OtherCID != visibility)))
-            {
-                await DbContext.Visibilities.AddAsync(new Visibility { CID = cid, OtherCID = visibility });
-            }
-
-            foreach (var visibility in visibilities.Where(v => currentVisibilityList.Contains(v.OtherCID)))
-            {
-                DbContext.Visibilities.Remove(visibility);
-            }
-
-            await DbContext.SaveChangesAsync();
-            st.Stop();
-            Debug.WriteLine("Visibility update took " + st.Elapsed);
-        }
-
         [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AUTH_SCHEME)]
         public async Task GetCharacterData(Dictionary<string, int> visibleCharacterWithJobs)
         {
@@ -96,13 +83,7 @@ namespace MareSynchronosServer.Hubs
                     DbContext.CharacterData.SingleOrDefaultAsync(c => c.UserId == pair.User.UID && c.JobId == dictEntry);
                 if (cachedChar != null)
                 {
-                    await Clients.User(uid).SendAsync("ReceiveCharacterData", new CharacterCacheDto()
-                    {
-                        FileReplacements = cachedChar.EquipmentData,
-                        Hash = cachedChar.Hash,
-                        JobId = cachedChar.JobId,
-                        GlamourerData = cachedChar.GlamourerData
-                    },
+                    await Clients.User(uid).SendAsync("ReceiveCharacterData", cachedChar.CharacterCache,
                         pair.User.CharacterIdentification);
                 }
             }
@@ -111,6 +92,8 @@ namespace MareSynchronosServer.Hubs
         [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AUTH_SCHEME)]
         public async Task PushCharacterData(CharacterCacheDto characterCache, List<string> visibleCharacterIds)
         {
+            Logger.LogInformation("User " + AuthenticatedUserId + " pushing character data");
+
             var uid = AuthenticatedUserId;
             var entriesHavingThisUser = DbContext.ClientPairs
                 .Include(w => w.User)
@@ -120,30 +103,27 @@ namespace MareSynchronosServer.Hubs
             var existingCharacterData =
                 await DbContext.CharacterData.SingleOrDefaultAsync(s =>
                     s.UserId == uid && s.JobId == characterCache.JobId);
+
             if (existingCharacterData != null && existingCharacterData.Hash != characterCache.Hash)
             {
-                existingCharacterData.EquipmentData =
-                    characterCache.FileReplacements;
+                existingCharacterData.CharacterCache = characterCache;
                 existingCharacterData.Hash = characterCache.Hash;
-                existingCharacterData.GlamourerData = characterCache.GlamourerData;
                 DbContext.CharacterData.Update(existingCharacterData);
+                await DbContext.SaveChangesAsync();
             }
             else if (existingCharacterData == null)
             {
                 CharacterData data = new CharacterData
                 {
                     UserId = AuthenticatedUserId,
-                    EquipmentData = characterCache.FileReplacements,
+                    CharacterCache = characterCache,
                     Hash = characterCache.Hash,
-                    GlamourerData = characterCache.GlamourerData,
                     JobId = characterCache.JobId
                 };
                 await DbContext.CharacterData.AddAsync(data);
                 await DbContext.SaveChangesAsync();
             }
 
-            if ((existingCharacterData != null && existingCharacterData.Hash != characterCache.Hash) || existingCharacterData == null)
-            {
             foreach (var pair in entriesHavingThisUser)
             {
                 var ownEntry = DbContext.ClientPairs.SingleOrDefault(w =>
@@ -153,33 +133,42 @@ namespace MareSynchronosServer.Hubs
                     pair.OtherUser.CharacterIdentification);
             }
         }
-        }
 
         [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AUTH_SCHEME)]
         public async Task<List<string>> SendCharacterNameHash(string characterNameHash)
         {
+            Logger.LogInformation("User " + AuthenticatedUserId + " sent character hash");
+
             var ownUser = DbContext.Users.Single(u => u.UID == AuthenticatedUserId);
             ownUser.CharacterIdentification = characterNameHash;
             await DbContext.SaveChangesAsync();
             var otherUsers = await DbContext.ClientPairs
                 .Include(u => u.User)
                 .Include(u => u.OtherUser)
-                .Where(w => w.User == ownUser)
+                .Where(w => w.User == ownUser && !w.IsPaused)
                 .Where(w => !string.IsNullOrEmpty(w.OtherUser.CharacterIdentification))
                 .Select(e => e.OtherUser).ToListAsync();
             var otherEntries = await DbContext.ClientPairs.Include(u => u.User)
-                .Where(u => otherUsers.Any(e => e == u.User) && u.OtherUser == ownUser).ToListAsync();
+                .Where(u => otherUsers.Any(e => e == u.User) && u.OtherUser == ownUser && !u.IsPaused).ToListAsync();
 
             await Clients.Users(otherEntries.Select(e => e.User.UID)).SendAsync("AddOnlinePairedPlayer", characterNameHash);
+            await Clients.All.SendAsync("UsersOnline",
+                await DbContext.Users.CountAsync(u => !string.IsNullOrEmpty(u.CharacterIdentification)));
             return otherEntries.Select(e => e.User.CharacterIdentification).ToList();
         }
 
         [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AUTH_SCHEME)]
         public async Task SendPairedClientAddition(string uid)
         {
+            if (uid == AuthenticatedUserId) return;
+
+            Logger.LogInformation("User " + AuthenticatedUserId + " added " + uid + " to whitelist");
             var user = await DbContext.Users.SingleAsync(u => u.UID == AuthenticatedUserId);
             var otherUser = await DbContext.Users.SingleOrDefaultAsync(u => u.UID == uid);
-            if (otherUser == null) return;
+            var existingEntry =
+                await DbContext.ClientPairs.SingleOrDefaultAsync(p =>
+                    p.User.UID == AuthenticatedUserId && p.OtherUser.UID == uid);
+            if (otherUser == null || existingEntry != null) return;
             ClientPair wl = new ClientPair()
             {
                 IsPaused = false,
@@ -194,12 +183,12 @@ namespace MareSynchronosServer.Hubs
                 {
                     OtherUID = otherUser.UID,
                     IsPaused = false,
-                    IsPausedFromOthers = otherEntry?.IsPaused ?? false,
+                    IsPausedFromOthers = false,
                     IsSynced = otherEntry != null
-                }, otherUser.CharacterIdentification);
+                }, string.Empty);
             if (otherEntry != null)
             {
-                if (string.IsNullOrEmpty(otherUser.CharacterIdentification))
+                if (!string.IsNullOrEmpty(otherUser.CharacterIdentification))
                 {
                     await Clients.User(user.UID)
                         .SendAsync("AddOnlinePairedPlayer", otherUser.CharacterIdentification);
@@ -221,11 +210,15 @@ namespace MareSynchronosServer.Hubs
         [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AUTH_SCHEME)]
         public async Task SendPairedClientRemoval(string uid)
         {
+            if (uid == AuthenticatedUserId) return;
+
+            Logger.LogInformation("User " + AuthenticatedUserId + " removed " + uid + " from whitelist");
             var user = await DbContext.Users.SingleAsync(u => u.UID == AuthenticatedUserId);
             var otherUser = await DbContext.Users.SingleOrDefaultAsync(u => u.UID == uid);
             if (otherUser == null) return;
             ClientPair wl =
                 await DbContext.ClientPairs.SingleOrDefaultAsync(w => w.User == user && w.OtherUser == otherUser);
+            if (wl == null) return;
             DbContext.ClientPairs.Remove(wl);
             await DbContext.SaveChangesAsync();
             var otherEntry = OppositeEntry(uid);
@@ -233,13 +226,11 @@ namespace MareSynchronosServer.Hubs
                 .SendAsync("UpdateClientPairs", new ClientPairDto()
                 {
                     OtherUID = otherUser.UID,
-                    IsPaused = false,
-                    IsPausedFromOthers = otherEntry?.IsPaused ?? false,
-                    IsSynced = otherEntry != null
+                    IsRemoved = true
                 }, otherUser.CharacterIdentification);
             if (otherEntry != null)
             {
-                if (string.IsNullOrEmpty(otherUser.CharacterIdentification))
+                if (!string.IsNullOrEmpty(otherUser.CharacterIdentification))
                 {
                     await Clients.User(user.UID)
                         .SendAsync("RemoveOnlinePairedPlayer", otherUser.CharacterIdentification);
@@ -259,6 +250,8 @@ namespace MareSynchronosServer.Hubs
         [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AUTH_SCHEME)]
         public async Task SendPairedClientPauseChange(string uid, bool isPaused)
         {
+            if (uid == AuthenticatedUserId) return;
+            Logger.LogInformation("User " + AuthenticatedUserId + " changed pause status with " + uid + " to " + isPaused);
             var user = DbContext.Users.Single(u => u.UID == AuthenticatedUserId);
             var otherUser = await DbContext.Users.SingleOrDefaultAsync(u => u.UID == uid);
             if (otherUser == null) return;
@@ -312,31 +305,62 @@ namespace MareSynchronosServer.Hubs
                 }).ToList();
         }
 
+        [Authorize(AuthenticationSchemes = SecretKeyAuthenticationHandler.AUTH_SCHEME)]
+        public async Task DeleteAccount()
+        {
+            Logger.LogInformation("User " + AuthenticatedUserId + " deleted their account");
 
-        public override Task OnDisconnectedAsync(Exception exception)
+            string userid = AuthenticatedUserId;
+            var userEntry = await DbContext.Users.SingleOrDefaultAsync(u => u.UID == userid);
+            var charData = DbContext.CharacterData.Where(u => u.UserId == userid);
+            DbContext.RemoveRange(charData);
+            await DbContext.SaveChangesAsync();
+            var ownPairData = DbContext.ClientPairs.Where(u => u.User.UID == userid);
+            DbContext.RemoveRange(ownPairData);
+            await DbContext.SaveChangesAsync();
+            var otherPairData = DbContext.ClientPairs.Include(u => u.User).Where(u => u.OtherUser.UID == userid);
+            foreach (var pair in otherPairData)
+            {
+                await Clients.User(pair.User.UID)
+                    .SendAsync("UpdateClientPairs", new ClientPairDto()
+                    {
+                        OtherUID = userid,
+                        IsRemoved = true
+                    }, userEntry.CharacterIdentification);
+            }
+
+            DbContext.RemoveRange(otherPairData);
+            DbContext.Remove(userEntry);
+            await DbContext.SaveChangesAsync();
+
+        }
+
+        public override async Task OnDisconnectedAsync(Exception exception)
         {
             var user = DbContext.Users.SingleOrDefault(u => u.UID == AuthenticatedUserId);
             if (user != null)
             {
-                var otherUsers = DbContext.ClientPairs
-                    .Include(u => u.User)
-                    .Include(u => u.OtherUser)
-                    .Where(w => w.User == user)
-                    .Where(w => !string.IsNullOrEmpty(w.OtherUser.CharacterIdentification))
-                    .Select(e => e.OtherUser).ToList();
-                var otherEntries = DbContext.ClientPairs.Include(u => u.User)
-                    .Where(u => otherUsers.Any(e => e == u.User) && u.OtherUser == user).ToList();
-                _ = Clients.Users(otherEntries.Select(e => e.User.UID)).SendAsync("RemoveOnlinePairedPlayer", user.CharacterIdentification);
+                Logger.LogInformation("Disconnect from " + AuthenticatedUserId);
 
-                var outdatedVisibilities = DbContext.Visibilities.Where(v => v.CID == user.CharacterIdentification);
-                DbContext.RemoveRange(outdatedVisibilities);
                 var outdatedCharacterData = DbContext.CharacterData.Where(v => v.UserId == user.UID);
                 DbContext.RemoveRange(outdatedCharacterData);
                 user.CharacterIdentification = null;
-                DbContext.SaveChanges();
+                await DbContext.SaveChangesAsync();
+
+                var otherUsers = DbContext.ClientPairs
+                    .Include(u => u.User)
+                    .Include(u => u.OtherUser)
+                    .Where(w => w.User == user && !w.IsPaused)
+                    .Where(w => !string.IsNullOrEmpty(w.OtherUser.CharacterIdentification))
+                    .Select(e => e.OtherUser).ToList();
+                var otherEntries = DbContext.ClientPairs.Include(u => u.User)
+                    .Where(u => otherUsers.Any(e => e == u.User) && u.OtherUser == user && !u.IsPaused).ToList();
+                await Clients.Users(otherEntries.Select(e => e.User.UID)).SendAsync("RemoveOnlinePairedPlayer", user.CharacterIdentification);
+                await Clients.All.SendAsync("UsersOnline",
+                    await DbContext.Users.CountAsync(u => !string.IsNullOrEmpty(u.CharacterIdentification)));
             }
 
-            return base.OnDisconnectedAsync(exception);
+            await base.OnDisconnectedAsync(exception);
         }
     }
 }

MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <UserSecretsId>aspnet-MareSynchronosServer-BA82A12A-0B30-463C-801D-B7E81318CD50</UserSecretsId>
   </PropertyGroup>
 
@@ -13,6 +13,7 @@
     <PackageReference Include="Microsoft.AspNetCore.Identity.UI" Version="5.0.17" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="5.0.17" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="5.0.17" />
+    <PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="6.0.0" />
   </ItemGroup>
 
   <ItemGroup>

MareSynchronosServer/MareSynchronosServer/Models/CharacterData.cs

@@ -7,8 +7,7 @@ namespace MareSynchronosServer.Models
     {
         public string UserId { get; set; }
         public int JobId { get; set; }
-        public List<FileReplacementDto> EquipmentData { get; set; }
-        public string GlamourerData { get; set; }
+        public CharacterCacheDto CharacterCache { get; set; }
         public string Hash { get; set; }
     }
 }

MareSynchronosServer/MareSynchronosServer/Models/Visibility.cs

@@ -1,8 +0,0 @@
-namespace MareSynchronosServer.Models
-{
-    public class Visibility
-    {
-        public string CID { get; set; }
-        public string OtherCID { get; set; }
-    }
-}

MareSynchronosServer/MareSynchronosServer/Program.cs

@@ -1,11 +1,6 @@
 using Microsoft.AspNetCore.Hosting;
-using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
-using System;
-using System.Collections.Generic;
 using System.Linq;
-using System.Threading.Tasks;
 using MareSynchronosServer.Data;
 using Microsoft.Extensions.DependencyInjection;
 
@@ -22,15 +17,23 @@ namespace MareSynchronosServer
                 var services = scope.ServiceProvider;
                 var context = services.GetRequiredService<MareDbContext>();
                 context.Database.EnsureCreated();
+                var users = context.Users.Where(u => u.CharacterIdentification != null);
+                foreach (var user in users)
+                {
+                    user.CharacterIdentification = string.Empty;
+                }
+                context.CharacterData.RemoveRange(context.CharacterData);
+                var looseFiles = context.Files.Where(f => f.Uploaded == false);
+                context.RemoveRange(looseFiles);
+                context.SaveChanges();
             }
 
             host.Run();
         }
 
-        // Server=localhost\SQLEXPRESS;Database=master;Trusted_Connection=True;
-
         public static IHostBuilder CreateHostBuilder(string[] args) =>
             Host.CreateDefaultBuilder(args)
+                .UseConsoleLifetime()
                 .ConfigureWebHostDefaults(webBuilder =>
                 {
                     webBuilder.UseStartup<Startup>();

MareSynchronosServer/MareSynchronosServer/Properties/launchSettings.json

@@ -1,18 +1,11 @@
 {
-  "iisSettings": {
-    "windowsAuthentication": false,
-    "anonymousAuthentication": true,
-    "iisExpress": {
-      "applicationUrl": "http://localhost:43136",
-      "sslPort": 44364
-    }
-  },
   "profiles": {
     "MareSynchronosServer": {
       "commandName": "Project",
       "dotnetRunMessages": "true",
       "launchBrowser": false,
-      "applicationUrl": "https://localhost:5001;http://localhost:5000;https://darkarchon.internet-box.ch:5001;http://darkarchon.internet-box.ch:5000",
+      //"applicationUrl": "https://localhost:5001;http://localhost:5000;https://192.168.1.124:5001;http://192.168.1.124:5000",
+      "applicationUrl": "http://localhost:5000;https://localhost:5001;https://darkarchon.internet-box.ch:5001",
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development"
       }

MareSynchronosServer/MareSynchronosServer/Startup.cs

@@ -1,26 +1,17 @@
+using System;
 using Microsoft.AspNetCore.Builder;
-using Microsoft.AspNetCore.Components;
-using Microsoft.AspNetCore.Components.Authorization;
 using Microsoft.AspNetCore.Hosting;
-using Microsoft.AspNetCore.HttpsPolicy;
-using Microsoft.AspNetCore.Identity;
-using Microsoft.AspNetCore.Identity.UI;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Threading.Tasks;
-using Bazinga.AspNetCore.Authentication.Basic;
 using MareSynchronosServer.Authentication;
 using MareSynchronosServer.Data;
 using MareSynchronosServer.Hubs;
 using Microsoft.AspNetCore.Authentication;
-using Microsoft.AspNetCore.ResponseCompression;
+using Microsoft.AspNetCore.Http.Connections;
 using Microsoft.AspNetCore.SignalR;
+using WebSocketOptions = Microsoft.AspNetCore.Builder.WebSocketOptions;
 
 namespace MareSynchronosServer
 {
@@ -41,19 +32,26 @@ namespace MareSynchronosServer
             services.AddSignalR(hubOptions =>
             {
                 hubOptions.MaximumReceiveMessageSize = long.MaxValue;
+                hubOptions.EnableDetailedErrors = true;
+                hubOptions.MaximumParallelInvocationsPerClient = 10;
+                hubOptions.StreamBufferCapacity = 200;
             });
+
             services.AddSingleton<IUserIdProvider, IdBasedUserIdProvider>();
 
-            services.AddResponseCompression(opts =>
-            {
-                opts.MimeTypes = ResponseCompressionDefaults.MimeTypes.Concat(new[] { "application/octet-stream" });
-            });
+
             services.AddDbContext<MareDbContext>(options =>
             {
                 options.UseSqlServer(Configuration.GetConnectionString("DefaultConnection"));
             });
+
+            services.AddHostedService<FileCleanupService>();
+
             services.AddDatabaseDeveloperPageExceptionFilter();
-            services.AddAuthentication(options => options.DefaultScheme = SecretKeyAuthenticationHandler.AUTH_SCHEME)
+            services.AddAuthentication(options =>
+                {
+                    options.DefaultScheme = SecretKeyAuthenticationHandler.AUTH_SCHEME;
+                })
                 .AddScheme<AuthenticationSchemeOptions, SecretKeyAuthenticationHandler>(SecretKeyAuthenticationHandler.AUTH_SCHEME, options => { });
         }
 
@@ -76,18 +74,31 @@ namespace MareSynchronosServer
             app.UseStaticFiles();
 
             app.UseRouting();
+            var webSocketOptions = new WebSocketOptions
+            {
+                KeepAliveInterval = TimeSpan.FromSeconds(10),
+            };
+
+            app.UseWebSockets(webSocketOptions);
 
             app.UseAuthentication();
             app.UseAuthorization();
 
             app.UseEndpoints(endpoints =>
             {
-                endpoints.MapHub<Connection>("/heartbeat");
-                endpoints.MapHub<User>("/user");
-                endpoints.MapHub<Files>("/files", options =>
+                endpoints.MapHub<ConnectionHub>("/heartbeat", options =>
+                {
+                    options.Transports = HttpTransportType.WebSockets | HttpTransportType.LongPolling;
+                });
+                endpoints.MapHub<UserHub>("/user", options =>
+                {
+                    options.Transports = HttpTransportType.WebSockets | HttpTransportType.LongPolling;
+                });
+                endpoints.MapHub<FilesHub>("/files", options =>
                 {
                     options.ApplicationMaxBufferSize = long.MaxValue;
                     options.TransportMaxBufferSize = long.MaxValue;
+                    options.Transports = HttpTransportType.WebSockets | HttpTransportType.LongPolling;
                 });
             });
         }

MareSynchronosServer/MareSynchronosServer/appsettings.json

@@ -6,8 +6,28 @@
     "LogLevel": {
       "Default": "Information",
       "Microsoft": "Warning",
-      "Microsoft.Hosting.Lifetime": "Information"
+      "Microsoft.Hosting.Lifetime": "Information",
+      "MareSynchronosServer.Authentication": "Warning",
+      "System.IO.IOException": "Warning"
     }
   },
-  "AllowedHosts": "*"
+  "UnusedFileRetentionPeriodInDays" : 7, 
+  "CacheDirectory":  "G:\\ServerTest", // do not delete this key and set it to the path where the files will be stored
+  "AllowedHosts": "*",
+  "Kestrel": {
+    "Endpoints": {
+      "Https": {
+        "Url": "https://+:5001",
+        "Certificate": {
+          "Subject": "darkarchon.internet-box.ch",
+          "Store": "Root",
+          "Location": "LocalMachine",
+          "AllowInvalid": false
+          //          "Path": "", //use path, keypath and password to provide a valid certificate if not using windows key store
+          //          "KeyPath": ""
+          //          "Password": ""
+        }
+      }
+    }
+  }
 }