This commit is contained in:
Zurazan
2025-08-27 03:02:29 +02:00
commit 80235a174b
344 changed files with 43249 additions and 0 deletions

View File

@@ -0,0 +1,42 @@
using LightlessSyncShared.Data;
using Microsoft.EntityFrameworkCore;
namespace LightlessSyncServer.Services;
public class CharaDataCleanupService : BackgroundService
{
private readonly ILogger<CharaDataCleanupService> _logger;
private readonly IDbContextFactory<LightlessDbContext> _dbContextFactory;
public CharaDataCleanupService(ILogger<CharaDataCleanupService> logger, IDbContextFactory<LightlessDbContext> dbContextFactory)
{
_logger = logger;
_dbContextFactory = dbContextFactory;
}
public override async Task StartAsync(CancellationToken cancellationToken)
{
await base.StartAsync(cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Chara Data Cleanup Service started");
}
protected override async Task ExecuteAsync(CancellationToken ct)
{
_logger.LogInformation("CharaData Cleanup Service started");
while (!ct.IsCancellationRequested)
{
using (var db = await _dbContextFactory.CreateDbContextAsync(ct).ConfigureAwait(false))
{
var dateTime = DateTime.UtcNow;
var expiredData = await db.CharaData.Where(c => c.ExpiryDate <= DateTime.UtcNow).ToListAsync(cancellationToken: ct).ConfigureAwait(false);
_logger.LogInformation("Removing {count} expired Chara Data entries", expiredData.Count);
db.RemoveRange(expiredData);
await db.SaveChangesAsync(ct).ConfigureAwait(false);
}
await Task.Delay(TimeSpan.FromHours(12), ct).ConfigureAwait(false);
}
}
}

View File

@@ -0,0 +1,190 @@

using LightlessSyncShared.Data;
using LightlessSyncShared.Models;
using LightlessSyncShared.Services;
using LightlessSyncShared.Utils.Configuration;
using Microsoft.EntityFrameworkCore;
using System.Collections.Concurrent;
using System.Data;
using System.Diagnostics;
namespace LightlessSyncServer.Services;
public class ClientPairPermissionsCleanupService(ILogger<ClientPairPermissionsCleanupService> _logger, IDbContextFactory<LightlessDbContext> _dbContextFactory,
IConfigurationService<ServerConfiguration> _configurationService)
: BackgroundService
{
public override async Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Client Pair Permissions Cleanup Service started");
await base.StartAsync(cancellationToken).ConfigureAwait(false);
}
private async Task AllUsersPermissionsCleanup(CancellationToken ct)
{
const int MaxParallelism = 8;
const int MaxProcessingPerChunk = 1000000;
long removedEntries = 0;
long priorRemovedEntries = 0;
ConcurrentDictionary<int, List<UserPermissionSet>> toRemovePermsParallel = [];
ConcurrentDictionary<int, bool> completionDebugPrint = [];
int parallelProcessed = 0;
int userNo = 0;
int lastUserNo = 0;
using var db = await _dbContextFactory.CreateDbContextAsync(ct).ConfigureAwait(false);
_logger.LogInformation("Building All Pairs");
_logger.LogInformation("Collecting Users");
var users = (await db.Users.Select(k => k.UID).AsNoTracking().ToListAsync(ct).ConfigureAwait(false)).Order(StringComparer.Ordinal).ToList();
Stopwatch st = Stopwatch.StartNew();
while (userNo < users.Count)
{
using CancellationTokenSource loopCts = new();
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(loopCts.Token, ct);
try
{
await Parallel.ForAsync(userNo, users.Count, new ParallelOptions()
{
MaxDegreeOfParallelism = MaxParallelism,
CancellationToken = linkedCts.Token
},
async (i, token) =>
{
var userNoInc = Interlocked.Increment(ref userNo);
using var db2 = await _dbContextFactory.CreateDbContextAsync(token).ConfigureAwait(false);
var user = users[i];
var personalPairs = await GetAllPairsForUser(user, db2, ct).ConfigureAwait(false);
toRemovePermsParallel[i] = await UserPermissionCleanup(i, users.Count, user, db2, personalPairs).ConfigureAwait(false);
var processedAdd = Interlocked.Add(ref parallelProcessed, toRemovePermsParallel[i].Count);
var completionPcnt = userNoInc / (double)users.Count;
var completionInt = (int)(completionPcnt * 100);
if (completionInt > 0 && (!completionDebugPrint.TryGetValue(completionInt, out bool posted) || !posted))
{
completionDebugPrint[completionInt] = true;
var elapsed = st.Elapsed;
var estimatedTimeLeft = (elapsed / completionPcnt) - elapsed;
_logger.LogInformation("Progress: {no}/{total} ({pct:P2}), removed so far: {removed}, planned next chunk: {planned}, estimated time left: {time}",
userNoInc, users.Count, completionPcnt, removedEntries, processedAdd, estimatedTimeLeft);
if (userNoInc / (double)users.Count - lastUserNo / (double)users.Count > 0.05)
{
// 5% processed without writing, might as well save at this point
await loopCts.CancelAsync().ConfigureAwait(false);
}
}
if (processedAdd > MaxProcessingPerChunk)
await loopCts.CancelAsync().ConfigureAwait(false);
}).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
// expected
}
removedEntries += parallelProcessed;
try
{
parallelProcessed = 0;
_logger.LogInformation("Removing {newDeleted} entities and writing to database", removedEntries - priorRemovedEntries);
db.Permissions.RemoveRange(toRemovePermsParallel.Values.SelectMany(v => v).ToList());
await db.SaveChangesAsync(ct).ConfigureAwait(false);
_logger.LogInformation("Removed {newDeleted} entities, settling...", removedEntries - priorRemovedEntries);
priorRemovedEntries = removedEntries;
lastUserNo = userNo;
}
catch (DBConcurrencyException ex)
{
_logger.LogWarning(ex, "Concurrency Exception during User Permissions Cleanup, restarting at {last}", lastUserNo);
userNo = lastUserNo;
removedEntries = priorRemovedEntries;
continue;
}
finally
{
toRemovePermsParallel.Clear();
}
}
st.Stop();
_logger.LogInformation("User Permissions Cleanup Finished, removed {total} stale permissions in {time}", removedEntries, st.Elapsed);
}
private async Task<List<UserPermissionSet>> UserPermissionCleanup(int userNr, int totalUsers, string uid, LightlessDbContext dbContext, List<string> pairs)
{
var perms = dbContext.Permissions.Where(p => p.UserUID == uid && !p.Sticky && !pairs.Contains(p.OtherUserUID));
var permsToRemoveCount = await perms.CountAsync().ConfigureAwait(false);
if (permsToRemoveCount == 0)
return [];
_logger.LogInformation("[{current}/{totalCount}] User {user}: Planning to remove {removed} permissions", userNr, totalUsers, uid, permsToRemoveCount);
return await perms.ToListAsync().ConfigureAwait(false);
}
private async Task<List<string>> GetAllPairsForUser(string uid, LightlessDbContext dbContext, CancellationToken ct)
{
var entries = await dbContext.ClientPairs.AsNoTracking().Where(k => k.UserUID == uid).Select(k => k.OtherUserUID)
.Concat(
dbContext.GroupPairs.Where(k => k.GroupUserUID == uid).AsNoTracking()
.Join(dbContext.GroupPairs.AsNoTracking(),
a => a.GroupGID,
b => b.GroupGID,
(a, b) => b.GroupUserUID)
.Where(a => a != uid))
.ToListAsync(ct).ConfigureAwait(false);
return entries.Distinct(StringComparer.Ordinal).ToList();
}
protected override async Task ExecuteAsync(CancellationToken ct)
{
if (!_configurationService.GetValueOrDefault(nameof(ServerConfiguration.RunPermissionCleanupOnStartup), defaultValue: true))
{
await WaitUntilNextCleanup(ct).ConfigureAwait(false);
}
while (!ct.IsCancellationRequested)
{
try
{
_logger.LogInformation("Starting Permissions Cleanup");
await AllUsersPermissionsCleanup(ct).ConfigureAwait(false);
}
catch (DbUpdateConcurrencyException ex)
{
_logger.LogWarning(ex, "Concurrency Exception during User Permissions Cleanup");
continue;
}
catch (Exception ex)
{
_logger.LogError(ex, "Unhandled Exception during User Permissions Cleanup");
}
await WaitUntilNextCleanup(ct).ConfigureAwait(false);
}
}
private async Task WaitUntilNextCleanup(CancellationToken token)
{
var now = DateTime.UtcNow;
var nextRun = new DateTime(now.Year, now.Month, now.Day, 12, 0, 0, DateTimeKind.Utc);
if (now > nextRun) nextRun = nextRun.AddDays(1);
var nextRunSpan = nextRun - now;
_logger.LogInformation("Permissions Cleanup next run in {span}", nextRunSpan);
await Task.Delay(nextRunSpan, token).ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,226 @@
using LightlessSync.API.Dto.CharaData;
using LightlessSync.API.SignalR;
using LightlessSyncServer.Hubs;
using Microsoft.AspNetCore.SignalR;
using StackExchange.Redis.Extensions.Core.Abstractions;
namespace LightlessSyncServer.Services;
public sealed class GPoseLobbyDistributionService : IHostedService, IDisposable
{
private CancellationTokenSource _runtimeCts = new();
private readonly Dictionary<string, Dictionary<string, WorldData>> _lobbyWorldData = [];
private readonly Dictionary<string, Dictionary<string, PoseData>> _lobbyPoseData = [];
private readonly SemaphoreSlim _lobbyPoseDataModificationSemaphore = new(1, 1);
private readonly SemaphoreSlim _lobbyWorldDataModificationSemaphore = new(1, 1);
public GPoseLobbyDistributionService(ILogger<GPoseLobbyDistributionService> logger, IRedisDatabase redisDb,
IHubContext<LightlessHub, ILightlessHub> hubContext)
{
_logger = logger;
_redisDb = redisDb;
_hubContext = hubContext;
}
private bool _disposed;
private readonly ILogger<GPoseLobbyDistributionService> _logger;
private readonly IRedisDatabase _redisDb;
private readonly IHubContext<LightlessHub, ILightlessHub> _hubContext;
public void Dispose()
{
if (_disposed)
{
return;
}
_runtimeCts.Cancel();
_runtimeCts.Dispose();
_lobbyPoseDataModificationSemaphore.Dispose();
_lobbyWorldDataModificationSemaphore.Dispose();
_disposed = true;
}
public async Task PushWorldData(string lobby, string user, WorldData worldData)
{
await _lobbyWorldDataModificationSemaphore.WaitAsync().ConfigureAwait(false);
try
{
if (!_lobbyWorldData.TryGetValue(lobby, out var worldDataDict))
{
_lobbyWorldData[lobby] = worldDataDict = new(StringComparer.Ordinal);
}
worldDataDict[user] = worldData;
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during Pushing World Data for Lobby {lobby} by User {user}", lobby, user);
}
finally
{
_lobbyWorldDataModificationSemaphore.Release();
}
}
public async Task PushPoseData(string lobby, string user, PoseData poseData)
{
await _lobbyPoseDataModificationSemaphore.WaitAsync().ConfigureAwait(false);
try
{
if (!_lobbyPoseData.TryGetValue(lobby, out var poseDataDict))
{
_lobbyPoseData[lobby] = poseDataDict = new(StringComparer.Ordinal);
}
poseDataDict[user] = poseData;
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during Pushing World Data for Lobby {lobby} by User {user}", lobby, user);
}
finally
{
_lobbyPoseDataModificationSemaphore.Release();
}
}
public Task StartAsync(CancellationToken cancellationToken)
{
_ = WorldDataDistribution(_runtimeCts.Token);
_ = PoseDataDistribution(_runtimeCts.Token);
return Task.CompletedTask;
}
private async Task WorldDataDistribution(CancellationToken token)
{
while (!token.IsCancellationRequested)
{
try
{
await DistributeWorldData(token).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during World Data Distribution");
}
await Task.Delay(TimeSpan.FromSeconds(1), token).ConfigureAwait(false);
}
}
private async Task PoseDataDistribution(CancellationToken token)
{
while (!token.IsCancellationRequested)
{
try
{
await DistributePoseData(token).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during Pose Data Distribution");
}
await Task.Delay(TimeSpan.FromSeconds(10), token).ConfigureAwait(false);
}
}
private async Task DistributeWorldData(CancellationToken token)
{
await _lobbyWorldDataModificationSemaphore.WaitAsync(token).ConfigureAwait(false);
Dictionary<string, Dictionary<string, WorldData>> clone = [];
try
{
clone = _lobbyWorldData.ToDictionary(k => k.Key, k => k.Value, StringComparer.Ordinal);
_lobbyWorldData.Clear();
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during Distributing World Data Clone generation");
_lobbyWorldData.Clear();
return;
}
finally
{
_lobbyWorldDataModificationSemaphore.Release();
}
foreach (var lobbyId in clone)
{
token.ThrowIfCancellationRequested();
try
{
if (!lobbyId.Value.Values.Any())
continue;
var gposeLobbyUsers = await _redisDb.GetAsync<List<string>>($"GposeLobby:{lobbyId.Key}").ConfigureAwait(false);
if (gposeLobbyUsers == null)
continue;
foreach (var data in lobbyId.Value)
{
await _hubContext.Clients.Users(gposeLobbyUsers.Where(k => !string.Equals(k, data.Key, StringComparison.Ordinal)))
.Client_GposeLobbyPushWorldData(new(data.Key), data.Value).ConfigureAwait(false);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during World Data Distribution for Lobby {lobby}", lobbyId.Key);
continue;
}
}
}
private async Task DistributePoseData(CancellationToken token)
{
await _lobbyPoseDataModificationSemaphore.WaitAsync(token).ConfigureAwait(false);
Dictionary<string, Dictionary<string, PoseData>> clone = [];
try
{
clone = _lobbyPoseData.ToDictionary(k => k.Key, k => k.Value, StringComparer.Ordinal);
_lobbyPoseData.Clear();
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during Distributing Pose Data Clone generation");
_lobbyPoseData.Clear();
return;
}
finally
{
_lobbyPoseDataModificationSemaphore.Release();
}
foreach (var lobbyId in clone)
{
token.ThrowIfCancellationRequested();
try
{
if (!lobbyId.Value.Values.Any())
continue;
var gposeLobbyUsers = await _redisDb.GetAsync<List<string>>($"GposeLobby:{lobbyId.Key}").ConfigureAwait(false);
if (gposeLobbyUsers == null)
continue;
foreach (var data in lobbyId.Value)
{
await _hubContext.Clients.Users(gposeLobbyUsers.Where(k => !string.Equals(k, data.Key, StringComparison.Ordinal)))
.Client_GposeLobbyPushPoseData(new(data.Key), data.Value).ConfigureAwait(false);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during Pose Data Distribution for Lobby {lobby}", lobbyId.Key);
continue;
}
}
}
public Task StopAsync(CancellationToken cancellationToken)
{
_runtimeCts.Cancel();
return Task.CompletedTask;
}
}

View File

@@ -0,0 +1,182 @@
using LightlessSync.API.Dto.User;
using Microsoft.VisualBasic.FileIO;
using Prometheus;
using System.Collections.Concurrent;
using System.Globalization;
namespace LightlessSyncServer.Services;
public class LightlessCensus : IHostedService
{
private record CensusEntry(ushort WorldId, short Race, short Subrace, short Gender)
{
public static CensusEntry FromDto(CensusDataDto dto)
{
return new CensusEntry(dto.WorldId, dto.RaceId, dto.TribeId, dto.Gender);
}
}
private readonly ConcurrentDictionary<string, CensusEntry> _censusEntries = new(StringComparer.Ordinal);
private readonly Dictionary<short, string> _dcs = new();
private readonly Dictionary<short, string> _gender = new();
private readonly ILogger<LightlessCensus> _logger;
private readonly Dictionary<short, string> _races = new();
private readonly Dictionary<short, string> _tribes = new();
private readonly Dictionary<ushort, (string, short)> _worlds = new();
private Gauge? _gauge;
public LightlessCensus(ILogger<LightlessCensus> logger)
{
_logger = logger;
}
private bool Initialized => _gauge != null;
public void ClearStatistics(string uid)
{
if (!Initialized) return;
if (_censusEntries.Remove(uid, out var censusEntry))
{
ModifyGauge(censusEntry, increase: false);
}
}
public void PublishStatistics(string uid, CensusDataDto? censusDataDto)
{
if (!Initialized || censusDataDto == null) return;
var newEntry = CensusEntry.FromDto(censusDataDto);
if (_censusEntries.TryGetValue(uid, out var entry))
{
if (entry != newEntry)
{
ModifyGauge(entry, increase: false);
ModifyGauge(newEntry, increase: true);
_censusEntries[uid] = newEntry;
}
}
else
{
_censusEntries[uid] = newEntry;
ModifyGauge(newEntry, increase: true);
}
}
public async Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Loading XIVAPI data");
using HttpClient client = new HttpClient();
Dictionary<ushort, short> worldDcs = new();
var dcs = await client.GetStringAsync("https://raw.githubusercontent.com/xivapi/ffxiv-datamining/master/csv/WorldDCGroupType.csv", cancellationToken).ConfigureAwait(false);
// dc: https://raw.githubusercontent.com/xivapi/ffxiv-datamining/master/csv/WorldDCGroupType.csv
// id, name, region
using var dcsReader = new StringReader(dcs);
using var dcsParser = new TextFieldParser(dcsReader);
dcsParser.Delimiters = [","];
// read 3 lines and discard
dcsParser.ReadLine(); dcsParser.ReadLine(); dcsParser.ReadLine();
while (!dcsParser.EndOfData)
{
var fields = dcsParser.ReadFields();
var id = short.Parse(fields[0], CultureInfo.InvariantCulture);
var name = fields[1];
if (string.IsNullOrEmpty(name) || id == 0) continue;
_logger.LogInformation("DC: ID: {id}, Name: {name}", id, name);
_dcs[id] = name;
}
var worlds = await client.GetStringAsync("https://raw.githubusercontent.com/xivapi/ffxiv-datamining/master/csv/World.csv", cancellationToken).ConfigureAwait(false);
// world: https://raw.githubusercontent.com/xivapi/ffxiv-datamining/master/csv/World.csv
// id, internalname, name, region, usertype, datacenter, ispublic
using var worldsReader = new StringReader(worlds);
using var worldsParser = new TextFieldParser(worldsReader);
worldsParser.Delimiters = [","];
// read 3 lines and discard
worldsParser.ReadLine(); worldsParser.ReadLine(); worldsParser.ReadLine();
while (!worldsParser.EndOfData)
{
var fields = worldsParser.ReadFields();
var id = ushort.Parse(fields[0], CultureInfo.InvariantCulture);
var name = fields[1];
var dc = short.Parse(fields[5], CultureInfo.InvariantCulture);
var isPublic = bool.Parse(fields[6]);
if (!_dcs.ContainsKey(dc) || !isPublic) continue;
_worlds[id] = (name, dc);
_logger.LogInformation("World: ID: {id}, Name: {name}, DC: {dc}", id, name, dc);
}
var races = await client.GetStringAsync("https://raw.githubusercontent.com/xivapi/ffxiv-datamining/master/csv/Race.csv", cancellationToken).ConfigureAwait(false);
// race: https://raw.githubusercontent.com/xivapi/ffxiv-datamining/master/csv/Race.csv
// id, masc name, fem name, other crap I don't care about
using var raceReader = new StringReader(races);
using var raceParser = new TextFieldParser(raceReader);
raceParser.Delimiters = [","];
// read 3 lines and discard
raceParser.ReadLine(); raceParser.ReadLine(); raceParser.ReadLine();
while (!raceParser.EndOfData)
{
var fields = raceParser.ReadFields();
var id = short.Parse(fields[0], CultureInfo.InvariantCulture);
var name = fields[1];
if (string.IsNullOrEmpty(name) || id == 0) continue;
_races[id] = name;
_logger.LogInformation("Race: ID: {id}, Name: {name}", id, name);
}
var tribe = await client.GetStringAsync("https://raw.githubusercontent.com/xivapi/ffxiv-datamining/master/csv/Tribe.csv", cancellationToken).ConfigureAwait(false);
// tribe: https://raw.githubusercontent.com/xivapi/ffxiv-datamining/master/csv/Tribe.csv
// id masc name, fem name, other crap I don't care about
using var tribeReader = new StringReader(tribe);
using var tribeParser = new TextFieldParser(tribeReader);
tribeParser.Delimiters = [","];
// read 3 lines and discard
tribeParser.ReadLine(); tribeParser.ReadLine(); tribeParser.ReadLine();
while (!tribeParser.EndOfData)
{
var fields = tribeParser.ReadFields();
var id = short.Parse(fields[0], CultureInfo.InvariantCulture);
var name = fields[1];
if (string.IsNullOrEmpty(name) || id == 0) continue;
_tribes[id] = name;
_logger.LogInformation("Tribe: ID: {id}, Name: {name}", id, name);
}
_gender[0] = "Male";
_gender[1] = "Female";
_gauge = Metrics.CreateGauge("lightless_census", "lightless informational census data", new[] { "dc", "world", "gender", "race", "subrace" });
}
public Task StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
private void ModifyGauge(CensusEntry censusEntry, bool increase)
{
var subraceSuccess = _tribes.TryGetValue(censusEntry.Subrace, out var subrace);
var raceSuccess = _races.TryGetValue(censusEntry.Race, out var race);
var worldSuccess = _worlds.TryGetValue(censusEntry.WorldId, out var world);
var genderSuccess = _gender.TryGetValue(censusEntry.Gender, out var gender);
if (subraceSuccess && raceSuccess && worldSuccess && genderSuccess && _dcs.TryGetValue(world.Item2, out var dc))
{
if (increase)
_gauge.WithLabels(dc, world.Item1, gender, race, subrace).Inc();
else
_gauge.WithLabels(dc, world.Item1, gender, race, subrace).Dec();
}
}
}

View File

@@ -0,0 +1,145 @@
using LightlessSyncShared.Metrics;
namespace LightlessSyncServer.Services;
public class OnlineSyncedPairCacheService
{
private readonly Dictionary<string, PairCache> _lastSeenCache = new(StringComparer.Ordinal);
private readonly SemaphoreSlim _cacheModificationSemaphore = new(1);
private readonly ILogger<OnlineSyncedPairCacheService> _logger;
private readonly ILoggerFactory _loggerFactory;
private readonly LightlessMetrics _lightlessMetrics;
public OnlineSyncedPairCacheService(ILogger<OnlineSyncedPairCacheService> logger, ILoggerFactory loggerFactory, LightlessMetrics lightlessMetrics)
{
_logger = logger;
_loggerFactory = loggerFactory;
_lightlessMetrics = lightlessMetrics;
}
public async Task InitPlayer(string user)
{
if (_lastSeenCache.ContainsKey(user)) return;
await _cacheModificationSemaphore.WaitAsync().ConfigureAwait(false);
try
{
_logger.LogDebug("Initializing {user}", user);
_lastSeenCache[user] = new(_loggerFactory.CreateLogger<PairCache>(), user, _lightlessMetrics);
}
finally
{
_cacheModificationSemaphore.Release();
}
}
public async Task DisposePlayer(string user)
{
if (!_lastSeenCache.ContainsKey(user)) return;
await _cacheModificationSemaphore.WaitAsync().ConfigureAwait(false);
try
{
_logger.LogDebug("Disposing {user}", user);
_lastSeenCache.Remove(user, out var pairCache);
pairCache?.Dispose();
}
finally
{
_cacheModificationSemaphore.Release();
}
}
public async Task<bool> AreAllPlayersCached(string sender, List<string> uids, CancellationToken ct)
{
if (!_lastSeenCache.ContainsKey(sender)) await InitPlayer(sender).ConfigureAwait(false);
_lastSeenCache.TryGetValue(sender, out var pairCache);
return await pairCache.AreAllPlayersCached(uids, ct).ConfigureAwait(false);
}
public async Task CachePlayers(string sender, List<string> uids, CancellationToken ct)
{
if (!_lastSeenCache.ContainsKey(sender)) await InitPlayer(sender).ConfigureAwait(false);
_lastSeenCache.TryGetValue(sender, out var pairCache);
await pairCache.CachePlayers(uids, ct).ConfigureAwait(false);
}
private sealed class PairCache : IDisposable
{
private readonly ILogger<PairCache> _logger;
private readonly string _owner;
private readonly LightlessMetrics _metrics;
private readonly Dictionary<string, DateTime> _lastSeenCache = new(StringComparer.Ordinal);
private readonly SemaphoreSlim _lock = new(1);
public PairCache(ILogger<PairCache> logger, string owner, LightlessMetrics metrics)
{
metrics.IncGauge(MetricsAPI.GaugeUserPairCacheUsers);
_logger = logger;
_owner = owner;
_metrics = metrics;
}
public async Task<bool> AreAllPlayersCached(List<string> uids, CancellationToken ct)
{
await _lock.WaitAsync(ct).ConfigureAwait(false);
try
{
var allCached = uids.TrueForAll(u => _lastSeenCache.TryGetValue(u, out var expiry) && expiry > DateTime.UtcNow);
_logger.LogDebug("AreAllPlayersCached:{uid}:{count}:{result}", _owner, uids.Count, allCached);
if (allCached) _metrics.IncCounter(MetricsAPI.CounterUserPairCacheHit);
else _metrics.IncCounter(MetricsAPI.CounterUserPairCacheMiss);
return allCached;
}
finally
{
_lock.Release();
}
}
public async Task CachePlayers(List<string> uids, CancellationToken ct)
{
await _lock.WaitAsync(ct).ConfigureAwait(false);
try
{
var lastSeen = DateTime.UtcNow.AddMinutes(60);
_logger.LogDebug("CacheOnlinePlayers:{uid}:{count}", _owner, uids.Count);
var newEntries = uids.Count(u => !_lastSeenCache.ContainsKey(u));
_metrics.IncCounter(MetricsAPI.CounterUserPairCacheNewEntries, newEntries);
_metrics.IncCounter(MetricsAPI.CounterUserPairCacheUpdatedEntries, uids.Count - newEntries);
_metrics.IncGauge(MetricsAPI.GaugeUserPairCacheEntries, newEntries);
uids.ForEach(u => _lastSeenCache[u] = lastSeen);
// clean up old entries
var outdatedEntries = _lastSeenCache.Where(u => u.Value < DateTime.UtcNow).Select(k => k.Key).ToList();
if (outdatedEntries.Any())
{
_metrics.DecGauge(MetricsAPI.GaugeUserPairCacheEntries, outdatedEntries.Count);
foreach (var entry in outdatedEntries)
{
_lastSeenCache.Remove(entry);
}
}
}
finally
{
_lock.Release();
}
}
public void Dispose()
{
_metrics.DecGauge(MetricsAPI.GaugeUserPairCacheUsers);
_metrics.DecGauge(MetricsAPI.GaugeUserPairCacheEntries, _lastSeenCache.Count);
_lock.Dispose();
}
}
}

View File

@@ -0,0 +1,84 @@
using LightlessSync.API.Dto;
using LightlessSync.API.SignalR;
using LightlessSyncServer.Hubs;
using LightlessSyncShared.Data;
using LightlessSyncShared.Metrics;
using LightlessSyncShared.Services;
using LightlessSyncShared.Utils.Configuration;
using Microsoft.AspNetCore.SignalR;
using Microsoft.EntityFrameworkCore;
using StackExchange.Redis.Extensions.Core.Abstractions;
namespace LightlessSyncServer.Services;
public sealed class SystemInfoService : BackgroundService
{
private readonly LightlessMetrics _lightlessMetrics;
private readonly IConfigurationService<ServerConfiguration> _config;
private readonly IDbContextFactory<LightlessDbContext> _dbContextFactory;
private readonly ILogger<SystemInfoService> _logger;
private readonly IHubContext<LightlessHub, ILightlessHub> _hubContext;
private readonly IRedisDatabase _redis;
public SystemInfoDto SystemInfoDto { get; private set; } = new();
public SystemInfoService(LightlessMetrics lightlessMetrics, IConfigurationService<ServerConfiguration> configurationService, IDbContextFactory<LightlessDbContext> dbContextFactory,
ILogger<SystemInfoService> logger, IHubContext<LightlessHub, ILightlessHub> hubContext, IRedisDatabase redisDb)
{
_lightlessMetrics = lightlessMetrics;
_config = configurationService;
_dbContextFactory = dbContextFactory;
_logger = logger;
_hubContext = hubContext;
_redis = redisDb;
}
public override async Task StartAsync(CancellationToken cancellationToken)
{
await base.StartAsync(cancellationToken).ConfigureAwait(false);
_logger.LogInformation("System Info Service started");
}
protected override async Task ExecuteAsync(CancellationToken ct)
{
var timeOut = _config.IsMain ? 15 : 30;
while (!ct.IsCancellationRequested)
{
try
{
ThreadPool.GetAvailableThreads(out int workerThreads, out int ioThreads);
_lightlessMetrics.SetGaugeTo(MetricsAPI.GaugeAvailableWorkerThreads, workerThreads);
_lightlessMetrics.SetGaugeTo(MetricsAPI.GaugeAvailableIOWorkerThreads, ioThreads);
var onlineUsers = (_redis.SearchKeysAsync("UID:*").GetAwaiter().GetResult()).Count();
SystemInfoDto = new SystemInfoDto()
{
OnlineUsers = onlineUsers,
};
if (_config.IsMain)
{
_logger.LogInformation("Sending System Info, Online Users: {onlineUsers}", onlineUsers);
await _hubContext.Clients.All.Client_UpdateSystemInfo(SystemInfoDto).ConfigureAwait(false);
using var db = await _dbContextFactory.CreateDbContextAsync(ct).ConfigureAwait(false);
_lightlessMetrics.SetGaugeTo(MetricsAPI.GaugeAuthorizedConnections, onlineUsers);
_lightlessMetrics.SetGaugeTo(MetricsAPI.GaugePairs, db.ClientPairs.AsNoTracking().Count());
_lightlessMetrics.SetGaugeTo(MetricsAPI.GaugePairsPaused, db.Permissions.AsNoTracking().Where(p => p.IsPaused).Count());
_lightlessMetrics.SetGaugeTo(MetricsAPI.GaugeGroups, db.Groups.AsNoTracking().Count());
_lightlessMetrics.SetGaugeTo(MetricsAPI.GaugeGroupPairs, db.GroupPairs.AsNoTracking().Count());
_lightlessMetrics.SetGaugeTo(MetricsAPI.GaugeUsersRegistered, db.Users.AsNoTracking().Count());
}
await Task.Delay(TimeSpan.FromSeconds(timeOut), ct).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to push system info");
}
}
}
}

View File

@@ -0,0 +1,194 @@
using LightlessSyncShared.Data;
using LightlessSyncShared.Metrics;
using LightlessSyncShared.Models;
using LightlessSyncShared.Services;
using LightlessSyncShared.Utils;
using LightlessSyncShared.Utils.Configuration;
using Microsoft.EntityFrameworkCore;
namespace LightlessSyncServer.Services;
public class UserCleanupService : IHostedService
{
private readonly LightlessMetrics metrics;
private readonly ILogger<UserCleanupService> _logger;
private readonly IDbContextFactory<LightlessDbContext> _lightlessDbContextFactory;
private readonly IConfigurationService<ServerConfiguration> _configuration;
private CancellationTokenSource _cleanupCts;
public UserCleanupService(LightlessMetrics metrics, ILogger<UserCleanupService> logger, IDbContextFactory<LightlessDbContext> lightlessDbContextFactory, IConfigurationService<ServerConfiguration> configuration)
{
this.metrics = metrics;
_logger = logger;
_lightlessDbContextFactory = lightlessDbContextFactory;
_configuration = configuration;
}
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Cleanup Service started");
_cleanupCts = new();
_ = CleanUp(_cleanupCts.Token);
return Task.CompletedTask;
}
private async Task CleanUp(CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
using (var dbContext = await _lightlessDbContextFactory.CreateDbContextAsync(ct).ConfigureAwait(false))
{
CleanUpOutdatedLodestoneAuths(dbContext);
await PurgeUnusedAccounts(dbContext).ConfigureAwait(false);
await PurgeTempInvites(dbContext).ConfigureAwait(false);
dbContext.SaveChanges();
}
var now = DateTime.Now;
TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % 10, 0);
var span = futureTime.AddMinutes(10) - currentTime;
_logger.LogInformation("User Cleanup Complete, next run at {date}", now.Add(span));
await Task.Delay(span, ct).ConfigureAwait(false);
}
}
private async Task PurgeTempInvites(LightlessDbContext dbContext)
{
try
{
var tempInvites = await dbContext.GroupTempInvites.ToListAsync().ConfigureAwait(false);
dbContext.RemoveRange(tempInvites.Where(i => i.ExpirationDate < DateTime.UtcNow));
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error during Temp Invite purge");
}
}
private async Task PurgeUnusedAccounts(LightlessDbContext dbContext)
{
try
{
if (_configuration.GetValueOrDefault(nameof(ServerConfiguration.PurgeUnusedAccounts), false))
{
var usersOlderThanDays = _configuration.GetValueOrDefault(nameof(ServerConfiguration.PurgeUnusedAccountsPeriodInDays), 14);
var maxGroupsByUser = _configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxGroupUserCount), 3);
_logger.LogInformation("Cleaning up users older than {usersOlderThanDays} days", usersOlderThanDays);
var allUsers = dbContext.Users.Where(u => string.IsNullOrEmpty(u.Alias)).ToList();
List<User> usersToRemove = new();
foreach (var user in allUsers)
{
if (user.LastLoggedIn < DateTime.UtcNow - TimeSpan.FromDays(usersOlderThanDays))
{
_logger.LogInformation("User outdated: {userUID}", user.UID);
usersToRemove.Add(user);
}
}
foreach (var user in usersToRemove)
{
await SharedDbFunctions.PurgeUser(_logger, user, dbContext, maxGroupsByUser).ConfigureAwait(false);
}
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error during user purge");
}
}
private void CleanUpOutdatedLodestoneAuths(LightlessDbContext dbContext)
{
try
{
_logger.LogInformation($"Cleaning up expired lodestone authentications");
var lodestoneAuths = dbContext.LodeStoneAuth.Include(u => u.User).Where(a => a.StartedAt != null).ToList();
List<LodeStoneAuth> expiredAuths = new List<LodeStoneAuth>();
foreach (var auth in lodestoneAuths)
{
if (auth.StartedAt < DateTime.UtcNow - TimeSpan.FromMinutes(15))
{
expiredAuths.Add(auth);
}
}
dbContext.Users.RemoveRange(expiredAuths.Where(u => u.User != null).Select(a => a.User));
dbContext.RemoveRange(expiredAuths);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error during expired auths cleanup");
}
}
public async Task PurgeUser(User user, LightlessDbContext dbContext)
{
_logger.LogInformation("Purging user: {uid}", user.UID);
var lodestone = dbContext.LodeStoneAuth.SingleOrDefault(a => a.User.UID == user.UID);
if (lodestone != null)
{
dbContext.Remove(lodestone);
}
var auth = dbContext.Auth.Single(a => a.UserUID == user.UID);
var userFiles = dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == user.UID).ToList();
dbContext.Files.RemoveRange(userFiles);
var ownPairData = dbContext.ClientPairs.Where(u => u.User.UID == user.UID).ToList();
dbContext.ClientPairs.RemoveRange(ownPairData);
var otherPairData = dbContext.ClientPairs.Include(u => u.User)
.Where(u => u.OtherUser.UID == user.UID).ToList();
dbContext.ClientPairs.RemoveRange(otherPairData);
var userJoinedGroups = await dbContext.GroupPairs.Include(g => g.Group).Where(u => u.GroupUserUID == user.UID).ToListAsync().ConfigureAwait(false);
foreach (var userGroupPair in userJoinedGroups)
{
bool ownerHasLeft = string.Equals(userGroupPair.Group.OwnerUID, user.UID, StringComparison.Ordinal);
if (ownerHasLeft)
{
var groupPairs = await dbContext.GroupPairs.Where(g => g.GroupGID == userGroupPair.GroupGID && g.GroupUserUID != user.UID).ToListAsync().ConfigureAwait(false);
if (!groupPairs.Any())
{
_logger.LogInformation("Group {gid} has no new owner, deleting", userGroupPair.GroupGID);
dbContext.Groups.Remove(userGroupPair.Group);
}
else
{
_ = await SharedDbFunctions.MigrateOrDeleteGroup(dbContext, userGroupPair.Group, groupPairs, _configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxExistingGroupsByUser), 3)).ConfigureAwait(false);
}
}
dbContext.GroupPairs.Remove(userGroupPair);
}
_logger.LogInformation("User purged: {uid}", user.UID);
dbContext.Auth.Remove(auth);
dbContext.Users.Remove(user);
await dbContext.SaveChangesAsync().ConfigureAwait(false);
}
public Task StopAsync(CancellationToken cancellationToken)
{
_cleanupCts.Cancel();
return Task.CompletedTask;
}
}