@@ -105,7 +114,14 @@ else
- @foreach (var leaderboard in LeaderboardData.Leaderboards)
+ @if(LeaderboardData.Length == 0)
+ {
+
+
No data was found for the selected servers and time range. Data is probably still collected and calculated. Please try again later. This will take a long time.
+
+ }
+
+ @foreach (var leaderboard in LeaderboardData)
{
@leaderboard.Name
if (leaderboard.ExtraInfo != null)
@@ -233,7 +249,7 @@ else
@code{
private bool IsLoading { get; set; } = true;
- private LeaderboardData? LeaderboardData { get; set; } = null;
+ private Models.Leaderboard[]? LeaderboardData { get; set; } = null;
private bool RequestedPrivate { get; set; } = false;
protected override async Task OnInitializedAsync()
@@ -267,7 +283,7 @@ else
}
}
- LeaderboardData? leaderboard = null;
+ Models.Leaderboard[]? leaderboard = null;
var authState = await AuthenticationStateProvider.GetAuthenticationStateAsync();
try
{
@@ -279,7 +295,7 @@ else
}
var selectedServersArray = selectedServers.Split(',');
- leaderboard = await LeaderboardService.GetLeaderboard(timeRangeEnum, username, selectedServersArray, authState, entries);
+ leaderboard = await LeaderboardService.GetLeaderboards(timeRangeEnum, username, selectedServersArray, authState, entries);
}
catch (UnauthorizedAccessException)
{
diff --git a/ReplayBrowser/Services/LeaderboardService.cs b/ReplayBrowser/Services/LeaderboardService.cs
index c36a26f..76dda83 100644
--- a/ReplayBrowser/Services/LeaderboardService.cs
+++ b/ReplayBrowser/Services/LeaderboardService.cs
@@ -22,17 +22,20 @@ public class LeaderboardService : IHostedService, IDisposable
];
private Timer? _timer = null;
- private readonly IMemoryCache _cache;
private readonly Ss14ApiHelper _apiHelper;
private readonly IServiceScopeFactory _scopeFactory;
private readonly AccountService _accountService;
private readonly IConfiguration _configuration;
- private List
RedactedAccounts;
+ private const int MaxConcurrentUpdates = 10;
- public LeaderboardService(IMemoryCache cache, Ss14ApiHelper apiHelper, IServiceScopeFactory factory, AccountService accountService, IConfiguration configuration)
+ public static bool IsUpdating { get; private set; } = false;
+ public static DateTime UpdateStarted { get; private set; } = DateTime.MinValue;
+ public static int UpdateProgress { get; private set; } = 0;
+ public static int UpdateTotal { get; private set; } = 0;
+
+ public LeaderboardService(Ss14ApiHelper apiHelper, IServiceScopeFactory factory, AccountService accountService, IConfiguration configuration)
{
- _cache = cache;
_apiHelper = apiHelper;
_scopeFactory = factory;
_accountService = accountService;
@@ -41,37 +44,80 @@ public LeaderboardService(IMemoryCache cache, Ss14ApiHelper apiHelper, IServiceS
public Task StartAsync(CancellationToken cancellationToken)
{
- _timer = new Timer(DoWork, null, TimeSpan.Zero, TimeSpan.FromHours(6));
+ _timer = new Timer(DoWork, null, TimeSpan.Zero, TimeSpan.FromHours(24));
return Task.CompletedTask;
}
private async void DoWork(object? state)
{
+ if (IsUpdating)
+ {
+ Log.Warning("Leaderboard update already in progress, skipping this update.");
+ return;
+ }
+
+ IsUpdating = true;
+ UpdateStarted = DateTime.UtcNow;
+
var sw = new Stopwatch();
sw.Start();
Log.Information("Updating leaderboards...");
- // Fetch all the redacted players, cache it
- // Yeah this ignores whether someone's an admin and doesn't let them bypass this
- // Better for performance though
+ var servers = _configuration.GetSection("ReplayUrls").Get()!.Select(x => x.FallBackServerName)
+ .Distinct().ToList();
- using (var scope = _scopeFactory.CreateScope()) {
- var context = scope.ServiceProvider.GetRequiredService();
- RedactedAccounts = await context.Accounts
- .Where(a => a.Settings.RedactInformation)
- .Select(a => a.Guid)
- .ToListAsync();
- }
+ var combinations = GetCombinations(servers).ToList();
+ Log.Information("Total combinations: {Combinations}", combinations.Count);
- // Loop through every range option.
- foreach (var rangeOption in Enum.GetValues())
+ UpdateTotal = combinations.Count * Enum.GetValues().Length;
+ var semaphore = new SemaphoreSlim(MaxConcurrentUpdates);
+ var tasks = new List();
+
+ foreach (var serverArr in combinations)
{
- var anonymousAuth = new AuthenticationState(new ClaimsPrincipal(new ClaimsIdentity()));
- await GetLeaderboard(rangeOption, null, [], anonymousAuth, 10, false);
+ var values = Enum.GetValues();
+ foreach (var rangeOption in values)
+ {
+ await semaphore.WaitAsync();
+ tasks.Add(Task.Run(async () =>
+ {
+ try
+ {
+ await GenerateLeaderboard(rangeOption, serverArr.ToArray());
+ UpdateProgress++;
+ }
+ finally
+ {
+ semaphore.Release();
+ }
+ }));
+ }
}
+ await Task.WhenAll(tasks);
+
sw.Stop();
Log.Information("Leaderboards updated in {Time}", sw.Elapsed);
+
+ IsUpdating = false;
+ }
+
+ static IEnumerable> GetCombinations(List list)
+ {
+ var subsetCount = 1 << list.Count;
+
+ for (var i = 1; i < subsetCount; i++)
+ {
+ var combination = new List();
+ for (var j = 0; j < list.Count; j++)
+ {
+ if ((i & (1 << j)) != 0)
+ {
+ combination.Add(list[j]);
+ }
+ }
+ yield return combination;
+ }
}
public Task StopAsync(CancellationToken cancellationToken)
@@ -85,7 +131,8 @@ public void Dispose()
_timer?.Dispose();
}
- public async Task GetLeaderboard(RangeOption rangeOption, string? username, string[]? servers, AuthenticationState authenticationState, int entries = 10, bool logAction = true)
+ public async Task GetLeaderboards(RangeOption rangeOption, string? username, string[]? servers,
+ AuthenticationState authenticationState, int entries = 10, bool logAction = true)
{
if (servers == null || servers.Length == 0)
{
@@ -128,30 +175,69 @@ public async Task GetLeaderboard(RangeOption rangeOption, strin
}
}
- // First, try to get the leaderboard from the cache
- var usernameCacheKey = username
- ?.ToLower()
- .Replace(" ", "-")
- .Replace(".", "-")
- .Replace("_", "-");
+ var redactedAccounts = await context.Accounts
+ .Where(a => a.Settings.RedactInformation)
+ .Select(a => a.Guid)
+ .ToListAsync();
+
+ entries += redactedAccounts.Count; // Add the redacted accounts to the count so that removed listings still show the correct amount of entries
+
+ var leaderboards = await context.Leaderboards
+ .Where(l => l.Servers.SequenceEqual(servers))
+ .Where(l => l.Position <= entries || l.Username == username)
+ .Include(l => l.LeaderboardDefinition)
+ .ToListAsync();
- var serversCacheKey = string.Join("-", servers);
+ var finalReturned = new Dictionary();
- var cacheKey = "leaderboard-" + rangeOption + "-" + usernameCacheKey + "-" + serversCacheKey + "-" + entries;
- if (_cache.TryGetValue(cacheKey, out LeaderboardData? leaderboardData))
+ foreach (var position in leaderboards)
{
- return leaderboardData!;
+ // Remove positions that are redacted
+ if (position.PlayerGuid != null && redactedAccounts.Contains((Guid)position.PlayerGuid))
+ {
+ continue;
+ }
+
+ if (!finalReturned.ContainsKey(position.LeaderboardDefinition.Name))
+ {
+ finalReturned.Add(position.LeaderboardDefinition.Name, new Leaderboard()
+ {
+ Name = position.LeaderboardDefinition.Name,
+ TrackedData = position.LeaderboardDefinition.TrackedData,
+ Data = new Dictionary()
+ });
+ }
+
+ finalReturned[position.LeaderboardDefinition.Name].Data[position.PlayerGuid.ToString() ?? GenerateRandomGuid().ToString()] = new PlayerCount()
+ {
+ Count = position.Count,
+ Player = new PlayerData()
+ {
+ PlayerGuid = position.PlayerGuid,
+ Username = position.Username
+ },
+ Position = position.Position
+ };
}
- var usernameGuid = Guid.Empty;
- if (!string.IsNullOrWhiteSpace(username))
+ var returnList = new List();
+ foreach (var (key, value) in finalReturned)
+ {
+ returnList.Add(await FinalizeLeaderboard(key, value.NameColumn, value, accountCaller?.Guid ?? Guid.Empty, authenticationState, entries));
+ }
+
+ return returnList.ToArray();
+ }
+
+ public async Task GenerateLeaderboard(RangeOption rangeOption, string[]? servers)
+ {
+ if (servers == null || servers.Length == 0)
{
- // Fetch the GUID for the username
- var player = await context.ReplayParticipants
- .FirstOrDefaultAsync(p => p.Username.ToLower() == username.ToLower());
- if (player != null) usernameGuid = player.PlayerGuid;
+ servers = _configuration.GetSection("ReplayUrls").Get()!.Select(x => x.FallBackServerName).ToArray();
}
+ using var scope = _scopeFactory.CreateScope();
+ var context = scope.ServiceProvider.GetRequiredService();
var stopwatch = new Stopwatch();
long stopwatchPrevious = 0;
@@ -400,40 +486,76 @@ public async Task GetLeaderboard(RangeOption rangeOption, strin
Log.Information("SQL queries took {TimeTotal}ms", stopwatch.ElapsedMilliseconds);
stopwatch.Restart();
- // Need to calculate the position of every player in the leaderboard.
+ // Set the positions
foreach (var leaderboard in leaderboards)
{
- var leaderboardResult = await GenerateLeaderboard(leaderboard.Key, leaderboard.Key, leaderboard.Value, usernameGuid, entries);
- leaderboards[leaderboard.Key].Data = leaderboardResult.Data;
+ var players = leaderboard.Value.Data.Values.ToList();
+ players.Sort((a, b) => b.Count.CompareTo(a.Count));
+ for (var i = 0; i < players.Count; i++)
+ {
+ players[i].Position = i + 1;
+ }
}
- stopwatch.Stop();
- Log.Information("Calculating leaderboard took {Time}ms", stopwatch.ElapsedMilliseconds);
-
- // Save leaderboard to cache (its expensive as fuck to calculate)
- var cacheEntryOptions = new MemoryCacheEntryOptions()
- .SetAbsoluteExpiration(TimeSpan.FromHours(5));
- var cacheLeaderboard = new LeaderboardData()
+ var dbLeaderboards = new List();
+ foreach (var (key, leaderboard) in leaderboards)
{
- Leaderboards = leaderboards.Values.ToList(),
- IsCache = true
- };
+ // Ensure a leaderboard definition exists
+ var leaderboardDefinition = await context.LeaderboardDefinitions
+ .FirstOrDefaultAsync(l => l.Name == key);
+
+ if (leaderboardDefinition == null)
+ {
+ leaderboardDefinition = new LeaderboardDefinition()
+ {
+ Name = key,
+ TrackedData = leaderboard.TrackedData,
+ NameColumn = leaderboard.NameColumn,
+ ExtraInfo = leaderboard.ExtraInfo
+ };
+ await context.LeaderboardDefinitions.AddAsync(leaderboardDefinition);
+ } else
+ {
+ leaderboardDefinition.TrackedData = leaderboard.TrackedData;
+ leaderboardDefinition.NameColumn = leaderboard.NameColumn;
+ leaderboardDefinition.ExtraInfo = leaderboard.ExtraInfo;
+ }
- _cache.Set(cacheKey, cacheLeaderboard, cacheEntryOptions);
+ await context.SaveChangesAsync();
+ foreach (var (s, value) in leaderboard.Data)
+ {
+ if (string.IsNullOrEmpty(value.Player?.Username) && value.Player?.PlayerGuid != null)
+ {
+ value.Player.Username = await GetNameFromDbOrApi((Guid)value.Player.PlayerGuid);
+ }
- return new LeaderboardData()
- {
- Leaderboards = leaderboards.Values.ToList(),
- IsCache = false
- };
+ // S is player guid
+ dbLeaderboards.Add(new LeaderboardPosition()
+ {
+ Servers = servers.ToList(),
+ Count = value.Count,
+ PlayerGuid = value.Player?.PlayerGuid,
+ Username = value.Player?.Username ?? string.Empty,
+ LeaderboardDefinitionName = key,
+ });
+ }
+ }
+
+ context.Leaderboards.RemoveRange(context.Leaderboards.Where(l => l.Servers.SequenceEqual(servers)));
+ await context.Leaderboards.AddRangeAsync(dbLeaderboards);
+ await context.SaveChangesAsync();
+
+ stopwatch.Stop();
+ Log.Information("Calculating leaderboard took {Time}ms", stopwatch.ElapsedMilliseconds);
}
- private async Task GenerateLeaderboard(
+ private async Task FinalizeLeaderboard(
string name,
string columnName,
Leaderboard data,
Guid targetPlayer,
+ AuthenticationState authenticationState,
int limit = 10
)
{
@@ -444,7 +566,25 @@ private async Task GenerateLeaderboard(
Data = new Dictionary()
};
- var players = data.Data.Values.Where(p => p.Player?.PlayerGuid is null || p.Player.PlayerGuid == Guid.Empty || !RedactedAccounts.Contains(p.Player?.PlayerGuid ?? Guid.Empty)).ToList();
+ var redactedAccounts = await _scopeFactory.CreateScope().ServiceProvider.GetRequiredService().Accounts
+ .Where(a => a.Settings.RedactInformation)
+ .Select(a => a.Guid)
+ .ToListAsync();
+
+ var account = await _accountService.GetAccount(authenticationState);
+ // Remove any redacted accounts
+ var players = data.Data.Values.ToList();
+ if (account == null || !account.IsAdmin)
+ {
+ players = players.Where(p =>
+ p.Player?.PlayerGuid == null
+ || (
+ !redactedAccounts.Contains((Guid)p.Player.PlayerGuid)
+ && account?.Guid != p.Player.PlayerGuid // Users can see their own data even if redacted
+ ))
+ .ToList();
+ }
+
players.Sort((a, b) => b.Count.CompareTo(a.Count));
for (var i = 0; i < players.Count; i++)
@@ -479,22 +619,7 @@ private async Task GenerateLeaderboard(
if (player.Value.Player?.PlayerGuid == null)
continue;
- // get the latest name from the db
- var playerData = await context.ReplayParticipants
- .Where(p => p.PlayerGuid == player.Value.Player.PlayerGuid)
- .OrderByDescending(p => p.Id)
- .FirstOrDefaultAsync();
-
- if (playerData == null)
- {
- // ??? try to get using api
- var playerDataApi = await _apiHelper.FetchPlayerDataFromGuid((Guid)player.Value.Player.PlayerGuid);
- player.Value.Player.Username = playerDataApi.Username;
- }
- else
- {
- player.Value.Player.Username = playerData.Username;
- }
+ player.Value.Player.Username = await GetNameFromDbOrApi((Guid)player.Value.Player.PlayerGuid);
}
stopwatch.Stop();
Log.Verbose("Fetching player data took {Time}ms", stopwatch.ElapsedMilliseconds);
@@ -502,6 +627,25 @@ private async Task GenerateLeaderboard(
return returnValue;
}
+ private async Task GetNameFromDbOrApi(Guid guid)
+ {
+ using var scope = _scopeFactory.CreateScope();
+ var context = scope.ServiceProvider.GetRequiredService();
+
+ // get the latest name from the db
+ var playerData = await context.ReplayParticipants
+ .Where(p => p.PlayerGuid == guid)
+ .OrderByDescending(p => p.Id)
+ .FirstOrDefaultAsync();
+
+ if (playerData != null) return playerData.Username;
+
+ // ??? try to get using api
+ var playerDataApi = await _apiHelper.FetchPlayerDataFromGuid(guid);
+ return playerDataApi.Username;
+
+ }
+
private Guid GenerateRandomGuid()
{
var guidBytes = new byte[16];