Skip to content

Commit

Permalink
Remove loop
Browse files Browse the repository at this point in the history
Since the queue consume gets called after the queue gets filled, a loop checking the queue is not needed.
Simyon264 authored Mar 8, 2024
1 parent a853f18 commit f22fe8a
Showing 1 changed file with 68 additions and 74 deletions.
142 changes: 68 additions & 74 deletions Server/ReplayParser.cs
Original file line number Diff line number Diff line change
@@ -51,92 +51,86 @@ public static Task AddParsedReplayToDb(string replay)

public static async Task ConsumeQueue(CancellationToken token)
{
while (!token.IsCancellationRequested)
// Consume the queue.
while (Queue.Count > 0)
{
// Consume the queue.
while (Queue.Count > 0)
{
var timeoutToken = new CancellationTokenSource(10000);
var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutToken.Token);
var startTime = DateTime.Now;
var timeoutToken = new CancellationTokenSource(10000);
var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutToken.Token);
var startTime = DateTime.Now;

// Since replays are like 200mb long, we want to parrallelize this.
var tasks = new List<Task>();
for (var i = 0; i < 10; i++)
// Since replays are like 200mb long, we want to parrallelize this.
var tasks = new List<Task>();
for (var i = 0; i < 10; i++)
{
if (Queue.Count == 0)
{
if (Queue.Count == 0)
{
break;
}
var replay = Queue[0];
Queue.RemoveAt(0);
// If it's already in the database, skip it.
if (await IsReplayParsed(replay))
{
continue;
}

tasks.Add(Task.Run(async () =>
break;
}
var replay = Queue[0];
Queue.RemoveAt(0);
// If it's already in the database, skip it.
if (await IsReplayParsed(replay))
{
continue;
}
tasks.Add(Task.Run(async () =>
{
try
{
var client = new HttpClient();
Log.Information("Downloading " + replay);
var fileStream = await client.GetStreamAsync(replay, token);
Replay? parsedReplay = null;
try
{
var client = new HttpClient();
Log.Information("Downloading " + replay);
var fileStream = await client.GetStreamAsync(replay, token);
Replay? parsedReplay = null;
try
{
parsedReplay = ParseReplay(fileStream);
}
catch (Exception e)
{
// Ignore
await AddParsedReplayToDb(replay);
return;
}
parsedReplay.Link = replay;
// See if the link matches the date regex, if it does set the date
var replayFileName = Path.GetFileName(replay);
var match = RegexList.ReplayRegex.Match(replayFileName);
if (match.Success)
{
var date = DateTime.ParseExact(match.Groups[1].Value, "yyyy_MM_dd-HH_mm", CultureInfo.InvariantCulture);
// Need to mark it as UTC, since the server is in UTC.
parsedReplay.Date = date.ToUniversalTime();
}

// One more check to see if it's already in the database.
if (await IsReplayParsed(replay))
{
return;
}

await AddReplayToDb(parsedReplay);
await AddParsedReplayToDb(replay);
Log.Information("Parsed " + replay);
parsedReplay = ParseReplay(fileStream);
}
catch (Exception e)

Check warning on line 88 in Server/ReplayParser.cs

GitHub Actions / deploy

The variable 'e' is declared but never used
{
Log.Error(e, "Error while parsing " + replay);
// Ignore
await AddParsedReplayToDb(replay);
return;
}
}, tokenSource.Token));
}
parsedReplay.Link = replay;
// See if the link matches the date regex, if it does set the date
var replayFileName = Path.GetFileName(replay);
var match = RegexList.ReplayRegex.Match(replayFileName);
if (match.Success)
{
var date = DateTime.ParseExact(match.Groups[1].Value, "yyyy_MM_dd-HH_mm", CultureInfo.InvariantCulture);
// Need to mark it as UTC, since the server is in UTC.
parsedReplay.Date = date.ToUniversalTime();
}

// One more check to see if it's already in the database.
if (await IsReplayParsed(replay))
{
return;
}

await AddReplayToDb(parsedReplay);
await AddParsedReplayToDb(replay);
Log.Information("Parsed " + replay);
}
catch (Exception e)
{
Log.Error(e, "Error while parsing " + replay);
}
}, tokenSource.Token));
}

// If the download takes too long, cancel it.
// 10 minutes should be enough
await Task.WhenAny(Task.WhenAll(tasks), Task.Delay(600000, token));
await timeoutToken.CancelAsync();
// Cancel the timeout token, so the background tasks cancel as well.
// If the download takes too long, cancel it.
// 10 minutes should be enough
await Task.WhenAny(Task.WhenAll(tasks), Task.Delay(600000, token));
await timeoutToken.CancelAsync();
// Cancel the timeout token, so the background tasks cancel as well.

// If we timed out, log a warning.
if (DateTime.Now - startTime > TimeSpan.FromMinutes(10))
{
Log.Warning("Parsing took too long for " + string.Join(", ", tasks.Select(x => x.Id)));
}
// If we timed out, log a warning.
if (DateTime.Now - startTime > TimeSpan.FromMinutes(10))
{
Log.Warning("Parsing took too long for " + string.Join(", ", tasks.Select(x => x.Id)));
}

await Task.Delay(5000, token);
}
}
}

/// <summary>
@@ -328,4 +322,4 @@ public static List<Replay> SearchReplays(SearchMode mode, string query, ReplayDb
throw new NotImplementedException();
}
}
}
}

0 comments on commit f22fe8a

Please sign in to comment.