Skip to content

Commit

Permalink
Comment fixes
Browse files Browse the repository at this point in the history
1. Reverted log stream threshold
2. Removed batching from missing archives download
3. Adjusted threshold for progress reporting in resumable downloader to 10mb
4. Added additional logging to ResumableDownloader
5. Typo
  • Loading branch information
AlexDickerson committed Dec 29, 2024
1 parent bdf3588 commit d281916
Show file tree
Hide file tree
Showing 4 changed files with 18 additions and 27 deletions.
2 changes: 0 additions & 2 deletions Wabbajack.App.Wpf/Models/LogStream.cs
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,6 @@ protected override void Dispose(bool disposing)

protected override void Write(LogEventInfo logEvent)
{
if(logEvent.Level < LogLevel.Error)
return;
_messages.OnNext(new LogMessage(logEvent));
}
public interface ILogMessage
Expand Down
2 changes: 1 addition & 1 deletion Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<OutputType>WinExe</OutputType>
<TargetFramework>net9.0-windows</TargetFramework>
<UseWPF>true</UseWPF>
<Platforms>x64</Platforms>
Expand Down
31 changes: 10 additions & 21 deletions Wabbajack.Installer/AInstaller.cs
Original file line number Diff line number Diff line change
Expand Up @@ -371,27 +371,16 @@ public async Task DownloadMissingArchives(List<Archive> missing, CancellationTok
}
}

var missingBatches = missing.Batch(100).ToList();

List<Task> downloadBatches = [];

foreach (var batch in missingBatches)
{
downloadBatches.Add(batch
.Shuffle()
.Where(a => a.State is not Manual)
.PDoAll(async archive =>
{
_logger.LogInformation("Downloading {Archive}", archive.Name);
var outputPath = _configuration.Downloads.Combine(archive.Name);
var hash = await DownloadArchive(archive, download, token, outputPath);
UpdateProgress(1);
}));

await Task.Delay(TimeSpan.FromSeconds(10)); //If we spin these up too fast we can hit nexus ddos protection regardless of remaining api limit so we slow it down a bit
}

await Task.WhenAll(downloadBatches);
await missing
.Shuffle()
.Where(a => a.State is not Manual)
.PDoAll(async archive =>
{
_logger.LogInformation("Downloading {Archive}", archive.Name);
var outputPath = _configuration.Downloads.Combine(archive.Name);
var hash = await DownloadArchive(archive, download, token, outputPath);
UpdateProgress(1);
});
}

private async Task SendDownloadMetrics(List<Archive> missing)
Expand Down
10 changes: 7 additions & 3 deletions Wabbajack.Networking.Http/ResumableDownloader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ private async Task<Hash> DownloadAndHash(HttpRequestMessage msg, AbsolutePath fi

if (retry == 0)
{
_logger.LogError(ex, "Failed to download '{name}'", filePath.FileName.ToString());

throw;
}

Expand All @@ -65,6 +67,8 @@ private async Task<Hash> DownloadAndHash(HttpRequestMessage msg, AbsolutePath fi

if (retry == 0)
{
_logger.LogError(ex, "Failed to download '{name}'", filePath.FileName.ToString());

throw;
}

Expand Down Expand Up @@ -102,8 +106,8 @@ private async Task<AbsolutePath> DownloadStreamDirectlyToFile(HttpRequestMessage

var responseStream = await response.Content.ReadAsStreamAsync(token);

long reportProgressThreshold = 100 * 1024 * 1024; // Report progress every 100MB
bool shoudlReportProgress = job.Size > reportProgressThreshold; // Reporting progress on small files just generates strain on the UI
long reportProgressThreshold = 10 * 1024 * 1024; // Report progress every 100MB
bool shouldReportProgress = job.Size > reportProgressThreshold; // Reporting progress on small files just generates strain on the UI

int reportEveryXBytesProcessed = (int) job.Size / 100; // Report progress every 1% of the file
long bytesProcessed = startingPosition;
Expand All @@ -115,7 +119,7 @@ private async Task<AbsolutePath> DownloadStreamDirectlyToFile(HttpRequestMessage
await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead), token);
bytesProcessed += bytesRead;

if (shoudlReportProgress && bytesProcessed >= reportEveryXBytesProcessed)
if (shouldReportProgress && bytesProcessed >= reportEveryXBytesProcessed)
{
job.ReportNoWait((int)bytesProcessed);
bytesProcessed = 0;
Expand Down

0 comments on commit d281916

Please sign in to comment.