Merge remote-tracking branch 'origin/main' into pre-release

Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj

@@ -15,6 +15,7 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
    <!-- <IncludeSymbolsInSingleFile>true</IncludeSymbolsInSingleFile> -->
     <AssemblyName>Wabbajack</AssemblyName>
+    <JsonSerializerIsReflectionEnabledByDefault>true</JsonSerializerIsReflectionEnabledByDefault>
   </PropertyGroup>
 
 

Wabbajack.CLI/VerbRegistration.cs

@@ -1,4 +1,4 @@
-
+
 using Microsoft.Extensions.DependencyInjection;
 namespace Wabbajack.CLI;
 using Wabbajack.CLI.Verbs;
@@ -37,6 +37,8 @@ public static void AddCLIVerbs(this IServiceCollection services) {
 services.AddSingleton<ListGames>();
 CommandLineBuilder.RegisterCommand<ListModlists>(ListModlists.Definition, c => ((ListModlists)c).Run);
 services.AddSingleton<ListModlists>();
+CommandLineBuilder.RegisterCommand<MegaLogin>(MegaLogin.Definition, c => ((MegaLogin)c).Run);
+services.AddSingleton<MegaLogin>();
 CommandLineBuilder.RegisterCommand<MirrorFile>(MirrorFile.Definition, c => ((MirrorFile)c).Run);
 services.AddSingleton<MirrorFile>();
 CommandLineBuilder.RegisterCommand<ModlistReport>(ModlistReport.Definition, c => ((ModlistReport)c).Run);

Wabbajack.CLI/Verbs/MegaLogin.cs

@@ -0,0 +1,40 @@
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Wabbajack.CLI.Builder;
+using Wabbajack.Downloaders.ModDB;
+using Wabbajack.Networking.Http.Interfaces;
+
+namespace Wabbajack.CLI.Verbs;
+
+public class MegaLogin
+{
+    private readonly ILogger<MegaLogin> _logger;
+
+    public MegaLogin(ILogger<MegaLogin> logger, ITokenProvider<MegaToken> tokenProvider)
+    {
+        _logger = logger;
+        _tokenProvider = tokenProvider;
+    }
+
+    public static VerbDefinition Definition = new VerbDefinition("mega-login",
+        "Hashes a file with Wabbajack's hashing routines", new[]
+        {
+            new OptionDefinition(typeof(string), "e", "email", "Email for the user account"),
+            new OptionDefinition(typeof(string), "p", "password", "Password for the user account"),
+        });
+
+    private readonly ITokenProvider<MegaToken> _tokenProvider;
+
+    public async Task<int> Run(string email, string password)
+    {
+        _logger.LogInformation("Logging into Mega");
+        await _tokenProvider.SetToken(new MegaToken
+        {
+            Email = email,
+            Password = password
+        });
+        return 0;
+    }
+}

Wabbajack.CLI/Verbs/ValidateLists.cs

@@ -1,11 +1,13 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.CommandLine;
 using System.CommandLine.NamingConventionBinder;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
@@ -55,6 +57,8 @@ public class ValidateLists
     private readonly IResource<HttpClient> _httpLimiter;
     private readonly AsyncLock _imageProcessLock;
 
+    private readonly ConcurrentBag<(Uri, Hash)> _proxyableFiles = new();
+
 
     public ValidateLists(ILogger<ValidateLists> logger, Networking.WabbajackClientApi.Client wjClient,
         Client gitHubClient, TemporaryFileManager temporaryFileManager,
@@ -92,10 +96,7 @@ public async Task<int> Run(AbsolutePath reports, AbsolutePath otherArchives)
 
         reports.CreateDirectory();
         var token = CancellationToken.None;
-
-        _logger.LogInformation("Scanning for existing patches/mirrors");
-        var mirroredFiles = (await _wjClient.GetAllMirroredFileDefinitions(token)).Select(m => m.Hash).ToHashSet();
-        _logger.LogInformation("Found {Count} mirrored files", mirroredFiles.Count);
+
         var patchFiles = await _wjClient.GetAllPatches(token);
         _logger.LogInformation("Found {Count} patches", patchFiles.Length);
 
@@ -168,30 +169,7 @@ public async Task<int> Run(AbsolutePath reports, AbsolutePath otherArchives)
                         Original = archive
                     };
                 }
-
-
-                if (result.Status == ArchiveStatus.InValid)
-                {
-                    if (mirroredFiles.Contains(archive.Hash))
-                    {
-                        return new ValidatedArchive
-                        {
-                            Status = ArchiveStatus.Mirrored,
-                            Original = archive,
-                            PatchedFrom = new Archive
-                            {
-                                State = new WabbajackCDN
-                                {
-                                    Url = _wjClient.GetMirrorUrl(archive.Hash)!
-                                },
-                                Size = archive.Size,
-                                Name = archive.Name,
-                                Hash = archive.Hash
-                            }
-                        };
-                    }
-                }
-
+
                 if (result.Status == ArchiveStatus.InValid)
                 {
                     _logger.LogInformation("Looking for patch for {Hash}", archive.Hash);
@@ -209,14 +187,23 @@ public async Task<int> Run(AbsolutePath reports, AbsolutePath otherArchives)
                         }
                     }
                 }
-
-                return new ValidatedArchive()
+                
+                return new ValidatedArchive
                 {
                     Status = ArchiveStatus.InValid,
                     Original = archive
                 };
             }).ToArray();
 
+            foreach (var archive in archives)
+            {
+                var downloader = _dispatcher.Downloader(archive.Original);
+                if (downloader is IProxyable proxyable)
+                {
+                    _proxyableFiles.Add((proxyable.UnParse(archive.Original.State), archive.Original.Hash));
+                }
+            }
+
             validatedList.Archives = archives;
             validatedList.Status = archives.Any(a => a.Status == ArchiveStatus.InValid)
                 ? ListStatus.Failed
@@ -255,12 +242,7 @@ public async Task<int> Run(AbsolutePath reports, AbsolutePath otherArchives)
         }
 
         await ExportReports(reports, validatedLists, token);
-
-        var usedMirroredFiles = validatedLists.SelectMany(a => a.Archives)
-            .Where(m => m.Status == ArchiveStatus.Mirrored)
-            .Select(m => m.Original.Hash)
-            .ToHashSet();
-        await DeleteOldMirrors(mirroredFiles, usedMirroredFiles);
+
 
         return 0;
     }
@@ -580,7 +562,14 @@ await _discord.SendAsync(Channel.Ham,
             .Open(FileMode.Create, FileAccess.Write, FileShare.None);
         await _dtos.Serialize(upgradedMetas, upgradedMetasFile, true);
 
-
+        await using var proxyFile = reports.Combine("proxyable.txt")
+            .Open(FileMode.Create, FileAccess.Write, FileShare.None);
+        await using var tw = new StreamWriter(proxyFile);
+        foreach (var file in _proxyableFiles)
+        {
+            var str = $"{file.Item1}#name={file.Item2.ToHex()}";
+            await tw.WriteLineAsync(str);
+        }
     }
 
     private async Task SendDefinitionToLoadOrderLibrary(ValidatedModList validatedModList, CancellationToken token)

Wabbajack.CLI/Wabbajack.CLI.csproj

@@ -14,6 +14,7 @@
         <NoWarn>CS8601</NoWarn>
         <NoWarn>CS8618</NoWarn>
         <TargetFramework>net8.0</TargetFramework>
+        <JsonSerializerIsReflectionEnabledByDefault>true</JsonSerializerIsReflectionEnabledByDefault>
     </PropertyGroup>
 
     <ItemGroup>

Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs

@@ -40,8 +40,11 @@ public DownloadDispatcher(ILogger<DownloadDispatcher> logger, IEnumerable<IDownl
         _limiter = limiter;
         _useProxyCache = useProxyCache;
         _verificationCache = verificationCache;
+
     }
 
+    public bool UseProxy { get; set; } = false;
+
     public async Task<Hash> Download(Archive a, AbsolutePath dest, CancellationToken token, bool? proxy = null)
     {
         if (token.IsCancellationRequested)
@@ -58,6 +61,7 @@ public async Task<Hash> Download(Archive a, AbsolutePath dest, CancellationToken
 
     public async Task<Archive> MaybeProxy(Archive a, CancellationToken token)
     {
+        if (!UseProxy) return a;
         var downloader = Downloader(a);
         if (downloader is not IProxyable p) return a;
 
@@ -134,7 +138,9 @@ public async Task<bool> Verify(Archive a, CancellationToken token)
                 return true;
             }
 
-            a = await MaybeProxy(a, token);
+            if (UseProxy) 
+                a = await MaybeProxy(a, token);
+
             var downloader = Downloader(a);
             using var job = await _limiter.Begin($"Verifying {a.State.PrimaryKeyString}", -1, token);
             var result = await downloader.Verify(a, job, token);

Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs

@@ -80,7 +80,7 @@ public async Task<T> DownloadStream<T>(Archive archive, Func<Stream, Task<T>> fn
     {
         var state = archive.State as DTOs.DownloadStates.GoogleDrive;
         var msg = await ToMessage(state, true, token);
-        using var result = await _client.SendAsync(msg, token);
+        using var result = await _client.SendAsync(msg, HttpCompletionOption.ResponseHeadersRead, token);
         HttpException.ThrowOnFailure(result);
         await using var stream = await result.Content.ReadAsStreamAsync(token);
         return await fn(stream);

Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs

@@ -79,7 +79,7 @@ public async Task<T> DownloadStream<T>(Archive archive, Func<Stream, Task<T>> fn
         var state = archive.State as DTOs.DownloadStates.MediaFire;
         var url = await Resolve(state!);
         var msg = new HttpRequestMessage(HttpMethod.Get, url!);
-        using var result = await _httpClient.SendAsync(msg, token);
+        using var result = await _httpClient.SendAsync(msg, HttpCompletionOption.ResponseHeadersRead, token);
         await using var stream = await result.Content.ReadAsStreamAsync(token);
         return await fn(stream);
     }

Wabbajack.Downloaders.Mega/MegaDownloader.cs

@@ -12,6 +12,7 @@
 using Wabbajack.DTOs.DownloadStates;
 using Wabbajack.DTOs.Validation;
 using Wabbajack.Hashing.xxHash64;
+using Wabbajack.Networking.Http.Interfaces;
 using Wabbajack.Paths;
 using Wabbajack.Paths.IO;
 using Wabbajack.RateLimiter;
@@ -24,17 +25,18 @@ public class MegaDownloader : ADownloader<Mega>, IUrlDownloader, IProxyable
     private const string MegaFilePrefix = "https://mega.nz/file/";
     private readonly MegaApiClient _apiClient;
     private readonly ILogger<MegaDownloader> _logger;
+    private readonly ITokenProvider<MegaToken> _tokenProvider;
 
-    public MegaDownloader(ILogger<MegaDownloader> logger, MegaApiClient apiClient)
+    public MegaDownloader(ILogger<MegaDownloader> logger, MegaApiClient apiClient, ITokenProvider<MegaToken> tokenProvider)
     {
         _logger = logger;
         _apiClient = apiClient;
+        _tokenProvider = tokenProvider;
     }
 
     public override async Task<bool> Prepare()
     {
-        if (!_apiClient.IsLoggedIn)
-            await _apiClient.LoginAsync();
+        await LoginIfNotLoggedIn();
         return true;
     }
 
@@ -64,19 +66,35 @@ public Uri UnParse(IDownloadState state)
     public async Task<T> DownloadStream<T>(Archive archive, Func<Stream, Task<T>> fn, CancellationToken token)
     {
         var state = archive.State as Mega;
-        if (!_apiClient.IsLoggedIn)
-            await _apiClient.LoginAsync();
-
+        await LoginIfNotLoggedIn();
+
         await using var ins = await _apiClient.DownloadAsync(state!.Url, cancellationToken: token);
         return await fn(ins);
     }
 
-    public override async Task<Hash> Download(Archive archive, Mega state, AbsolutePath destination, IJob job,
-        CancellationToken token)
+    private async Task LoginIfNotLoggedIn()
     {
         if (!_apiClient.IsLoggedIn)
-            await _apiClient.LoginAsync();
+        {
+            if (_tokenProvider.HaveToken())
+            {
+                var authInfo = await _tokenProvider.Get();
+                _logger.LogInformation("Logging into Mega with {Email}", authInfo!.Email);
+                await _apiClient.LoginAsync(authInfo!.Email, authInfo.Password);
+            }
+            else
+            {
+                _logger.LogInformation("Logging into Mega without credentials");
+                await _apiClient.LoginAsync();
+            }
+        }
+    }
 
+    public override async Task<Hash> Download(Archive archive, Mega state, AbsolutePath destination, IJob job,
+        CancellationToken token)
+    {
+        await LoginIfNotLoggedIn();
+
         await using var ous = destination.Open(FileMode.Create, FileAccess.Write, FileShare.None);
         await using var ins = await _apiClient.DownloadAsync(state.Url, cancellationToken: token);
         return await ins.HashingCopy(ous, token, job);
@@ -93,9 +111,8 @@ public override async Task<Hash> Download(Archive archive, Mega state, AbsoluteP
 
     public override async Task<bool> Verify(Archive archive, Mega archiveState, IJob job, CancellationToken token)
     {
-        if (!_apiClient.IsLoggedIn)
-            await _apiClient.LoginAsync();
-
+        await LoginIfNotLoggedIn();
+
         for (var times = 0; times < 5; times++)
         {
             try

Wabbajack.Downloaders.Mega/MegaToken.cs

@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace Wabbajack.Downloaders.ModDB;
+
+public class MegaToken
+{
+    [JsonPropertyName("email")]
+    public string Email { get; set; }
+
+    [JsonPropertyName("password")]
+    public string Password { get; set; }
+}