diff --git a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj index 31ef5b3af..7c708116e 100644 --- a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj +++ b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj @@ -15,6 +15,7 @@ true Wabbajack + true diff --git a/Wabbajack.CLI/VerbRegistration.cs b/Wabbajack.CLI/VerbRegistration.cs index b000d2019..3854dce44 100644 --- a/Wabbajack.CLI/VerbRegistration.cs +++ b/Wabbajack.CLI/VerbRegistration.cs @@ -1,4 +1,4 @@ - + using Microsoft.Extensions.DependencyInjection; namespace Wabbajack.CLI; using Wabbajack.CLI.Verbs; @@ -37,6 +37,8 @@ public static void AddCLIVerbs(this IServiceCollection services) { services.AddSingleton(); CommandLineBuilder.RegisterCommand(ListModlists.Definition, c => ((ListModlists)c).Run); services.AddSingleton(); +CommandLineBuilder.RegisterCommand(MegaLogin.Definition, c => ((MegaLogin)c).Run); +services.AddSingleton(); CommandLineBuilder.RegisterCommand(MirrorFile.Definition, c => ((MirrorFile)c).Run); services.AddSingleton(); CommandLineBuilder.RegisterCommand(ModlistReport.Definition, c => ((ModlistReport)c).Run); diff --git a/Wabbajack.CLI/Verbs/MegaLogin.cs b/Wabbajack.CLI/Verbs/MegaLogin.cs new file mode 100644 index 000000000..bb1164f8c --- /dev/null +++ b/Wabbajack.CLI/Verbs/MegaLogin.cs @@ -0,0 +1,40 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.CLI.Builder; +using Wabbajack.Downloaders.ModDB; +using Wabbajack.Networking.Http.Interfaces; + +namespace Wabbajack.CLI.Verbs; + +public class MegaLogin +{ + private readonly ILogger _logger; + + public MegaLogin(ILogger logger, ITokenProvider tokenProvider) + { + _logger = logger; + _tokenProvider = tokenProvider; + } + + public static VerbDefinition Definition = new VerbDefinition("mega-login", + "Hashes a file with Wabbajack's hashing routines", new[] + { + new OptionDefinition(typeof(string), "e", "email", "Email for the user account"), + new OptionDefinition(typeof(string), "p", "password", "Password for the user account"), + }); + + private readonly ITokenProvider _tokenProvider; + + public async Task Run(string email, string password) + { + _logger.LogInformation("Logging into Mega"); + await _tokenProvider.SetToken(new MegaToken + { + Email = email, + Password = password + }); + return 0; + } +} \ No newline at end of file diff --git a/Wabbajack.CLI/Verbs/ValidateLists.cs b/Wabbajack.CLI/Verbs/ValidateLists.cs index 45bf40b21..72e6285df 100644 --- a/Wabbajack.CLI/Verbs/ValidateLists.cs +++ b/Wabbajack.CLI/Verbs/ValidateLists.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.CommandLine; using System.CommandLine.NamingConventionBinder; @@ -6,6 +7,7 @@ using System.IO; using System.Linq; using System.Net.Http; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -55,6 +57,8 @@ public class ValidateLists private readonly IResource _httpLimiter; private readonly AsyncLock _imageProcessLock; + private readonly ConcurrentBag<(Uri, Hash)> _proxyableFiles = new(); + public ValidateLists(ILogger logger, Networking.WabbajackClientApi.Client wjClient, Client gitHubClient, TemporaryFileManager temporaryFileManager, @@ -92,10 +96,7 @@ public async Task Run(AbsolutePath reports, AbsolutePath otherArchives) reports.CreateDirectory(); var token = CancellationToken.None; - - _logger.LogInformation("Scanning for existing patches/mirrors"); - var mirroredFiles = (await _wjClient.GetAllMirroredFileDefinitions(token)).Select(m => m.Hash).ToHashSet(); - _logger.LogInformation("Found {Count} mirrored files", mirroredFiles.Count); + var patchFiles = await _wjClient.GetAllPatches(token); _logger.LogInformation("Found {Count} patches", patchFiles.Length); @@ -168,30 +169,7 @@ public async Task Run(AbsolutePath reports, AbsolutePath otherArchives) Original = archive }; } - - - if (result.Status == ArchiveStatus.InValid) - { - if (mirroredFiles.Contains(archive.Hash)) - { - return new ValidatedArchive - { - Status = ArchiveStatus.Mirrored, - Original = archive, - PatchedFrom = new Archive - { - State = new WabbajackCDN - { - Url = _wjClient.GetMirrorUrl(archive.Hash)! - }, - Size = archive.Size, - Name = archive.Name, - Hash = archive.Hash - } - }; - } - } - + if (result.Status == ArchiveStatus.InValid) { _logger.LogInformation("Looking for patch for {Hash}", archive.Hash); @@ -209,14 +187,23 @@ public async Task Run(AbsolutePath reports, AbsolutePath otherArchives) } } } - - return new ValidatedArchive() + + return new ValidatedArchive { Status = ArchiveStatus.InValid, Original = archive }; }).ToArray(); + foreach (var archive in archives) + { + var downloader = _dispatcher.Downloader(archive.Original); + if (downloader is IProxyable proxyable) + { + _proxyableFiles.Add((proxyable.UnParse(archive.Original.State), archive.Original.Hash)); + } + } + validatedList.Archives = archives; validatedList.Status = archives.Any(a => a.Status == ArchiveStatus.InValid) ? ListStatus.Failed @@ -255,12 +242,7 @@ public async Task Run(AbsolutePath reports, AbsolutePath otherArchives) } await ExportReports(reports, validatedLists, token); - - var usedMirroredFiles = validatedLists.SelectMany(a => a.Archives) - .Where(m => m.Status == ArchiveStatus.Mirrored) - .Select(m => m.Original.Hash) - .ToHashSet(); - await DeleteOldMirrors(mirroredFiles, usedMirroredFiles); + return 0; } @@ -580,7 +562,14 @@ await _discord.SendAsync(Channel.Ham, .Open(FileMode.Create, FileAccess.Write, FileShare.None); await _dtos.Serialize(upgradedMetas, upgradedMetasFile, true); - + await using var proxyFile = reports.Combine("proxyable.txt") + .Open(FileMode.Create, FileAccess.Write, FileShare.None); + await using var tw = new StreamWriter(proxyFile); + foreach (var file in _proxyableFiles) + { + var str = $"{file.Item1}#name={file.Item2.ToHex()}"; + await tw.WriteLineAsync(str); + } } private async Task SendDefinitionToLoadOrderLibrary(ValidatedModList validatedModList, CancellationToken token) diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index 27c0da7c9..21553404b 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -14,6 +14,7 @@ CS8601 CS8618 net8.0 + true diff --git a/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs b/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs index cc6f75f3c..c01eb31e6 100644 --- a/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs +++ b/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs @@ -40,8 +40,11 @@ public DownloadDispatcher(ILogger logger, IEnumerable Download(Archive a, AbsolutePath dest, CancellationToken token, bool? proxy = null) { if (token.IsCancellationRequested) @@ -58,6 +61,7 @@ public async Task Download(Archive a, AbsolutePath dest, CancellationToken public async Task MaybeProxy(Archive a, CancellationToken token) { + if (!UseProxy) return a; var downloader = Downloader(a); if (downloader is not IProxyable p) return a; @@ -134,7 +138,9 @@ public async Task Verify(Archive a, CancellationToken token) return true; } - a = await MaybeProxy(a, token); + if (UseProxy) + a = await MaybeProxy(a, token); + var downloader = Downloader(a); using var job = await _limiter.Begin($"Verifying {a.State.PrimaryKeyString}", -1, token); var result = await downloader.Verify(a, job, token); diff --git a/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs b/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs index c49066948..d19bb9090 100644 --- a/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs +++ b/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs @@ -80,7 +80,7 @@ public async Task DownloadStream(Archive archive, Func> fn { var state = archive.State as DTOs.DownloadStates.GoogleDrive; var msg = await ToMessage(state, true, token); - using var result = await _client.SendAsync(msg, token); + using var result = await _client.SendAsync(msg, HttpCompletionOption.ResponseHeadersRead, token); HttpException.ThrowOnFailure(result); await using var stream = await result.Content.ReadAsStreamAsync(token); return await fn(stream); diff --git a/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs b/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs index c65d12eef..8a1180804 100644 --- a/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs +++ b/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs @@ -79,7 +79,7 @@ public async Task DownloadStream(Archive archive, Func> fn var state = archive.State as DTOs.DownloadStates.MediaFire; var url = await Resolve(state!); var msg = new HttpRequestMessage(HttpMethod.Get, url!); - using var result = await _httpClient.SendAsync(msg, token); + using var result = await _httpClient.SendAsync(msg, HttpCompletionOption.ResponseHeadersRead, token); await using var stream = await result.Content.ReadAsStreamAsync(token); return await fn(stream); } diff --git a/Wabbajack.Downloaders.Mega/MegaDownloader.cs b/Wabbajack.Downloaders.Mega/MegaDownloader.cs index 9cdce1fea..d0d34d40a 100644 --- a/Wabbajack.Downloaders.Mega/MegaDownloader.cs +++ b/Wabbajack.Downloaders.Mega/MegaDownloader.cs @@ -12,6 +12,7 @@ using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.Validation; using Wabbajack.Hashing.xxHash64; +using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Paths; using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; @@ -24,17 +25,18 @@ public class MegaDownloader : ADownloader, IUrlDownloader, IProxyable private const string MegaFilePrefix = "https://mega.nz/file/"; private readonly MegaApiClient _apiClient; private readonly ILogger _logger; + private readonly ITokenProvider _tokenProvider; - public MegaDownloader(ILogger logger, MegaApiClient apiClient) + public MegaDownloader(ILogger logger, MegaApiClient apiClient, ITokenProvider tokenProvider) { _logger = logger; _apiClient = apiClient; + _tokenProvider = tokenProvider; } public override async Task Prepare() { - if (!_apiClient.IsLoggedIn) - await _apiClient.LoginAsync(); + await LoginIfNotLoggedIn(); return true; } @@ -64,19 +66,35 @@ public Uri UnParse(IDownloadState state) public async Task DownloadStream(Archive archive, Func> fn, CancellationToken token) { var state = archive.State as Mega; - if (!_apiClient.IsLoggedIn) - await _apiClient.LoginAsync(); - + await LoginIfNotLoggedIn(); + await using var ins = await _apiClient.DownloadAsync(state!.Url, cancellationToken: token); return await fn(ins); } - public override async Task Download(Archive archive, Mega state, AbsolutePath destination, IJob job, - CancellationToken token) + private async Task LoginIfNotLoggedIn() { if (!_apiClient.IsLoggedIn) - await _apiClient.LoginAsync(); + { + if (_tokenProvider.HaveToken()) + { + var authInfo = await _tokenProvider.Get(); + _logger.LogInformation("Logging into Mega with {Email}", authInfo!.Email); + await _apiClient.LoginAsync(authInfo!.Email, authInfo.Password); + } + else + { + _logger.LogInformation("Logging into Mega without credentials"); + await _apiClient.LoginAsync(); + } + } + } + public override async Task Download(Archive archive, Mega state, AbsolutePath destination, IJob job, + CancellationToken token) + { + await LoginIfNotLoggedIn(); + await using var ous = destination.Open(FileMode.Create, FileAccess.Write, FileShare.None); await using var ins = await _apiClient.DownloadAsync(state.Url, cancellationToken: token); return await ins.HashingCopy(ous, token, job); @@ -93,9 +111,8 @@ public override async Task Download(Archive archive, Mega state, AbsoluteP public override async Task Verify(Archive archive, Mega archiveState, IJob job, CancellationToken token) { - if (!_apiClient.IsLoggedIn) - await _apiClient.LoginAsync(); - + await LoginIfNotLoggedIn(); + for (var times = 0; times < 5; times++) { try diff --git a/Wabbajack.Downloaders.Mega/MegaToken.cs b/Wabbajack.Downloaders.Mega/MegaToken.cs new file mode 100644 index 000000000..1f14fe422 --- /dev/null +++ b/Wabbajack.Downloaders.Mega/MegaToken.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace Wabbajack.Downloaders.ModDB; + +public class MegaToken +{ + [JsonPropertyName("email")] + public string Email { get; set; } + + [JsonPropertyName("password")] + public string Password { get; set; } +} \ No newline at end of file diff --git a/Wabbajack.Hashing.xxHash64/StreamExtensions.cs b/Wabbajack.Hashing.xxHash64/StreamExtensions.cs index 51d10f92d..9bbaa0b0e 100644 --- a/Wabbajack.Hashing.xxHash64/StreamExtensions.cs +++ b/Wabbajack.Hashing.xxHash64/StreamExtensions.cs @@ -15,9 +15,9 @@ public static async Task Hash(this Stream stream, CancellationToken token) } public static async Task HashingCopy(this Stream inputStream, Stream outputStream, - CancellationToken token, IJob? job = null) + CancellationToken token, IJob? job = null, int buffserSize = 1024 * 1024) { - using var rented = MemoryPool.Shared.Rent(1024 * 1024); + using var rented = MemoryPool.Shared.Rent(buffserSize); var buffer = rented.Memory; var hasher = new xxHashAlgorithm(0); @@ -126,4 +126,84 @@ public static async Task HashingCopy(this Stream inputStream, Func Fn) HashingPull(this Stream src) + { + var stream = new PullingStream(src); + return (new BufferedStream(stream), () => stream.Hash); + } + + class PullingStream : Stream + { + private readonly Stream _src; + private readonly xxHashAlgorithm _hasher; + private ulong? _hash; + + public PullingStream(Stream src) + { + _src = src; + _hasher = new xxHashAlgorithm(0); + } + + public Hash Hash => new(_hash ?? throw new InvalidOperationException("Hash not yet computed")); + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotImplementedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => _src.Length; + public override long Position + { + get { throw new NotSupportedException(); } + set { throw new NotSupportedException(); } + } + + public override void Flush() + { + throw new NotImplementedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + if (_hash.HasValue) + throw new InvalidDataException("HashingPull can only be read once"); + + var sized = count >> 5 << 5; + if (sized == 0) + throw new ArgumentException("count must be a multiple of 32, got " + count, nameof(count)); + + + var read = _src.ReadAtLeast(buffer, sized); + + if (read == 0) + return 0; + + + if (read == sized) + { + _hasher.TransformByteGroupsInternal(buffer.AsSpan(offset, read)); + } + else + { + _hash = _hasher.FinalizeHashValueInternal(buffer.AsSpan(offset, read)); + return read; + } + + return read; + } + } } \ No newline at end of file diff --git a/Wabbajack.Server/ApiKeyAuthorizationHandler.cs b/Wabbajack.Server/ApiKeyAuthorizationHandler.cs deleted file mode 100644 index 11c52a50b..000000000 --- a/Wabbajack.Server/ApiKeyAuthorizationHandler.cs +++ /dev/null @@ -1,169 +0,0 @@ -using System.Security.Claims; -using System.Text.Encodings.Web; -using Microsoft.AspNetCore.Authentication; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.Networking.GitHub; -using Wabbajack.Networking.GitHub.DTOs; -using Wabbajack.Server.DataModels; -using Wabbajack.Server.DTOs; - -namespace Wabbajack.BuildServer; - -public class ApiKeyAuthenticationOptions : AuthenticationSchemeOptions -{ - public const string DefaultScheme = "API Key"; - public string AuthenticationType = DefaultScheme; - public string Scheme => DefaultScheme; -} - -public class ApiKeyAuthenticationHandler : AuthenticationHandler -{ - private const string ProblemDetailsContentType = "application/problem+json"; - public const string ApiKeyHeaderName = "X-Api-Key"; - private readonly DTOSerializer _dtos; - private readonly AppSettings _settings; - private readonly AuthorKeys _authorKeys; - private readonly Metrics _metricsStore; - private readonly TarLog _tarLog; - private readonly Client _githubClient; - private readonly MemoryCache _githubCache; - - public ApiKeyAuthenticationHandler( - IOptionsMonitor options, - AuthorKeys authorKeys, - Client githubClient, - ILoggerFactory logger, - UrlEncoder encoder, - ISystemClock clock, - DTOSerializer dtos, - Metrics metricsStore, - TarLog tarlog, - AppSettings settings) : base(options, logger, encoder, clock) - { - - _tarLog = tarlog; - _metricsStore = metricsStore; - _dtos = dtos; - _authorKeys = authorKeys; - _settings = settings; - _githubClient = githubClient; - _githubCache = new MemoryCache(new MemoryCacheOptions()); - } - - protected override async Task HandleAuthenticateAsync() - { - var metricsKey = Request.Headers[_settings.MetricsKeyHeader].FirstOrDefault(); - // Never needed this, disabled for now - //await LogRequest(metricsKey); - if (metricsKey != default) - { - if (await _tarLog.Contains(metricsKey)) - { - await _metricsStore.Ingest(new Metric - { - Subject = metricsKey, - Action = "tarlog", - MetricsKey = metricsKey, - UserAgent = Request.Headers.UserAgent, - Ip = Request.HttpContext.Connection.RemoteIpAddress?.ToString() ?? "" - }); - await Task.Delay(TimeSpan.FromSeconds(20)); - throw new Exception("Error, lipsum timeout of the cross distant cloud."); - } - } - - var authorKey = Request.Headers[ApiKeyHeaderName].FirstOrDefault(); - - if (authorKey == null) - Request.Cookies.TryGetValue(ApiKeyHeaderName, out authorKey); - - - if (authorKey == null && metricsKey == null) return AuthenticateResult.NoResult(); - - if (authorKey != null) - { - - var owner = await _authorKeys.AuthorForKey(authorKey); - if (owner == null) - { - var ghUser = await GetGithubUserInfo(authorKey); - - if (ghUser == null) - return AuthenticateResult.Fail("Invalid author key"); - - owner = "github/" + ghUser.Login; - } - - if (await _tarLog.Contains(owner)) - return AuthenticateResult.Fail("Banned author key"); - - var claims = new List {new(ClaimTypes.Name, owner)}; - - claims.Add(new Claim(ClaimTypes.Role, "Author")); - claims.Add(new Claim(ClaimTypes.Role, "User")); - - var identity = new ClaimsIdentity(claims, Options.AuthenticationType); - var identities = new List {identity}; - var principal = new ClaimsPrincipal(identities); - var ticket = new AuthenticationTicket(principal, Options.Scheme); - - return AuthenticateResult.Success(ticket); - } - - - if (!string.IsNullOrWhiteSpace(metricsKey)) - { - var claims = new List {new(ClaimTypes.Role, "User")}; - - - var identity = new ClaimsIdentity(claims, Options.AuthenticationType); - var identities = new List {identity}; - var principal = new ClaimsPrincipal(identities); - var ticket = new AuthenticationTicket(principal, Options.Scheme); - - return AuthenticateResult.Success(ticket); - } - - return AuthenticateResult.NoResult(); - } - - protected async Task GetGithubUserInfo(string authToken) - { - if (_githubCache.TryGetValue(authToken, out var value)) return value; - - var info = await _githubClient.GetUserInfoFromPAT(authToken); - if (info != null) - _githubCache.Set(authToken, info, - new MemoryCacheEntryOptions().SetSlidingExpiration(TimeSpan.FromHours(6))); - - return info; - } - - protected override async Task HandleChallengeAsync(AuthenticationProperties properties) - { - Response.StatusCode = 401; - Response.ContentType = ProblemDetailsContentType; - await Response.WriteAsync("Unauthorized"); - } - - protected override async Task HandleForbiddenAsync(AuthenticationProperties properties) - { - Response.StatusCode = 403; - Response.ContentType = ProblemDetailsContentType; - await Response.WriteAsync("forbidden"); - } -} - -public static class ApiKeyAuthorizationHandlerExtensions -{ - public static AuthenticationBuilder AddApiKeySupport(this AuthenticationBuilder authenticationBuilder, - Action options) - { - return authenticationBuilder.AddScheme( - ApiKeyAuthenticationOptions.DefaultScheme, options); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/AppSettings.cs b/Wabbajack.Server/AppSettings.cs deleted file mode 100644 index bbaddce77..000000000 --- a/Wabbajack.Server/AppSettings.cs +++ /dev/null @@ -1,61 +0,0 @@ -using Amazon.S3; -using Microsoft.Extensions.Configuration; -using Wabbajack.Paths; - -namespace Wabbajack.BuildServer; - -public class AppSettings -{ - public AppSettings(IConfiguration config) - { - config.Bind("WabbajackSettings", this); - } - public bool TestMode { get; set; } - public bool RunBackendNexusRoutines { get; set; } = true; - public string AuthorAPIKeyFile { get; set; } - - public string TarKeyFile { get; set; } - public string WabbajackBuildServerUri { get; set; } = "https://build.wabbajack.org/"; - public string MetricsKeyHeader { get; set; } = "x-metrics-key"; - public string TempFolder { get; set; } - - public string ProxyFolder { get; set; } - public AbsolutePath ProxyPath => (AbsolutePath) ProxyFolder; - public AbsolutePath TempPath => (AbsolutePath) TempFolder; - public string SpamWebHook { get; set; } = ""; - public string HamWebHook { get; set; } = ""; - - public string DiscordKey { get; set; } - - public string PatchesFilesFolder { get; set; } - public string MirrorFilesFolder { get; set; } - public string NexusCacheFolder { get; set; } - public string MetricsFolder { get; set; } = ""; - public string TarLogPath { get; set; } - public string GitHubKey { get; set; } = ""; - - public CouchDBSetting CesiDB { get; set; } - public CouchDBSetting MetricsDB { get; set; } - - public S3Settings S3 { get; set; } -} - -public class S3Settings -{ - public string AccessKey { get; set; } - public string SecretKey { get; set; } - public string ServiceUrl { get; set; } - - public string AuthoredFilesBucket { get; set; } - public string ProxyFilesBucket { get; set; } - - public string AuthoredFilesBucketCache { get; set; } -} - -public class CouchDBSetting -{ - public Uri Endpoint { get; set; } - public string Database { get; set; } - public string Username { get; set; } - public string Password { get; set; } -} \ No newline at end of file diff --git a/Wabbajack.Server/Badge.cs b/Wabbajack.Server/Badge.cs deleted file mode 100644 index da8013773..000000000 --- a/Wabbajack.Server/Badge.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Wabbajack.Server; - -public class Badge -{ - public Badge(string _label, string _message) - { - label = _label; - message = _message; - } - - public int schemaVersion { get; set; } = 1; - public string label { get; set; } - public string message { get; set; } - public string color { get; set; } -} \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/AuthorControls.cs b/Wabbajack.Server/Controllers/AuthorControls.cs deleted file mode 100644 index 71cb8c643..000000000 --- a/Wabbajack.Server/Controllers/AuthorControls.cs +++ /dev/null @@ -1,179 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Security.Claims; -using System.Text.Json; -using System.Threading.Tasks; -using FluentFTP.Helpers; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Nettle; -using Wabbajack.Common; -using Wabbajack.DTOs; -using Wabbajack.DTOs.GitHub; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.Networking.GitHub; -using Wabbajack.Paths.IO; -using Wabbajack.RateLimiter; -using Wabbajack.Server.DataModels; -using Wabbajack.Server.Extensions; -using Wabbajack.Server.Services; - -namespace Wabbajack.BuildServer.Controllers; - -[Authorize(Roles = "Author")] -[Route("/author_controls")] -public class AuthorControls : ControllerBase -{ - private readonly HttpClient _client; - private readonly DTOSerializer _dtos; - private readonly Client _gitHubClient; - private readonly QuickSync _quickSync; - private readonly AppSettings _settings; - private readonly ILogger _logger; - private readonly AuthorFiles _authorFiles; - private readonly IResource _limiter; - - public AuthorControls(ILogger logger, QuickSync quickSync, HttpClient client, - AppSettings settings, DTOSerializer dtos, AuthorFiles authorFiles, - Client gitHubClient, IResource limiter) - { - _logger = logger; - _quickSync = quickSync; - _client = client; - _settings = settings; - _dtos = dtos; - _gitHubClient = gitHubClient; - _authorFiles = authorFiles; - _limiter = limiter; - } - - [Route("login/{authorKey}")] - [AllowAnonymous] - public async Task Login(string authorKey) - { - Response.Cookies.Append(ApiKeyAuthenticationHandler.ApiKeyHeaderName, authorKey); - return Redirect($"{_settings.WabbajackBuildServerUri}author_controls/home"); - } - - [Route("lists")] - [HttpGet] - public async Task AuthorLists() - { - var user = User.FindFirstValue(ClaimTypes.Name); - var lists = (await LoadLists()) - .Where(l => l.Maintainers.Contains(user)) - .Select(l => l.NamespacedName) - .ToArray(); - - return Ok(lists); - } - - public async Task LoadLists() - { - var repos = await LoadRepositories(); - - return await repos.PMapAll(async url => - { - try - { - return (await _client.GetFromJsonAsync(_limiter, - new HttpRequestMessage(HttpMethod.Get, url.Value), - _dtos.Options))!.Select(meta => - { - meta.RepositoryName = url.Key; - return meta; - }); - } - catch (JsonException ex) - { - _logger.LogError(ex, "While loading repository {Name} from {Url}", url.Key, url.Value); - return Enumerable.Empty(); - } - }) - .SelectMany(x => x) - .ToArray(); - } - - public async Task> LoadRepositories() - { - var repositories = await _client.GetFromJsonAsync>(_limiter, - new HttpRequestMessage(HttpMethod.Get, - "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/repositories.json"), _dtos.Options); - return repositories!; - } - - [Route("whoami")] - [HttpGet] - public async Task GetWhoAmI() - { - var user = User.FindFirstValue(ClaimTypes.Name); - return Ok(user!); - } - - - [Route("lists/download_metadata")] - [HttpPost] - public async Task PostDownloadMetadata() - { - var user = User.FindFirstValue(ClaimTypes.Name); - var data = await _dtos.DeserializeAsync(Request.Body); - try - { - await _gitHubClient.UpdateList(user, data); - } - catch (Exception ex) - { - _logger.LogError(ex, "During posting of download_metadata"); - return BadRequest(ex); - } - - return Ok(data); - } - - private static async Task HomePageTemplate(object o) - { - var data = await KnownFolders.EntryPoint.Combine(@"Controllers\Templates\AuthorControls.html") - .ReadAllTextAsync(); - var func = NettleEngine.GetCompiler().Compile(data); - return func(o); - } - - [Route("home")] - [Authorize("")] - public async Task HomePage() - { - var user = User.FindFirstValue(ClaimTypes.Name); - var files = _authorFiles.AllDefinitions - .Where(af => af.Definition.Author == user) - .Select(af => new - { - Size = af.Definition.Size.FileSizeToString(), - OriginalSize = af.Definition.Size, - Name = af.Definition.OriginalFileName, - MangledName = af.Definition.MungedName, - UploadedDate = af.Updated - }) - .OrderBy(f => f.Name) - .ThenBy(f => f.UploadedDate) - .ToList(); - - var result = HomePageTemplate(new - { - User = user, - TotalUsage = files.Select(f => f.OriginalSize).Sum().ToFileSizeString(), - WabbajackFiles = files.Where(f => f.Name.EndsWith(Ext.Wabbajack.ToString())), - OtherFiles = files.Where(f => !f.Name.EndsWith(Ext.Wabbajack.ToString())) - }); - - return new ContentResult - { - ContentType = "text/html", - StatusCode = (int) HttpStatusCode.OK, - Content = await result - }; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/AuthoredFiles.cs b/Wabbajack.Server/Controllers/AuthoredFiles.cs deleted file mode 100644 index 107b79569..000000000 --- a/Wabbajack.Server/Controllers/AuthoredFiles.cs +++ /dev/null @@ -1,183 +0,0 @@ -using System.Net; -using System.Security.Claims; -using Humanizer; -using Humanizer.Localisation; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Primitives; -using Nettle; -using Wabbajack.Common; -using Wabbajack.DTOs.CDN; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Server.DataModels; -using Wabbajack.Server.DTOs; -using Wabbajack.Server.Services; - -namespace Wabbajack.BuildServer.Controllers; - -[Authorize(Roles = "Author")] -[Route("/authored_files")] -public class AuthoredFiles : ControllerBase -{ - private readonly DTOSerializer _dtos; - - private readonly DiscordWebHook _discord; - private readonly ILogger _logger; - private readonly AppSettings _settings; - private readonly AuthorFiles _authoredFiles; - private readonly Func _authoredFilesTemplate; - - - public AuthoredFiles(ILogger logger, AuthorFiles authorFiles, AppSettings settings, DiscordWebHook discord, - DTOSerializer dtos) - { - _logger = logger; - _settings = settings; - _discord = discord; - _dtos = dtos; - _authoredFiles = authorFiles; - using var stream = typeof(AuthoredFiles).Assembly - .GetManifestResourceStream("Wabbajack.Server.Resources.Reports.AuthoredFiles.html"); - _authoredFilesTemplate = NettleEngine.GetCompiler().Compile(stream.ReadAllText()); - } - - [HttpPut] - [Route("{serverAssignedUniqueId}/part/{index}")] - public async Task UploadFilePart(CancellationToken token, string serverAssignedUniqueId, long index) - { - var user = User.FindFirstValue(ClaimTypes.Name); - var definition = await _authoredFiles.ReadDefinitionForServerId(serverAssignedUniqueId); - if (definition.Author != user) - return Forbid("File Id does not match authorized user"); - _logger.Log(LogLevel.Information, - $"Uploading File part {definition.OriginalFileName} - ({index} / {definition.Parts.Length})"); - - var part = definition.Parts[index]; - - await using var ms = new MemoryStream(); - await Request.Body.CopyToLimitAsync(ms, (int) part.Size, token); - ms.Position = 0; - if (ms.Length != part.Size) - return BadRequest($"Couldn't read enough data for part {part.Size} vs {ms.Length}"); - - var hash = await ms.Hash(token); - if (hash != part.Hash) - return BadRequest( - $"Hashes don't match for index {index}. Sizes ({ms.Length} vs {part.Size}). Hashes ({hash} vs {part.Hash}"); - - ms.Position = 0; - await _authoredFiles.WritePart(definition.MungedName, (int) index, ms); - return Ok(part.Hash.ToBase64()); - } - - [HttpPut] - [Route("create")] - public async Task CreateUpload() - { - var user = User.FindFirstValue(ClaimTypes.Name); - - var definition = (await _dtos.DeserializeAsync(Request.Body))!; - - _logger.Log(LogLevel.Information, "Creating File upload {originalFileName}", definition.OriginalFileName); - - definition.ServerAssignedUniqueId = Guid.NewGuid().ToString(); - definition.Author = user; - await _authoredFiles.WriteDefinition(definition); - - await _discord.Send(Channel.Ham, - new DiscordMessage - { - Content = - $"{user} has started uploading {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})" - }); - - return Ok(definition.ServerAssignedUniqueId); - } - - [HttpPut] - [Route("{serverAssignedUniqueId}/finish")] - public async Task CreateUpload(string serverAssignedUniqueId) - { - var user = User.FindFirstValue(ClaimTypes.Name); - var definition = await _authoredFiles.ReadDefinitionForServerId(serverAssignedUniqueId); - if (definition.Author != user) - return Forbid("File Id does not match authorized user"); - _logger.Log(LogLevel.Information, $"Finalizing file upload {definition.OriginalFileName}"); - - await _discord.Send(Channel.Ham, - new DiscordMessage - { - Content = - $"{user} has finished uploading {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})" - }); - - var host = _settings.TestMode ? "test-files" : "authored-files"; - return Ok($"https://{host}.wabbajack.org/{definition.MungedName}"); - } - - [HttpDelete] - [Route("{serverAssignedUniqueId}")] - public async Task DeleteUpload(string serverAssignedUniqueId) - { - var user = User.FindFirstValue(ClaimTypes.Name); - var definition = _authoredFiles.AllDefinitions - .First(f => f.Definition.ServerAssignedUniqueId == serverAssignedUniqueId) - .Definition; - if (definition.Author != user) - return Forbid("File Id does not match authorized user"); - await _discord.Send(Channel.Ham, - new DiscordMessage - { - Content = - $"{user} is deleting {definition.MungedName}, {definition.Size.ToFileSizeString()} to be freed" - }); - _logger.Log(LogLevel.Information, $"Deleting upload {definition.OriginalFileName}"); - - await _authoredFiles.DeleteFile(definition); - return Ok(); - } - - [HttpGet] - [AllowAnonymous] - [Route("")] - public async Task UploadedFilesGet() - { - var files = _authoredFiles.AllDefinitions - .ToArray(); - var response = _authoredFilesTemplate(new - { - Files = files.OrderByDescending(f => f.Updated).ToArray(), - UsedSpace = _authoredFiles.UsedSpace.Bytes().Humanize("#.##"), - }); - return new ContentResult - { - ContentType = "text/html", - StatusCode = (int) HttpStatusCode.OK, - Content = response, - }; - } - - [HttpGet] - [AllowAnonymous] - [Route("direct_link/{mungedName}")] - public async Task DirectLink(string mungedName) - { - mungedName = _authoredFiles.DecodeName(mungedName); - var definition = await _authoredFiles.ReadDefinition(mungedName); - Response.Headers.ContentDisposition = - new StringValues($"attachment; filename={definition.OriginalFileName}"); - Response.Headers.ContentType = new StringValues("application/octet-stream"); - Response.Headers.ContentLength = definition.Size; - Response.Headers.ETag = definition.MungedName + "_direct"; - - foreach (var part in definition.Parts.OrderBy(p => p.Index)) - { - await _authoredFiles.StreamForPart(mungedName, (int)part.Index, async stream => - { - await stream.CopyToAsync(Response.Body); - }); - } - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/Cesi.cs b/Wabbajack.Server/Controllers/Cesi.cs deleted file mode 100644 index cc8fa19f4..000000000 --- a/Wabbajack.Server/Controllers/Cesi.cs +++ /dev/null @@ -1,106 +0,0 @@ -using cesi.DTOs; -using CouchDB.Driver; -using CouchDB.Driver.Views; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Wabbajack.Common; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.DTOs.Texture; -using Wabbajack.DTOs.Vfs; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Paths; -using Wabbajack.VFS; - -namespace Wabbajack.Server.Controllers; - -[Route("/cesi")] -public class Cesi : ControllerBase -{ - private readonly ILogger _logger; - private readonly ICouchDatabase _db; - private readonly DTOSerializer _dtos; - - public Cesi(ILogger logger, ICouchDatabase db, DTOSerializer serializer) - { - _logger = logger; - _db = db; - _dtos = serializer; - } - - [HttpGet("entry/{hash}")] - public async Task Entry(string hash) - { - return Ok(await _db.FindAsync(hash)); - } - - [HttpGet("vfs/{hash}")] - public async Task Vfs(string hash) - { - var entry = await _db.FindAsync(ReverseHash(hash)); - if (entry == null) return NotFound(new {Message = "Entry not found", Hash = hash, ReverseHash = ReverseHash(hash)}); - - - var indexed = new IndexedVirtualFile - { - Hash = Hash.FromHex(ReverseHash(entry.xxHash64)), - Size = entry.Size, - ImageState = GetImageState(entry), - Children = await GetChildrenState(entry), - }; - - - return Ok(_dtos.Serialize(indexed, true)); - } - - private async Task> GetChildrenState(Analyzed entry) - { - if (entry.Archive == null) return new List(); - - var children = await _db.GetViewAsync("Indexes", "ArchiveContents", new CouchViewOptions - { - IncludeDocs = true, - Key = entry.xxHash64 - }); - - var indexed = children.ToLookup(d => d.Document.xxHash64, v => v.Document); - - return await entry.Archive.Entries.SelectAsync(async e => - { - var found = indexed[e.Value].First(); - return new IndexedVirtualFile - { - Name = e.Key.ToRelativePath(), - Size = found.Size, - Hash = Hash.FromHex(ReverseHash(found.xxHash64)), - ImageState = GetImageState(found), - Children = await GetChildrenState(found), - }; - - }).ToList(); - } - - private ImageState? GetImageState(Analyzed entry) - { - if (entry.DDS == null) return null; - return new ImageState - { - Width = entry.DDS.Width, - Height = entry.DDS.Height, - Format = Enum.Parse(entry.DDS.Format), - PerceptualHash = new PHash(entry.DDS.PHash.FromHex()) - }; - } - - - private Hash ReverseHash(Hash hash) - { - return Hash.FromHex(hash.ToArray().Reverse().ToArray().ToHex()); - } - private string ReverseHash(string hash) - { - return hash.FromHex().Reverse().ToArray().ToHex(); - } - - -} \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/Github.cs b/Wabbajack.Server/Controllers/Github.cs deleted file mode 100644 index 087f7dcc6..000000000 --- a/Wabbajack.Server/Controllers/Github.cs +++ /dev/null @@ -1,52 +0,0 @@ -using System.Security.Claims; -using System.Text; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Wabbajack.Common; -using Wabbajack.Networking.GitHub; -using Wabbajack.Server.DTOs; -using Wabbajack.Server.Services; - -namespace Wabbajack.Server.Controllers; - -[Authorize(Roles = "Author")] -[Route("/github")] -public class Github : ControllerBase -{ - private readonly Client _client; - private readonly ILogger _logger; - private readonly DiscordWebHook _discord; - - public Github(ILogger logger, Client client, DiscordWebHook discord) - { - _client = client; - _logger = logger; - _discord = discord; - } - - [HttpGet] - public async Task GetContent([FromQuery] string owner, [FromQuery] string repo, [FromQuery] string path) - { - var (sha, content) = await _client.GetData(owner, repo, path); - Response.StatusCode = 200; - Response.Headers.Add("x-content-sha", sha); - await Response.WriteAsync(content); - } - - [HttpPost] - public async Task SetContent([FromQuery] string owner, [FromQuery] string repo, [FromQuery] string path, [FromQuery] string oldSha) - { - var user = User.FindFirstValue(ClaimTypes.Name)!; - _logger.LogInformation("Updating {Owner}/{Repo}/{Path} on behalf of {User}", owner, repo, path, user); - - await _discord.Send(Channel.Ham, - new DiscordMessage {Content = $"Updating {owner}/{repo}/{path} on behalf of {user}"}); - - var content = Encoding.UTF8.GetString(await Request.Body.ReadAllAsync()); - await _client.PutData(owner, repo, path, $"Update on behalf of {user}", content, oldSha); - return Ok(); - } - -} \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/Heartbeat.cs b/Wabbajack.Server/Controllers/Heartbeat.cs deleted file mode 100644 index 41e2d4bfc..000000000 --- a/Wabbajack.Server/Controllers/Heartbeat.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Wabbajack.Server; -using Wabbajack.Server.DTOs; -using Wabbajack.Server.Services; - -namespace Wabbajack.BuildServer.Controllers; - -[Route("/heartbeat")] -public class Heartbeat : ControllerBase -{ - private static readonly DateTime _startTime; - - private readonly GlobalInformation _globalInformation; - static Heartbeat() - { - _startTime = DateTime.Now; - } - - public Heartbeat(ILogger logger, GlobalInformation globalInformation, - QuickSync quickSync) - { - _globalInformation = globalInformation; - } - - [HttpGet] - public async Task GetHeartbeat() - { - return Ok(new HeartbeatResult - { - Uptime = DateTime.Now - _startTime, - }); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/Metrics.cs b/Wabbajack.Server/Controllers/Metrics.cs deleted file mode 100644 index 085dcf79e..000000000 --- a/Wabbajack.Server/Controllers/Metrics.cs +++ /dev/null @@ -1,178 +0,0 @@ -using System.Reflection; -using System.Text.Json; -using Chronic.Core; -using CouchDB.Driver; -using CouchDB.Driver.Views; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Nettle; -using Wabbajack.Common; -using Wabbajack.DTOs.ServerResponses; -using Wabbajack.Server.DataModels; -using Wabbajack.Server.DTOs; - -namespace Wabbajack.BuildServer.Controllers; - -[ApiController] -[Route("/metrics")] -public class MetricsController : ControllerBase -{ - private static readonly Func ReportTemplate = NettleEngine.GetCompiler().Compile(@" - -

Tar Report for {{$.key}}

-

Ban Status: {{$.status}}

- - {{each $.log }} - - - - - - {{/each}} -
{{$.Timestamp}}{{$.Path}}{{$.Key}}
- - "); - - private static Func _totalListTemplate; - private readonly AppSettings _settings; - private ILogger _logger; - private readonly Metrics _metricsStore; - private readonly ICouchDatabase _db; - - public MetricsController(ILogger logger, Metrics metricsStore, - AppSettings settings, ICouchDatabase db) - { - _logger = logger; - _settings = settings; - _metricsStore = metricsStore; - _db = db; - } - - - private static Func TotalListTemplate - { - get - { - if (_totalListTemplate == null) - { - var resource = Assembly.GetExecutingAssembly() - .GetManifestResourceStream("Wabbajack.Server.Controllers.Templates.TotalListTemplate.html")! - .ReadAllText(); - _totalListTemplate = NettleEngine.GetCompiler().Compile(resource); - } - - return _totalListTemplate; - } - } - - [HttpGet] - [Route("{subject}/{value}")] - public async Task LogMetricAsync(string subject, string value) - { - var date = DateTime.UtcNow; - var metricsKey = Request.Headers[_settings.MetricsKeyHeader].FirstOrDefault(); - - // Used in tests - if (value is "Default" or "untitled" || subject == "failed_download" || Guid.TryParse(value, out _)) - return new Result {Timestamp = date}; - - await _db.AddAsync(new Metric - { - Timestamp = date, - Action = subject, - Subject = value, - MetricsKey = metricsKey, - UserAgent = Request.Headers.UserAgent.FirstOrDefault() ?? "", - Ip = Request.Headers["cf-connecting-ip"].FirstOrDefault() ?? - (Request.HttpContext.Connection.RemoteIpAddress?.ToString() ?? "") - }); - - return new Result {Timestamp = date}; - } - - private static byte[] EOL = {(byte)'\n'}; - private static byte[] LBRACKET = {(byte)'['}; - private static byte[] RBRACKET = {(byte)']'}; - private static byte[] COMMA = {(byte) ','}; - - [HttpGet] - [Route("dump")] - public async Task GetMetrics([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to, [FromQuery] string? subject) - { - throw new NotImplementedException(); - } - - [HttpGet] - [Route("report")] - [ResponseCache(Duration = 60 * 10, VaryByQueryKeys = new [] {"action", "from", "to"})] - public async Task GetReport([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to) - { - var parser = new Parser(); - - to ??= "now"; - - var toDate = parser.Parse(to).Start!.Value.TruncateToDate(); - - var groupFilterStart = parser.Parse("three days ago").Start!.Value.TruncateToDate(); - toDate = new DateTime(toDate.Year, toDate.Month, toDate.Day); - - var prefetch = (await GetByAction(action, groupFilterStart, toDate)) - .Select(d => d.Subject) - .ToHashSet(); - - var fromDate = parser.Parse(from).Start!.Value.TruncateToDate(); - - var counts = (await GetByAction(action, fromDate, toDate)) - .Where(r => prefetch.Contains(r.Subject)) - .ToDictionary(kv => (kv.Date, kv.Subject), kv => kv.Count); - - Response.Headers.ContentType = "application/json"; - var row = new Dictionary(); - - Response.Body.Write(LBRACKET); - for (var d = fromDate; d <= toDate; d = d.AddDays(1)) - { - row["_Timestamp"] = d; - foreach (var group in prefetch) - { - if (counts.TryGetValue((d, group), out var found)) - row[group] = found; - else - row[group] = 0; - } - await JsonSerializer.SerializeAsync(Response.Body, row); - Response.Body.Write(EOL); - if (d != toDate) - Response.Body.Write(COMMA); - } - - Response.Body.Write(RBRACKET); - - } - - - private async Task> GetByAction(string action, DateTime from, DateTime to) - { - var records = await _db.GetViewAsync("Indexes", "ActionDaySubject", - new CouchViewOptions - { - StartKey = new object?[]{action, from.Year, from.Month, from.Day, null}, - EndKey = new object?[]{action, to.Year, to.Month, to.Day, new()}, - Reduce = true, - GroupLevel = 10, - Group = true - }); - - var results = records - .Where(r => r.Key.Length >= 4 && r.Key[4] != null) - .Select(r => - (new DateTime((int)(long)r.Key[1]!, (int)(long)r.Key[2]!, (int)(long)r.Key[3]!), (string)r.Key[4]!, r.Value)); - return results.ToList(); - } - - - public class Result - { - public DateTime Timestamp { get; set; } - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/MirroredFiles.cs b/Wabbajack.Server/Controllers/MirroredFiles.cs deleted file mode 100644 index b62d56705..000000000 --- a/Wabbajack.Server/Controllers/MirroredFiles.cs +++ /dev/null @@ -1,221 +0,0 @@ - - -using System.IO.Compression; -using System.Net; -using System.Security.Claims; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Primitives; -using Wabbajack.BuildServer; -using Wabbajack.Common; -using Wabbajack.DTOs.CDN; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Paths; -using Wabbajack.Paths.IO; -using Wabbajack.Server.DataModels; -using Wabbajack.Server.DTOs; -using Wabbajack.Server.Services; - -namespace Wabbajack.Server.Controllers; - -[Authorize(Roles = "Author")] -[Route("/mirrored_files")] -public class MirroredFiles : ControllerBase -{ - private readonly DTOSerializer _dtos; - - private readonly DiscordWebHook _discord; - private readonly ILogger _logger; - private readonly AppSettings _settings; - - public AbsolutePath MirrorFilesLocation => _settings.MirrorFilesFolder.ToAbsolutePath(); - - - public MirroredFiles(ILogger logger, AppSettings settings, DiscordWebHook discord, - DTOSerializer dtos) - { - _logger = logger; - _settings = settings; - _discord = discord; - _dtos = dtos; - } - - [HttpPut] - [Route("{hashAsHex}/part/{index}")] - public async Task UploadFilePart(CancellationToken token, string hashAsHex, long index) - { - var user = User.FindFirstValue(ClaimTypes.Name); - var definition = await ReadDefinition(hashAsHex); - if (definition.Author != user) - return Forbid("File Id does not match authorized user"); - _logger.Log(LogLevel.Information, - $"Uploading File part {definition.OriginalFileName} - ({index} / {definition.Parts.Length})"); - - var part = definition.Parts[index]; - - await using var ms = new MemoryStream(); - await Request.Body.CopyToLimitAsync(ms, (int) part.Size, token); - ms.Position = 0; - if (ms.Length != part.Size) - return BadRequest($"Couldn't read enough data for part {part.Size} vs {ms.Length}"); - - var hash = await ms.Hash(token); - if (hash != part.Hash) - return BadRequest( - $"Hashes don't match for index {index}. Sizes ({ms.Length} vs {part.Size}). Hashes ({hash} vs {part.Hash}"); - - ms.Position = 0; - await using var partStream = await CreatePart(hashAsHex, (int)index); - await ms.CopyToAsync(partStream, token); - return Ok(part.Hash.ToBase64()); - } - - [HttpPut] - [Route("create/{hashAsHex}")] - public async Task CreateUpload(string hashAsHex) - { - var user = User.FindFirstValue(ClaimTypes.Name); - - var definition = (await _dtos.DeserializeAsync(Request.Body))!; - - _logger.Log(LogLevel.Information, "Creating File upload {Hash}", hashAsHex); - - definition.ServerAssignedUniqueId = hashAsHex; - definition.Author = user; - await WriteDefinition(definition); - - await _discord.Send(Channel.Ham, - new DiscordMessage - { - Content = - $"{user} has started mirroring {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})" - }); - - return Ok(definition.ServerAssignedUniqueId); - } - - [HttpPut] - [Route("{hashAsHex}/finish")] - public async Task FinishUpload(string hashAsHex) - { - var user = User.FindFirstValue(ClaimTypes.Name); - var definition = await ReadDefinition(hashAsHex); - if (definition.Author != user) - return Forbid("File Id does not match authorized user"); - _logger.Log(LogLevel.Information, "Finalizing file upload {Hash}", hashAsHex); - - await _discord.Send(Channel.Ham, - new DiscordMessage - { - Content = - $"{user} has finished uploading {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})" - }); - - var host = _settings.TestMode ? "test-files" : "authored-files"; - return Ok($"https://{host}.wabbajack.org/{definition.MungedName}"); - } - - [HttpDelete] - [Route("{hashAsHex}")] - public async Task DeleteMirror(string hashAsHex) - { - var user = User.FindFirstValue(ClaimTypes.Name); - var definition = await ReadDefinition(hashAsHex); - - await _discord.Send(Channel.Ham, - new DiscordMessage - { - Content = - $"{user} is deleting {hashAsHex}, {definition.Size.ToFileSizeString()} to be freed" - }); - _logger.Log(LogLevel.Information, "Deleting upload {Hash}", hashAsHex); - - RootPath(hashAsHex).DeleteDirectory(); - return Ok(); - } - - [HttpGet] - [AllowAnonymous] - [Route("")] - public async Task MirroredFilesGet() - { - var files = await AllMirroredFiles(); - foreach (var file in files) - file.Parts = Array.Empty(); - return Ok(_dtos.Serialize(files)); - } - - - public IEnumerable AllDefinitions => MirrorFilesLocation.EnumerateFiles("definition.json.gz"); - public async Task AllMirroredFiles() - { - var defs = new List(); - foreach (var file in AllDefinitions) - { - defs.Add(await ReadDefinition(file)); - } - return defs.ToArray(); - } - - public async Task ReadDefinition(string hashAsHex) - { - return await ReadDefinition(RootPath(hashAsHex).Combine("definition.json.gz")); - } - - private async Task ReadDefinition(AbsolutePath file) - { - var gz = new GZipStream(new MemoryStream(await file.ReadAllBytesAsync()), CompressionMode.Decompress); - var definition = (await _dtos.DeserializeAsync(gz))!; - return definition; - } - - public async Task WriteDefinition(FileDefinition definition) - { - var path = RootPath(definition.Hash.ToHex()).Combine("definition.json.gz"); - path.Parent.CreateDirectory(); - path.Parent.Combine("parts").CreateDirectory(); - - await using var ms = new MemoryStream(); - await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true)) - { - await _dtos.Serialize(definition, gz); - } - - await path.WriteAllBytesAsync(ms.ToArray()); - } - - public AbsolutePath RootPath(string hashAsHex) - { - // Make sure it's a true hash before splicing into the path - return MirrorFilesLocation.Combine(Hash.FromHex(hashAsHex).ToHex()); - } - - - [HttpGet] - [AllowAnonymous] - [Route("direct_link/{hashAsHex}")] - public async Task DirectLink(string hashAsHex) - { - var definition = await ReadDefinition(hashAsHex); - Response.Headers.ContentDisposition = - new StringValues($"attachment; filename={definition.OriginalFileName}"); - Response.Headers.ContentType = new StringValues("application/octet-stream"); - foreach (var part in definition.Parts) - { - await using var partStream = await StreamForPart(hashAsHex, (int)part.Index); - await partStream.CopyToAsync(Response.Body); - } - } - - public async Task StreamForPart(string hashAsHex, int part) - { - return RootPath(hashAsHex).Combine("parts", part.ToString()).Open(FileMode.Open); - } - - public async Task CreatePart(string hashAsHex, int part) - { - return RootPath(hashAsHex).Combine("parts", part.ToString()).Open(FileMode.Create, FileAccess.Write, FileShare.None); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/ModFiles.cs b/Wabbajack.Server/Controllers/ModFiles.cs deleted file mode 100644 index b28d7ea81..000000000 --- a/Wabbajack.Server/Controllers/ModFiles.cs +++ /dev/null @@ -1,29 +0,0 @@ -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Wabbajack.DTOs; -using Wabbajack.DTOs.JsonConverters; - -namespace Wabbajack.BuildServer.Controllers; - -[Authorize(Roles = "User")] -[ApiController] -[Route("/mod_files")] -public class ModFilesForHash : ControllerBase -{ - private readonly DTOSerializer _dtos; - private ILogger _logger; - - public ModFilesForHash(ILogger logger, DTOSerializer dtos) - { - _logger = logger; - _dtos = dtos; - } - - [HttpGet("by_hash/{hashAsHex}")] - public async Task GetByHash(string hashAsHex) - { - var empty = Array.Empty(); - return Ok(_dtos.Serialize(empty)); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/NexusCache.cs b/Wabbajack.Server/Controllers/NexusCache.cs deleted file mode 100644 index 753c8d1a4..000000000 --- a/Wabbajack.Server/Controllers/NexusCache.cs +++ /dev/null @@ -1,94 +0,0 @@ -using System; -using System.Net.Http; -using System.Text.Json; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Wabbajack.Common; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.Networking.NexusApi.DTOs; -using Wabbajack.Paths.IO; -using Wabbajack.Server.Services; - -namespace Wabbajack.BuildServer.Controllers; - -//[Authorize] -[ApiController] -[Authorize(Roles = "User")] -[Route("/v1/games/")] -public class NexusCache : ControllerBase -{ - private readonly ILogger _logger; - private readonly HttpClient _client; - private readonly DTOSerializer _dtos; - private readonly NexusCacheManager _cache; - - public NexusCache(ILogger logger, HttpClient client, NexusCacheManager cache, DTOSerializer dtos) - { - _logger = logger; - _client = client; - _cache = cache; - _dtos = dtos; - } - - private async Task ForwardRequest(HttpRequest src, CancellationToken token) - { - _logger.LogInformation("Nexus Cache Forwarding: {path}", src.Path); - var request = new HttpRequestMessage(HttpMethod.Get, (Uri?)new Uri("https://api.nexusmods.com/" + src.Path)); - request.Headers.Add("apikey", (string?)src.Headers["apikey"]); - request.Headers.Add("User-Agent", (string?)src.Headers.UserAgent); - using var response = await _client.SendAsync(request, token); - return (await JsonSerializer.DeserializeAsync(await response.Content.ReadAsStreamAsync(token), _dtos.Options, token))!; - } - - /// - /// Looks up the mod details for a given Gamename/ModId pair. If the entry is not found in the cache it will - /// be requested from the server (using the caller's Nexus API key if provided). - /// - /// - /// The Nexus game name - /// The Nexus mod id - /// A Mod Info result - [HttpGet] - [Route("{GameName}/mods/{ModId}.json")] - public async Task GetModInfo(string GameName, long ModId, CancellationToken token) - { - var key = $"modinfo_{GameName}_{ModId}"; - await ReturnCachedResult(key, token); - } - - private async Task ReturnCachedResult(string key, CancellationToken token) - { - key = key.ToLowerInvariant(); - var cached = await _cache.GetCache(key, token); - if (cached == null) - { - var returned = await ForwardRequest(Request, token); - await _cache.SaveCache(key, returned, token); - Response.StatusCode = 200; - Response.ContentType = "application/json"; - await JsonSerializer.SerializeAsync(Response.Body, returned, _dtos.Options, cancellationToken: token); - return; - } - - await JsonSerializer.SerializeAsync(Response.Body, cached, _dtos.Options, cancellationToken: token); - } - - [HttpGet] - [Route("{GameName}/mods/{ModId}/files.json")] - public async Task GetModFiles(string GameName, long ModId, CancellationToken token) - { - var key = $"modfiles_{GameName}_{ModId}"; - await ReturnCachedResult(key, token); - } - - [HttpGet] - [Route("{GameName}/mods/{ModId}/files/{FileId}.json")] - public async Task GetModFile(string GameName, long ModId, long FileId, CancellationToken token) - { - var key = $"modfile_{GameName}_{ModId}_{FileId}"; - await ReturnCachedResult(key, token); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/Patches.cs b/Wabbajack.Server/Controllers/Patches.cs deleted file mode 100644 index 3359628ea..000000000 --- a/Wabbajack.Server/Controllers/Patches.cs +++ /dev/null @@ -1,75 +0,0 @@ -using System.Security.Claims; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Mvc; -using Wabbajack.BuildServer; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Paths; -using Wabbajack.Paths.IO; -using Wabbajack.Server.DTOs; -using Wabbajack.Server.Services; - -namespace Wabbajack.Server.Controllers; - -[ApiController] -[Authorize(Roles = "Author")] -[Route("/patches")] -public class Patches : ControllerBase -{ - private readonly AppSettings _settings; - private readonly DiscordWebHook _discord; - private readonly DTOSerializer _dtos; - - public Patches(AppSettings appSettings, DiscordWebHook discord, DTOSerializer dtos) - { - _settings = appSettings; - _discord = discord; - _dtos = dtos; - } - - [HttpPost] - public async Task WritePart(CancellationToken token, [FromQuery] string name, [FromQuery] long start) - { - var path = GetPath(name); - if (!path.FileExists()) - { - - var user = User.FindFirstValue(ClaimTypes.Name)!; - await _discord.Send(Channel.Ham, - new DiscordMessage {Content = $"{user} is uploading a new forced-healing patch file"}); - } - - await using var file = path.Open(FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read); - file.Position = start; - var hash = await Request.Body.HashingCopy(file, token); - await file.FlushAsync(token); - return Ok(hash.ToHex()); - } - - private AbsolutePath GetPath(string name) - { - return _settings.PatchesFilesFolder.ToAbsolutePath().Combine(name); - } - - [HttpGet] - [Route("list")] - public async Task ListPatches(CancellationToken token) - { - var root = _settings.PatchesFilesFolder.ToAbsolutePath(); - var files = root.EnumerateFiles() - .ToDictionary(f => f.RelativeTo(root).ToString(), f => f.Size()); - return Ok(_dtos.Serialize(files)); - } - - [HttpDelete] - public async Task DeletePart([FromQuery] string name) - { - var user = User.FindFirstValue(ClaimTypes.Name)!; - await _discord.Send(Channel.Ham, - new DiscordMessage {Content = $"{user} is deleting a new forced-healing patch file"}); - - GetPath(name).Delete(); - return Ok(name); - } - -} \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/Proxy.cs b/Wabbajack.Server/Controllers/Proxy.cs deleted file mode 100644 index 918351f56..000000000 --- a/Wabbajack.Server/Controllers/Proxy.cs +++ /dev/null @@ -1,228 +0,0 @@ -using System.Text; -using Amazon.Runtime; -using Amazon.S3; -using Amazon.S3.Model; -using FluentFTP.Helpers; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Microsoft.Net.Http.Headers; -using Wabbajack.BuildServer; -using Wabbajack.Downloaders; -using Wabbajack.Downloaders.Interfaces; -using Wabbajack.DTOs; -using Wabbajack.DTOs.DownloadStates; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Paths; -using Wabbajack.Paths.IO; -using Wabbajack.RateLimiter; -using Wabbajack.VFS; - -namespace Wabbajack.Server.Controllers; - -[ApiController] -[Route("/proxy")] -public class Proxy : ControllerBase -{ - private readonly ILogger _logger; - private readonly DownloadDispatcher _dispatcher; - private readonly TemporaryFileManager _tempFileManager; - private readonly AppSettings _appSettings; - private readonly FileHashCache _hashCache; - private readonly IAmazonS3 _s3; - private readonly string _bucket; - - private string _redirectUrl = "https://proxy.wabbajack.org/"; - private readonly IResource _resource; - - public Proxy(ILogger logger, DownloadDispatcher dispatcher, TemporaryFileManager tempFileManager, - FileHashCache hashCache, AppSettings appSettings, IAmazonS3 s3, IResource resource) - { - _logger = logger; - _dispatcher = dispatcher; - _tempFileManager = tempFileManager; - _appSettings = appSettings; - _hashCache = hashCache; - _s3 = s3; - _bucket = _appSettings.S3.ProxyFilesBucket; - _resource = resource; - } - - [HttpHead] - public async Task ProxyHead(CancellationToken token, [FromQuery] Uri uri, [FromQuery] string? name, - [FromQuery] string? hash) - { - var cacheName = (await Encoding.UTF8.GetBytes(uri.ToString()).Hash()).ToHex(); - return new RedirectResult(_redirectUrl + cacheName); - } - - [HttpGet] - public async Task ProxyGet(CancellationToken token, [FromQuery] Uri uri, [FromQuery] string? name, [FromQuery] string? hash) - { - - Hash hashResult = default; - var shouldMatch = hash != null ? Hash.FromHex(hash) : default; - - _logger.LogInformation("Got proxy request for {Uri}", uri); - var state = _dispatcher.Parse(uri); - var cacheName = (await Encoding.UTF8.GetBytes(uri.ToString()).Hash()).ToHex(); - var cacheFile = await GetCacheEntry(cacheName); - - if (state == null) - { - return BadRequest(new {Type = "Could not get state from Uri", Uri = uri.ToString()}); - } - - var archive = new Archive - { - Name = name ?? "", - State = state, - Hash = shouldMatch - - }; - - var downloader = _dispatcher.Downloader(archive); - if (downloader is not IProxyable) - { - return BadRequest(new {Type = "Downloader is not IProxyable", Downloader = downloader.GetType().FullName}); - } - - if (cacheFile != null && (DateTime.UtcNow - cacheFile.LastModified) > TimeSpan.FromHours(4)) - { - try - { - var verify = await _dispatcher.Verify(archive, token); - if (verify) - await TouchCacheEntry(cacheName); - } - catch (Exception ex) - { - _logger.LogInformation(ex, "When trying to verify cached file ({Hash}) {Url}", - cacheFile.Hash, uri); - await TouchCacheEntry(cacheName); - } - } - - if (cacheFile != null && (DateTime.Now - cacheFile.LastModified) > TimeSpan.FromHours(24)) - { - try - { - await DeleteCacheEntry(cacheName); - } - catch (Exception ex) - { - _logger.LogError(ex, "When trying to delete expired file"); - } - } - - - var redirectUrl = _redirectUrl + cacheName + "?response-content-disposition=attachment;filename=" + (name ?? "unknown"); - if (cacheFile != null) - { - if (hash != default) - { - if (cacheFile.Hash != shouldMatch) - return BadRequest(new {Type = "Unmatching Hashes", Expected = shouldMatch.ToHex(), Found = hashResult.ToHex()}); - } - return new RedirectResult(redirectUrl); - } - - _logger.LogInformation("Downloading proxy request for {Uri}", uri); - - var tempFile = _tempFileManager.CreateFile(deleteOnDispose:false); - - var proxyDownloader = _dispatcher.Downloader(archive) as IProxyable; - - using var job = await _resource.Begin("Downloading file", 0, token); - hashResult = await proxyDownloader!.Download(archive, tempFile.Path, job, token); - - - if (hash != default && hashResult != shouldMatch) - { - if (tempFile.Path.FileExists()) - tempFile.Path.Delete(); - return NotFound(); - } - - await PutCacheEntry(tempFile.Path, cacheName, hashResult); - - _logger.LogInformation("Returning proxy request for {Uri}", uri); - return new RedirectResult(redirectUrl); - } - - private async Task GetCacheEntry(string name) - { - GetObjectMetadataResponse info; - try - { - info = await _s3.GetObjectMetadataAsync(new GetObjectMetadataRequest() - { - BucketName = _bucket, - Key = name, - }); - } - catch (Exception _) - { - return null; - } - - if (info.HttpStatusCode == System.Net.HttpStatusCode.NotFound) - return null; - - if (info.Metadata["WJ-Hash"] == null) - return null; - - if (!Hash.TryGetFromHex(info.Metadata["WJ-Hash"], out var hash)) - return null; - - return new CacheStatus - { - LastModified = info.LastModified, - Size = info.ContentLength, - Hash = hash - }; - } - - private async Task TouchCacheEntry(string name) - { - await _s3.CopyObjectAsync(new CopyObjectRequest() - { - SourceBucket = _bucket, - DestinationBucket = _bucket, - SourceKey = name, - DestinationKey = name, - MetadataDirective = S3MetadataDirective.REPLACE, - }); - } - - private async Task PutCacheEntry(AbsolutePath path, string name, Hash hash) - { - var obj = new PutObjectRequest - { - BucketName = _bucket, - Key = name, - FilePath = path.ToString(), - ContentType = "application/octet-stream", - DisablePayloadSigning = true - }; - obj.Metadata.Add("WJ-Hash", hash.ToHex()); - await _s3.PutObjectAsync(obj); - } - - private async Task DeleteCacheEntry(string name) - { - await _s3.DeleteObjectAsync(new DeleteObjectRequest - { - BucketName = _bucket, - Key = name - }); - } - - record CacheStatus - { - public DateTime LastModified { get; init; } - public long Size { get; init; } - - public Hash Hash { get; init; } - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/Templates/AuthorControls.html b/Wabbajack.Server/Controllers/Templates/AuthorControls.html deleted file mode 100644 index 69840ab83..000000000 --- a/Wabbajack.Server/Controllers/Templates/AuthorControls.html +++ /dev/null @@ -1,73 +0,0 @@ - - - - - Author Controls - {{$.User}} - {{$.TotalUsage}} - - -

Author Controls - {{$.User}} - {{$.TotalUsage}}

-
-

Wabbajack Files

- - - - - - - - - {{each $.WabbajackFiles }} - - - - - - - - - {{/each}} -
CommandsNameSizeFinished UploadingUnique Name
- - {{$.Name}}{{$.Size}}{{$.UploadedDate}}{{$.MangledName}}
- -

Other Files

- - - - - - - - - - {{each $.OtherFiles }} - - - - - - - - {{/each}} -
CommandsNameSizeFinished UploadingUnique Name
- - {{$.Name}}{{$.Size}}{{$.UploadedDate}}{{$.MangledName}}
- - - \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/Templates/TotalListTemplate.html b/Wabbajack.Server/Controllers/Templates/TotalListTemplate.html deleted file mode 100644 index 395af2ae0..000000000 --- a/Wabbajack.Server/Controllers/Templates/TotalListTemplate.html +++ /dev/null @@ -1,21 +0,0 @@ - - - - - Total Installs - - - -

{{$.Title}} - Total: {{$.Total}}

- - - {{each $.Items }} - - - - - {{/each}} -
{{$.Count}}{{$.Title}}
- - - \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/Users.cs b/Wabbajack.Server/Controllers/Users.cs deleted file mode 100644 index d4d06d6dc..000000000 --- a/Wabbajack.Server/Controllers/Users.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System.Threading.Tasks; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Wabbajack.Paths.IO; -using Wabbajack.Server.DataLayer; - -namespace Wabbajack.BuildServer.Controllers; - -[Authorize] -[Route("/users")] -public class Users : ControllerBase -{ - private ILogger _logger; - private readonly AppSettings _settings; - private readonly SqlService _sql; - - public Users(ILogger logger, SqlService sql, AppSettings settings) - { - _settings = settings; - _logger = logger; - _sql = sql; - } - - [HttpGet] - [Route("add/{Name}")] - public async Task AddUser(string Name) - { - return await _sql.AddLogin(Name); - } - - [HttpGet] - [Route("export")] - public async Task Export() - { - var mainFolder = _settings.TempPath.Combine("exported_users"); - mainFolder.CreateDirectory(); - - foreach (var (owner, key) in await _sql.GetAllUserKeys()) - { - var folder = mainFolder.Combine(owner); - folder.CreateDirectory(); - await folder.Combine(_settings.AuthorAPIKeyFile).WriteAllTextAsync(key); - } - - return "done"; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DTOs/DiscordMessage.cs b/Wabbajack.Server/DTOs/DiscordMessage.cs deleted file mode 100644 index 3e8a04142..000000000 --- a/Wabbajack.Server/DTOs/DiscordMessage.cs +++ /dev/null @@ -1,76 +0,0 @@ -using System.Text.Json.Serialization; - -namespace Wabbajack.Server.DTOs; - -public class DiscordMessage -{ - [JsonPropertyName("username")] - public string? UserName { get; set; } - - [JsonPropertyName("avatar_url")] - public Uri? AvatarUrl { get; set; } - - [JsonPropertyName("content")] - public string? Content { get; set; } - - [JsonPropertyName("embeds")] - public DiscordEmbed[]? Embeds { get; set; } -} - -public class DiscordEmbed -{ - [JsonPropertyName("title")] public string Title { get; set; } - - [JsonPropertyName("color")] public int Color { get; set; } - - [JsonPropertyName("author")] public DiscordAuthor Author { get; set; } - - [JsonPropertyName("url")] public Uri Url { get; set; } - - [JsonPropertyName("description")] public string Description { get; set; } - - [JsonPropertyName("fields")] public DiscordField Field { get; set; } - - [JsonPropertyName("thumbnail")] public DiscordThumbnail Thumbnail { get; set; } - - [JsonPropertyName("image")] public DiscordImage Image { get; set; } - - [JsonPropertyName("footer")] public DiscordFooter Footer { get; set; } - - [JsonPropertyName("timestamp")] public DateTime Timestamp { get; set; } = DateTime.UtcNow; -} - -public class DiscordAuthor -{ - [JsonPropertyName("name")] public string Name { get; set; } - - [JsonPropertyName("url")] public Uri Url { get; set; } - - [JsonPropertyName("icon_url")] public Uri IconUrl { get; set; } -} - -public class DiscordField -{ - [JsonPropertyName("name")] public string Name { get; set; } - - [JsonPropertyName("value")] public string Value { get; set; } - - [JsonPropertyName("inline")] public bool Inline { get; set; } -} - -public class DiscordThumbnail -{ - [JsonPropertyName("Url")] public Uri Url { get; set; } -} - -public class DiscordImage -{ - [JsonPropertyName("Url")] public Uri Url { get; set; } -} - -public class DiscordFooter -{ - [JsonPropertyName("text")] public string Text { get; set; } - - [JsonPropertyName("icon_url")] public Uri icon_url { get; set; } -} \ No newline at end of file diff --git a/Wabbajack.Server/DTOs/HeartbeatResult.cs b/Wabbajack.Server/DTOs/HeartbeatResult.cs deleted file mode 100644 index e38d4b9ee..000000000 --- a/Wabbajack.Server/DTOs/HeartbeatResult.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System; - -namespace Wabbajack.Server.DTOs; - -public class HeartbeatResult -{ - public TimeSpan Uptime { get; set; } - public TimeSpan LastNexusUpdate { get; set; } - - public TimeSpan LastListValidation { get; set; } -} \ No newline at end of file diff --git a/Wabbajack.Server/DTOs/Metric.cs b/Wabbajack.Server/DTOs/Metric.cs deleted file mode 100644 index 164564a98..000000000 --- a/Wabbajack.Server/DTOs/Metric.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; -using CouchDB.Driver.Types; -using Microsoft.Extensions.Primitives; - -namespace Wabbajack.Server.DTOs; - -public class Metric : CouchDocument -{ - public DateTime Timestamp { get; set; } - public string Action { get; set; } - public string Subject { get; set; } - public string MetricsKey { get; set; } - public string UserAgent { get; set; } - public string Ip { get; set; } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataModels/AuthorFiles.cs b/Wabbajack.Server/DataModels/AuthorFiles.cs deleted file mode 100644 index a4c3bc6c3..000000000 --- a/Wabbajack.Server/DataModels/AuthorFiles.cs +++ /dev/null @@ -1,279 +0,0 @@ -using System.Collections.Concurrent; -using System.Diagnostics; -using System.IO.Compression; -using System.Web; -using Amazon.S3; -using Amazon.S3.Model; -using Microsoft.Extensions.Logging; -using Microsoft.IO; -using Wabbajack.BuildServer; -using Wabbajack.Common; -using Wabbajack.DTOs.CDN; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.Paths; -using Wabbajack.Paths.IO; - - -namespace Wabbajack.Server.DataModels; - -public class AuthorFiles -{ - private readonly ILogger _logger; - private readonly AppSettings _settings; - private readonly DTOSerializer _dtos; - private ConcurrentDictionary _byServerId = new(); - private readonly IAmazonS3 _s3; - private readonly ConcurrentDictionary _fileCache; - private readonly string _bucketName; - private ConcurrentDictionary _allObjects = new(); - private HashSet _mangledNames; - private readonly RecyclableMemoryStreamManager _streamPool; - private readonly HttpClient _httpClient; - private readonly AbsolutePath _cacheFile; - - private Uri _baseUri => new($"https://authored-files.wabbajack.org/"); - - public AuthorFiles(ILogger logger, AppSettings settings, DTOSerializer dtos, IAmazonS3 s3, HttpClient client) - { - _httpClient = client; - _s3 = s3; - _logger = logger; - _settings = settings; - _dtos = dtos; - _fileCache = new ConcurrentDictionary(); - _bucketName = settings.S3.AuthoredFilesBucket; - _ = PrimeCache(); - _streamPool = new RecyclableMemoryStreamManager(); - _cacheFile = _settings.S3.AuthoredFilesBucketCache.ToAbsolutePath(); - } - - private async Task PrimeCache() - { - try - { - if (!_cacheFile.FileExists()) - { - var allObjects = await AllObjects().ToArrayAsync(); - foreach (var obje in allObjects) - { - _allObjects.TryAdd(obje.Key.ToRelativePath(), obje.LastModified.ToFileTimeUtc()); - } - SaveBucketCacheFile(_cacheFile); - } - else - { - LoadBucketCacheFile(_cacheFile); - } - - - _mangledNames = _allObjects - .Where(f => f.Key.EndsWith("definition.json.gz")) - .Select(f => f.Key.Parent) - .ToHashSet(); - - await Parallel.ForEachAsync(_mangledNames, async (name, _) => - { - if (!_allObjects.TryGetValue(name.Combine("definition.json.gz"), out var value)) - return; - - _logger.LogInformation("Priming {Name}", name); - var definition = await PrimeDefinition(name); - var metadata = new FileDefinitionMetadata() - { - Definition = definition, - Updated = DateTime.FromFileTimeUtc(value) - }; - _fileCache.TryAdd(definition.MungedName, metadata); - _byServerId.TryAdd(definition.ServerAssignedUniqueId!, definition); - }); - - _logger.LogInformation("Finished priming cache, {Count} files {Size} GB cached", _fileCache.Count, - _fileCache.Sum(s => s.Value.Definition.Size) / (1024 * 1024 * 1024)); - - } - catch (Exception ex) - { - _logger.LogCritical(ex, "Failed to prime cache"); - } - } - - private void SaveBucketCacheFile(AbsolutePath cacheFile) - { - using var file = cacheFile.Open(FileMode.Create, FileAccess.Write); - using var sw = new StreamWriter(file); - foreach(var entry in _allObjects) - { - sw.WriteLine($"{entry.Key}||{entry.Value}"); - } - } - - private void LoadBucketCacheFile(AbsolutePath cacheFile) - { - using var file = cacheFile.Open(FileMode.Open, FileAccess.Read); - using var sr = new StreamReader(file); - while (!sr.EndOfStream) - { - var line = sr.ReadLine(); - var parts = line!.Split("||"); - _allObjects.TryAdd(parts[0].ToRelativePath(), long.Parse(parts[1])); - } - } - - private async Task PrimeDefinition(RelativePath name) - { - return await CircuitBreaker.WithAutoRetryAllAsync(_logger, async () => - { - var uri = _baseUri + $"{name}/definition.json.gz"; - using var response = await _httpClient.GetAsync(uri); - return await ReadDefinition(await response.Content.ReadAsStreamAsync()); - }); - } - - private async IAsyncEnumerable AllObjects() - { - var sw = Stopwatch.StartNew(); - var total = 0; - _logger.Log(LogLevel.Information, "Listing all objects in S3"); - var results = await _s3.ListObjectsV2Async(new ListObjectsV2Request() - { - BucketName = _bucketName, - }); - TOP: - total += results.S3Objects.Count; - _logger.Log(LogLevel.Information, "Got {S3ObjectsCount} objects, {Total} total", results.S3Objects.Count, total); - foreach (var result in results.S3Objects) - { - yield return result; - } - - if (results.IsTruncated) - { - results = await _s3.ListObjectsV2Async(new ListObjectsV2Request - { - ContinuationToken = results.NextContinuationToken, - BucketName = _bucketName, - }); - goto TOP; - } - _logger.LogInformation("Finished listing all objects in S3 in {Elapsed}", sw.Elapsed); - } - - public IEnumerable AllDefinitions => _fileCache.Values; - - /// - /// Used space in bytes - /// - public long UsedSpace => _fileCache.Sum(s => s.Value.Definition.Size); - - public async Task StreamForPart(string mungedName, int part, Func func) - { - var definition = _fileCache[mungedName].Definition; - - if (part >= definition.Parts.Length) - throw new ArgumentOutOfRangeException(nameof(part)); - - var uri = _baseUri + $"{mungedName}/parts/{part}"; - using var response = await _httpClient.GetAsync(uri); - await func(await response.Content.ReadAsStreamAsync()); - } - - public async Task WritePart(string mungedName, int part, Stream ms) - { - await _s3.PutObjectAsync(new PutObjectRequest - { - BucketName = _bucketName, - Key = mungedName.ToRelativePath().Combine("parts", part.ToString()).ToString().Replace("\\", "/"), - InputStream = ms, - DisablePayloadSigning = true, - ContentType = "application/octet-stream" - }); - } - - public async Task WriteDefinition(FileDefinition definition) - { - await using var ms = new MemoryStream(); - await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true)) - { - await _dtos.Serialize(definition, gz); - } - ms.Position = 0; - - await _s3.PutObjectAsync(new PutObjectRequest - { - BucketName = _bucketName, - Key = definition.MungedName.ToRelativePath().Combine("definition.json.gz").ToString().Replace("\\", "/"), - InputStream = ms, - DisablePayloadSigning = true, - ContentType = "application/octet-stream" - }); - _fileCache.TryAdd(definition.MungedName, new FileDefinitionMetadata - { - Definition = definition, - Updated = DateTime.UtcNow - }); - _byServerId.TryAdd(definition.ServerAssignedUniqueId!, definition); - } - - public async Task ReadDefinition(string mungedName) - { - return _fileCache[mungedName].Definition; - } - - public bool IsDefinition(string mungedName) - { - return _fileCache.ContainsKey(mungedName); - } - - - private async Task ReadDefinition(Stream stream) - { - var gz = new GZipStream(stream, CompressionMode.Decompress); - var definition = (await _dtos.DeserializeAsync(gz))!; - return definition; - } - - public class FileDefinitionMetadata - { - public FileDefinition Definition { get; set; } - public DateTime Updated { get; set; } - public string HumanSize => Definition.Size.ToFileSizeString(); - } - - public async Task DeleteFile(FileDefinition definition) - { - var allFiles = _allObjects.Where(f => f.Key.TopParent.ToString() == definition.MungedName) - .Select(f => f.Key).ToList(); - foreach (var batch in allFiles.Batch(512)) - { - var batchedArray = batch.ToHashSet(); - _logger.LogInformation("Deleting {Count} files for prefix {Prefix}", batchedArray.Count, definition.MungedName); - await _s3.DeleteObjectsAsync(new DeleteObjectsRequest - { - BucketName = _bucketName, - - Objects = batchedArray.Select(f => new KeyVersion - { - Key = f.ToString().Replace("\\", "/") - }).ToList() - }); - foreach (var key in batchedArray) - { - _allObjects.TryRemove(key, out _); - } - } - - _byServerId.TryRemove(definition.ServerAssignedUniqueId!, out _); - _fileCache.TryRemove(definition.MungedName, out _); - } - - public async ValueTask ReadDefinitionForServerId(string serverAssignedUniqueId) - { - return _byServerId[serverAssignedUniqueId]; - } - - public string DecodeName(string mungedName) - { - var decoded = HttpUtility.UrlDecode(mungedName); - return IsDefinition(decoded) ? decoded : mungedName; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataModels/AuthorKeys.cs b/Wabbajack.Server/DataModels/AuthorKeys.cs deleted file mode 100644 index 03a0e49e4..000000000 --- a/Wabbajack.Server/DataModels/AuthorKeys.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System.Threading.Tasks; -using Wabbajack.BuildServer; -using Wabbajack.Paths; -using Wabbajack.Paths.IO; - -namespace Wabbajack.Server.DataModels; - -public class AuthorKeys -{ - private readonly AppSettings _settings; - private AbsolutePath AuthorKeysPath => _settings.AuthorAPIKeyFile.ToAbsolutePath(); - - public AuthorKeys(AppSettings settings) - { - _settings = settings; - } - - public async Task AuthorForKey(string key) - { - await foreach (var line in AuthorKeysPath.ReadAllLinesAsync()) - { - var parts = line.Split("\t"); - if (parts[0].Trim() == key) - return parts[1].Trim(); - } - return null; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataModels/Metrics.cs b/Wabbajack.Server/DataModels/Metrics.cs deleted file mode 100644 index bbf7d5051..000000000 --- a/Wabbajack.Server/DataModels/Metrics.cs +++ /dev/null @@ -1,138 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Concurrent; -using System.IO; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using Chronic.Core; -using Microsoft.Toolkit.HighPerformance; -using Wabbajack.BuildServer; -using Wabbajack.Common; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.DTOs.ServerResponses; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Paths; -using Wabbajack.Paths.IO; -using Wabbajack.Server.DTOs; - -namespace Wabbajack.Server.DataModels; - -public class Metrics -{ - private readonly AppSettings _settings; - public SemaphoreSlim _lock = new(1); - private readonly DTOSerializer _dtos; - - public Metrics(AppSettings settings, DTOSerializer dtos) - { - _settings = settings; - _dtos = dtos; - } - - public async Task Ingest(Metric metric) - { - using var _ = await _lock.Lock(); - var data = Encoding.UTF8.GetBytes(_dtos.Serialize(metric)); - var metricsFile = _settings.MetricsFolder.ToAbsolutePath().Combine(DateTime.Now.ToString("yyyy_MM_dd") + ".json"); - await using var fs = metricsFile.Open(FileMode.Append, FileAccess.Write, FileShare.Read); - fs.Write(data); - fs.Write(Encoding.UTF8.GetBytes("\n")); - } - - private IEnumerable GetDates(DateTime fromDate, DateTime toDate) - { - for (var d = new DateTime(fromDate.Year, fromDate.Month, fromDate.Day); d <= toDate; d = d.AddDays(1)) - { - yield return d; - } - } - - - public async IAsyncEnumerable GetRecords(DateTime fromDate, DateTime toDate, string action) - { - ulong GetMetricKey(string key) - { - var hash = new xxHashAlgorithm(0); - Span bytes = stackalloc byte[key.Length]; - Encoding.ASCII.GetBytes(key, bytes); - return hash.HashBytes(bytes); - } - - foreach (var file in GetFiles(fromDate, toDate)) - { - await foreach (var line in file.ReadAllLinesAsync()) - { - var metric = _dtos.Deserialize(line)!; - if (metric.Action != action) continue; - if (metric.Timestamp >= fromDate && metric.Timestamp <= toDate) - { - yield return new MetricResult - { - Timestamp = metric.Timestamp, - Subject = metric.Subject, - Action = metric.Action, - MetricKey = GetMetricKey(metric.MetricsKey), - UserAgent = metric.UserAgent, - GroupingSubject = GetGroupingSubject(metric.Subject) - }; - } - } - } - } - - public ParallelQuery GetRecordsParallel(DateTime fromDate, DateTime toDate, string action) - { - ulong GetMetricKey(string key) - { - if (string.IsNullOrWhiteSpace(key)) return 0; - var hash = new xxHashAlgorithm(0); - Span bytes = stackalloc byte[key.Length]; - Encoding.ASCII.GetBytes(key, bytes); - return hash.HashBytes(bytes); - } - - var rows = GetFiles(fromDate, toDate).AsParallel() - .SelectMany(file => file.ReadAllLines()) - .Select(row => _dtos.Deserialize(row)!) - .Where(m => m.Action == action) - .Where(m => m.Timestamp >= fromDate && m.Timestamp <= toDate) - .Select(m => new MetricResult - { - Timestamp = m.Timestamp, - Subject = m.Subject, - Action = m.Action, - MetricKey = GetMetricKey(m.MetricsKey), - UserAgent = m.UserAgent, - GroupingSubject = GetGroupingSubject(m.Subject) - }); - return rows; - } - - private Regex groupingRegex = new("^[^0-9]*"); - private string GetGroupingSubject(string metricSubject) - { - try - { - var result = groupingRegex.Match(metricSubject).Groups[0].ToString(); - return string.IsNullOrEmpty(result) ? metricSubject : result; - } - catch (Exception) - { - return metricSubject; - } - } - - private IEnumerable GetFiles(DateTime fromDate, DateTime toDate) - { - var folder = _settings.MetricsFolder.ToAbsolutePath(); - foreach (var day in GetDates(fromDate, toDate)) - { - var file = folder.Combine(day.ToString("yyyy_MM_dd") + ".json"); - if (file.FileExists()) - yield return file; - } - } - -} \ No newline at end of file diff --git a/Wabbajack.Server/DataModels/TarLog.cs b/Wabbajack.Server/DataModels/TarLog.cs deleted file mode 100644 index b306e8b8a..000000000 --- a/Wabbajack.Server/DataModels/TarLog.cs +++ /dev/null @@ -1,48 +0,0 @@ -using Microsoft.Extensions.Logging; -using Wabbajack.BuildServer; -using Wabbajack.Paths; -using Wabbajack.Paths.IO; - -namespace Wabbajack.Server.DataModels; - -public class TarLog -{ - private Task> _tarKeys; - private readonly AppSettings _settings; - private readonly ILogger _logger; - - public TarLog(AppSettings settings, ILogger logger) - { - _settings = settings; - _logger = logger; - Load(); - } - - private void Load() - { - if (_settings.TarKeyFile.ToAbsolutePath().FileExists()) - { - _tarKeys = Task.Run(async () => - { - var keys = await _settings.TarKeyFile.ToAbsolutePath() - .ReadAllLinesAsync() - .Select(line => line.Trim()) - .ToHashSetAsync(); - _logger.LogInformation("Loaded {Count} tar keys", keys.Count); - return keys; - }); - } - else - { - _tarKeys = Task.Run(async () => new HashSet()); - } - - } - - public async Task Contains(string metricsKey) - { - return (await _tarKeys).Contains(metricsKey); - } - - -} \ No newline at end of file diff --git a/Wabbajack.Server/Extensions/NettleFunctions.cs b/Wabbajack.Server/Extensions/NettleFunctions.cs deleted file mode 100644 index d5795a96b..000000000 --- a/Wabbajack.Server/Extensions/NettleFunctions.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System.Text.RegularExpressions; -using System.Web; -using Nettle.Compiler; -using Nettle.Functions; - -namespace Wabbajack.Server.Extensions; - -public static class NettleFunctions -{ - public static INettleCompiler RegisterWJFunctions(this INettleCompiler compiler) - { - compiler.RegisterFunction(new Escape()); - return compiler; - } - - private sealed class UrlEncode : FunctionBase - { - public UrlEncode() : base() - { - DefineRequiredParameter("text", "text to encode", typeof(string)); - } - - protected override object GenerateOutput(TemplateContext context, params object[] parameterValues) - { - var value = GetParameterValue("text", parameterValues); - return HttpUtility.UrlEncode(value); - } - - public override string Description => "URL encodes a string"; - } - - private sealed class Escape : FunctionBase - { - public Escape() : base() - { - DefineRequiredParameter("text", "text to escape", typeof(string)); - } - - protected override object GenerateOutput(TemplateContext context, params object[] parameterValues) - { - var value = GetParameterValue("text", parameterValues); - return Regex.Escape(value).Replace("'", "\\'"); - } - - public override string Description => "Escapes a string"; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/GlobalInformation.cs b/Wabbajack.Server/GlobalInformation.cs deleted file mode 100644 index 80bc74c04..000000000 --- a/Wabbajack.Server/GlobalInformation.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System; - -namespace Wabbajack.Server; - -public class GlobalInformation -{ - public TimeSpan NexusAPIPollRate = TimeSpan.FromMinutes(15); - public TimeSpan NexusRSSPollRate = TimeSpan.FromMinutes(1); - public DateTime LastNexusSyncUTC { get; set; } - public TimeSpan TimeSinceLastNexusSync => DateTime.UtcNow - LastNexusSyncUTC; -} \ No newline at end of file diff --git a/Wabbajack.Server/Program.cs b/Wabbajack.Server/Program.cs deleted file mode 100644 index 8ab46d379..000000000 --- a/Wabbajack.Server/Program.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System.Linq; -using System.Net; -using System.Security.Cryptography.X509Certificates; -using Microsoft.AspNetCore.Hosting; -using Microsoft.Extensions.Hosting; - -namespace Wabbajack.Server; - -public class Program -{ - public static void Main(string[] args) - { - var testMode = args.Contains("TESTMODE"); - CreateHostBuilder(args, testMode).Build().Run(); - } - - public static IHostBuilder CreateHostBuilder(string[] args, bool testMode) - { - return Host.CreateDefaultBuilder(args) - .ConfigureWebHostDefaults(webBuilder => - { - webBuilder.UseStartup() - .UseKestrel(options => - { - options.AllowSynchronousIO = true; - options.Listen(IPAddress.Any, 5000); - options.Limits.MaxRequestBodySize = null; - }); - }); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Resources/Reports/AuthoredFiles.html b/Wabbajack.Server/Resources/Reports/AuthoredFiles.html deleted file mode 100644 index a7a216d71..000000000 --- a/Wabbajack.Server/Resources/Reports/AuthoredFiles.html +++ /dev/null @@ -1,45 +0,0 @@ - - - - Authored Files Report - - - - - - - - -

Authored Files:

-

{{$.UsedSpace}}

- - - - - - - - - - - {{each $.Files }} - - - - - - - - {{/each}} -
NameSizeAuthorUpdatedDirect Link
{{$.Definition.OriginalFileName}}{{$.HumanSize}}{{$.Definition.Author}}{{$.Updated}}(Slow) HTTP Direct Link
- - - \ No newline at end of file diff --git a/Wabbajack.Server/Services/AbstractService.cs b/Wabbajack.Server/Services/AbstractService.cs deleted file mode 100644 index ae7ef1180..000000000 --- a/Wabbajack.Server/Services/AbstractService.cs +++ /dev/null @@ -1,107 +0,0 @@ -using System; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Builder; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Wabbajack.BuildServer; - -namespace Wabbajack.Server.Services; - -public interface IStartable -{ - public Task Start(); -} - -public interface IReportingService -{ - public TimeSpan Delay { get; } - public DateTime LastStart { get; } - public DateTime LastEnd { get; } - - public (string, DateTime)[] ActiveWorkStatus { get; } -} - -public abstract class AbstractService : IStartable, IReportingService -{ - protected ILogger _logger; - protected QuickSync _quickSync; - protected AppSettings _settings; - - public AbstractService(ILogger logger, AppSettings settings, QuickSync quickSync, TimeSpan delay) - { - _settings = settings; - Delay = delay; - _logger = logger; - _quickSync = quickSync; - } - - public TimeSpan Delay { get; } - - public DateTime LastStart { get; private set; } - public DateTime LastEnd { get; private set; } - public (string, DateTime)[] ActiveWorkStatus { get; private set; } = { }; - - public async Task Start() - { - await Setup(); - await _quickSync.Register(this); - - while (true) - { - await _quickSync.ResetToken(); - try - { - _logger.LogInformation($"Running: {GetType().Name}"); - ActiveWorkStatus = Array.Empty<(string, DateTime)>(); - LastStart = DateTime.UtcNow; - await Execute(); - LastEnd = DateTime.UtcNow; - } - catch (Exception ex) - { - _logger.LogError(ex, "Running Service Loop"); - } - - var token = await _quickSync.GetToken(); - try - { - await Task.Delay(Delay, token); - } - catch (TaskCanceledException) - { - } - } - } - - public virtual async Task Setup() - { - } - - public abstract Task Execute(); - - protected void ReportStarting(string value) - { - lock (this) - { - ActiveWorkStatus = ActiveWorkStatus.Append((value, DateTime.UtcNow)).ToArray(); - } - } - - protected void ReportEnding(string value) - { - lock (this) - { - ActiveWorkStatus = ActiveWorkStatus.Where(x => x.Item1 != value).ToArray(); - } - } -} - -public static class AbstractServiceExtensions -{ - public static void UseService(this IApplicationBuilder b) - { - var poll = (IStartable) b.ApplicationServices.GetRequiredService(typeof(T)); - poll.Start(); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Services/DiscordBackend.cs b/Wabbajack.Server/Services/DiscordBackend.cs deleted file mode 100644 index c30ecdbe3..000000000 --- a/Wabbajack.Server/Services/DiscordBackend.cs +++ /dev/null @@ -1,104 +0,0 @@ -using Discord; -using Discord.WebSocket; -using Microsoft.Extensions.Logging; -using Wabbajack.BuildServer; - -namespace Wabbajack.Server.Services; - -public class DiscordBackend -{ - private readonly AppSettings _settings; - private readonly ILogger _logger; - private readonly DiscordSocketClient _client; - private readonly NexusCacheManager _nexusCacheManager; - - public DiscordBackend(ILogger logger, AppSettings settings, NexusCacheManager nexusCacheManager) - { - _settings = settings; - _logger = logger; - _nexusCacheManager = nexusCacheManager; - _client = new DiscordSocketClient(new DiscordSocketConfig() - { - - }); - _client.Log += LogAsync; - _client.Ready += ReadyAsync; - _client.MessageReceived += MessageReceivedAsync; - Task.Run(async () => - { - await _client.LoginAsync(TokenType.Bot, settings.DiscordKey); - await _client.StartAsync(); - }); - } - - private async Task MessageReceivedAsync(SocketMessage arg) - { - _logger.LogInformation(arg.Content); - - if (arg.Content.StartsWith("!dervenin")) - { - var parts = arg.Content.Split(" ", StringSplitOptions.RemoveEmptyEntries); - if (parts[0] != "!dervenin") - return; - - if (parts.Length == 1) - { - await ReplyTo(arg, "Wat?"); - } - - if (parts[1] == "purge-nexus-cache") - { - if (parts.Length != 3) - { - await ReplyTo(arg, "Welp you did that wrong, gotta give me a mod-id or url"); - return; - } - var rows = await _nexusCacheManager.Purge(parts[2]); - await ReplyTo(arg, $"Purged {rows} rows"); - } - - if (parts[1] == "nft") - { - await ReplyTo(arg, "No Fucking Thanks."); - } - - } - } - - private async Task ReplyTo(SocketMessage socketMessage, string message) - { - await socketMessage.Channel.SendMessageAsync(message); - } - - - private async Task ReadyAsync() - { - } - - private async Task LogAsync(LogMessage arg) - { - switch (arg.Severity) - { - case LogSeverity.Info: - _logger.LogInformation(arg.Message); - break; - case LogSeverity.Warning: - _logger.LogWarning(arg.Message); - break; - case LogSeverity.Critical: - _logger.LogCritical(arg.Message); - break; - case LogSeverity.Error: - _logger.LogError(arg.Exception, arg.Message); - break; - case LogSeverity.Verbose: - _logger.LogTrace(arg.Message); - break; - case LogSeverity.Debug: - _logger.LogDebug(arg.Message); - break; - default: - throw new ArgumentOutOfRangeException(); - } - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Services/DiscordWebHook.cs b/Wabbajack.Server/Services/DiscordWebHook.cs deleted file mode 100644 index f33909255..000000000 --- a/Wabbajack.Server/Services/DiscordWebHook.cs +++ /dev/null @@ -1,83 +0,0 @@ -using System; -using System.Net.Http; -using System.Reflection; -using System.Text; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Wabbajack.BuildServer; -using Wabbajack.Common; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.Server.DTOs; - -namespace Wabbajack.Server.Services; - -public enum Channel -{ - // High volume messaging, really only useful for internal devs - Spam, - - // Low volume messages designed for admins - Ham -} - -public class DiscordWebHook : AbstractService -{ - private readonly HttpClient _client; - private readonly DTOSerializer _dtos; - private readonly Random _random = new(); - - public DiscordWebHook(ILogger logger, AppSettings settings, QuickSync quickSync, HttpClient client, - DTOSerializer dtos) : base(logger, settings, quickSync, TimeSpan.FromHours(1)) - { - _settings = settings; - _logger = logger; - _client = client; - _dtos = dtos; - - Task.Run(async () => - { - - var message = new DiscordMessage - { - Content = $"\"{await GetQuote()}\" - Sheogorath (as he brings the server online)" - }; - await Send(Channel.Ham, message); - await Send(Channel.Spam, message); - }); - } - - public async Task Send(Channel channel, DiscordMessage message) - { - try - { - var url = channel switch - { - Channel.Spam => _settings.SpamWebHook, - Channel.Ham => _settings.HamWebHook, - _ => null - }; - if (string.IsNullOrWhiteSpace(url)) return; - - await _client.PostAsync(url, - new StringContent(_dtos.Serialize(message), Encoding.UTF8, "application/json")); - } - catch (Exception ex) - { - _logger.LogError(ex, "While sending discord message"); - } - } - - private async Task GetQuote() - { - var lines = - await Assembly.GetExecutingAssembly()!.GetManifestResourceStream("Wabbajack.Server.sheo_quotes.txt")! - .ReadLinesAsync() - .ToList(); - return lines[_random.Next(lines.Count)].Trim(); - } - - public override async Task Execute() - { - return 0; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Services/NexusCacheManager.cs b/Wabbajack.Server/Services/NexusCacheManager.cs deleted file mode 100644 index 01997b71d..000000000 --- a/Wabbajack.Server/Services/NexusCacheManager.cs +++ /dev/null @@ -1,172 +0,0 @@ -using System.Text.Json; -using K4os.Compression.LZ4.Internal; -using Microsoft.Extensions.Logging; -using Wabbajack.BuildServer; -using Wabbajack.Common; -using Wabbajack.DTOs; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.Networking.NexusApi; -using Wabbajack.Networking.NexusApi.DTOs; -using Wabbajack.Paths; -using Wabbajack.Paths.IO; -using Wabbajack.Server.DTOs; - -namespace Wabbajack.Server.Services; - -public class NexusCacheManager -{ - private readonly ILogger _loggger; - private readonly DTOSerializer _dtos; - private readonly AppSettings _configuration; - private readonly AbsolutePath _cacheFolder; - private readonly SemaphoreSlim _lockObject; - private readonly NexusApi _nexusAPI; - private readonly Timer _timer; - private readonly DiscordWebHook _discord; - - public NexusCacheManager(ILogger logger, DTOSerializer dtos, AppSettings configuration, NexusApi nexusApi, DiscordWebHook discord) - { - _loggger = logger; - _dtos = dtos; - _configuration = configuration; - _cacheFolder = configuration.NexusCacheFolder.ToAbsolutePath(); - _lockObject = new SemaphoreSlim(1); - _nexusAPI = nexusApi; - _discord = discord; - - if (configuration.RunBackendNexusRoutines) - { - _timer = new Timer(_ => UpdateNexusCacheAPI().FireAndForget(), null, TimeSpan.FromSeconds(2), - TimeSpan.FromHours(4)); - } - } - - - private AbsolutePath CacheFile(string key) - { - return _cacheFolder.Combine(key).WithExtension(Ext.Json); - } - - - private bool HaveCache(string key) - { - return CacheFile(key).FileExists(); - } - - public async Task SaveCache(string key, T value, CancellationToken token) - { - var ms = new MemoryStream(); - await JsonSerializer.SerializeAsync(ms, value, _dtos.Options, token); - await ms.FlushAsync(token); - var data = ms.ToArray(); - await _lockObject.WaitAsync(token); - try - { - await CacheFile(key).WriteAllBytesAsync(data, token: token); - } - finally - { - _lockObject.Release(); - } - } - - public async Task GetCache(string key, CancellationToken token) - { - if (!HaveCache(key)) return default; - - var file = CacheFile(key); - await _lockObject.WaitAsync(token); - byte[] data; - try - { - data = await file.ReadAllBytesAsync(token); - } - catch (FileNotFoundException) - { - return default; - } - finally - { - _lockObject.Release(); - } - return await JsonSerializer.DeserializeAsync(new MemoryStream(data), _dtos.Options, token); - } - - public async Task UpdateNexusCacheAPI() - { - var gameTasks = GameRegistry.Games.Values - .Where(g => g.NexusName != null) - .SelectAsync(async game => - { - var mods = await _nexusAPI.GetUpdates(game.Game, CancellationToken.None); - return (game, mods); - }); - - - var purgeList = new List<(string Key, DateTime Date)>(); - - await foreach (var (game, mods) in gameTasks) - { - foreach (var mod in mods.Item1) - { - var date = Math.Max(mod.LastestModActivity, mod.LatestFileUpdate).AsUnixTime(); - purgeList.Add(($"_{game.Game.MetaData().NexusName!.ToLowerInvariant()}_{mod.ModId}_", date)); - } - } - - // This is O(m * n) where n and m are 15,000 items, we really should improve this - var files = (from file in _cacheFolder.EnumerateFiles().AsParallel() - from entry in purgeList - where file.FileName.ToString().Contains(entry.Key) - where file.LastModifiedUtc() < entry.Date - select file).ToHashSet(); - - foreach (var file in files) - { - await PurgeCacheEntry(file); - } - - await _discord.Send(Channel.Ham, new DiscordMessage - { - Content = $"Cleared {files.Count} Nexus cache entries due to updates" - }); - } - - private async Task PurgeCacheEntry(AbsolutePath file) - { - await _lockObject.WaitAsync(); - try - { - if (file.FileExists()) file.Delete(); - } - catch (FileNotFoundException) - { - return; - } - finally - { - _lockObject.Release(); - } - } - - public async Task Purge(string mod) - { - if (Uri.TryCreate(mod, UriKind.Absolute, out var url)) - { - mod = Enumerable.Last(url.AbsolutePath.Split("/", StringSplitOptions.RemoveEmptyEntries)); - } - - var count = 0; - if (!int.TryParse(mod, out var mod_id)) return count; - - foreach (var file in _cacheFolder.EnumerateFiles()) - { - if (!file.FileName.ToString().Contains($"_{mod_id}")) continue; - - await PurgeCacheEntry(file); - count++; - } - - return count; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Services/QuickSync.cs b/Wabbajack.Server/Services/QuickSync.cs deleted file mode 100644 index 5cf93be5a..000000000 --- a/Wabbajack.Server/Services/QuickSync.cs +++ /dev/null @@ -1,69 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Wabbajack.Common; - -namespace Wabbajack.Server.Services; - -public class QuickSync -{ - private readonly AsyncLock _lock = new(); - private readonly ILogger _logger; - private readonly Dictionary _services = new(); - private readonly Dictionary _syncs = new(); - - public QuickSync(ILogger logger) - { - _logger = logger; - } - - public async Task> - Report() - { - using var _ = await _lock.WaitAsync(); - return _services.ToDictionary(s => s.Key, - s => (s.Value.Delay, DateTime.UtcNow - s.Value.LastEnd, s.Value.ActiveWorkStatus)); - } - - public async Task Register(T service) - where T : IReportingService - { - using var _ = await _lock.WaitAsync(); - _services[service.GetType()] = service; - } - - public async Task GetToken() - { - using var _ = await _lock.WaitAsync(); - if (_syncs.TryGetValue(typeof(T), out var result)) return result.Token; - var token = new CancellationTokenSource(); - _syncs[typeof(T)] = token; - return token.Token; - } - - public async Task ResetToken() - { - using var _ = await _lock.WaitAsync(); - if (_syncs.TryGetValue(typeof(T), out var ct)) ct.Cancel(); - _syncs[typeof(T)] = new CancellationTokenSource(); - } - - public async Task Notify() - { - _logger.LogInformation($"Quicksync {typeof(T).Name}"); - // Needs debugging - using var _ = await _lock.WaitAsync(); - if (_syncs.TryGetValue(typeof(T), out var ct)) ct.Cancel(); - } - - public async Task Notify(Type t) - { - _logger.LogInformation($"Quicksync {t.Name}"); - // Needs debugging - using var _ = await _lock.WaitAsync(); - if (_syncs.TryGetValue(t, out var ct)) ct.Cancel(); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs deleted file mode 100644 index d67d07269..000000000 --- a/Wabbajack.Server/Startup.cs +++ /dev/null @@ -1,278 +0,0 @@ -using System.Diagnostics; -using System.IO; -using System.Net.Http; -using System.Runtime.InteropServices; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading.Tasks; -using Amazon.Runtime; -using Amazon.S3; -using Amazon.Util.Internal; -using cesi.DTOs; -using CouchDB.Driver; -using CouchDB.Driver.Options; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Http.Features; -using Microsoft.AspNetCore.ResponseCompression; -using Microsoft.AspNetCore.StaticFiles; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.FileProviders; -using Microsoft.Extensions.Hosting; -using Nettle; -using Nettle.Compiler; -using Newtonsoft.Json; -using Octokit; -using Wabbajack.BuildServer; -using Wabbajack.Configuration; -using Wabbajack.Downloaders; -using Wabbajack.Downloaders.VerificationCache; -using Wabbajack.DTOs; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.DTOs.Logins; -using Wabbajack.Networking.GitHub; -using Wabbajack.Networking.Http; -using Wabbajack.Networking.Http.Interfaces; -using Wabbajack.Networking.NexusApi; -using Wabbajack.Paths; -using Wabbajack.RateLimiter; -using Wabbajack.Server.DataModels; -using Wabbajack.Server.Extensions; -using Wabbajack.Server.Services; -using Wabbajack.Services.OSIntegrated.TokenProviders; -using Wabbajack.Networking.WabbajackClientApi; -using Wabbajack.Paths.IO; -using Wabbajack.VFS; -using YamlDotNet.Serialization.NamingConventions; -using Client = Wabbajack.Networking.GitHub.Client; -using Metric = Wabbajack.Server.DTOs.Metric; -using SettingsManager = Wabbajack.Services.OSIntegrated.SettingsManager; - -namespace Wabbajack.Server; - -public class TestStartup : Startup -{ - public TestStartup(IConfiguration configuration) : base(configuration) - { - } -} - -public class Startup -{ - public Startup(IConfiguration configuration) - { - Configuration = configuration; - } - - public IConfiguration Configuration { get; } - - // This method gets called by the runtime. Use this method to add services to the container. - public void ConfigureServices(IServiceCollection services) - { - services.AddAuthentication(options => - { - options.DefaultAuthenticateScheme = ApiKeyAuthenticationOptions.DefaultScheme; - options.DefaultChallengeScheme = ApiKeyAuthenticationOptions.DefaultScheme; - }) - .AddApiKeySupport(options => { }); - - services.Configure(x => - { - x.ValueLengthLimit = int.MaxValue; - x.MultipartBodyLengthLimit = int.MaxValue; - }); - - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddAllSingleton(); - services.AddDownloadDispatcher(useLoginDownloaders:false, useProxyCache:false); - services.AddSingleton(s => - { - var appSettings = s.GetRequiredService(); - var settings = new BasicAWSCredentials(appSettings.S3.AccessKey, - appSettings.S3.SecretKey); - return new AmazonS3Client(settings, new AmazonS3Config - { - ServiceURL = appSettings.S3.ServiceUrl, - }); - }); - services.AddTransient(s => - { - var settings = s.GetRequiredService(); - return new TemporaryFileManager(settings.TempPath.Combine(Environment.ProcessId + "_" + Guid.NewGuid())); - }); - services.AddSingleton(); - - services.AddAllSingleton, WabbajackApiTokenProvider>(); - services.AddAllSingleton>(s => new Resource("Downloads", 12)); - services.AddAllSingleton>(s => new Resource("File Hashing", 12)); - services.AddAllSingleton>(s => - new Resource("Wabbajack Client", 4)); - - services.AddSingleton(s => - new FileHashCache(KnownFolders.AppDataLocal.Combine("Wabbajack", "GlobalHashCache.sqlite"), - s.GetService>()!)); - - services.AddAllSingleton, NexusApiTokenProvider>(); - services.AddAllSingleton>(s => new Resource("Web Requests", 12)); - // Application Info - - var version = - $"{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Patch}{ThisAssembly.Git.SemVer.DashLabel}"; - services.AddSingleton(s => new ApplicationInfo - { - ApplicationSlug = "Wabbajack", - ApplicationName = Environment.ProcessPath?.ToAbsolutePath().FileName.ToString() ?? "Wabbajack", - ApplicationSha = ThisAssembly.Git.Sha, - Platform = RuntimeInformation.ProcessArchitecture.ToString(), - OperatingSystemDescription = RuntimeInformation.OSDescription, - RuntimeIdentifier = RuntimeInformation.RuntimeIdentifier, - OSVersion = Environment.OSVersion.VersionString, - Version = version - }); - - - services.AddResponseCaching(); - services.AddSingleton(s => - { - var settings = s.GetService()!; - if (string.IsNullOrWhiteSpace(settings.GitHubKey)) - return new GitHubClient(new ProductHeaderValue("wabbajack")); - - var creds = new Credentials(settings.GitHubKey); - return new GitHubClient(new ProductHeaderValue("wabbajack")) {Credentials = creds}; - }); - services.AddDTOSerializer(); - services.AddDTOConverters(); - - services.AddSingleton(s => new Wabbajack.Services.OSIntegrated.Configuration - { - EncryptedDataLocation = KnownFolders.WabbajackAppLocal.Combine("encrypted"), - ModListsDownloadLocation = KnownFolders.EntryPoint.Combine("downloaded_mod_lists"), - SavedSettingsLocation = KnownFolders.WabbajackAppLocal.Combine("saved_settings"), - LogLocation = KnownFolders.LauncherAwarePath.Combine("logs"), - ImageCacheLocation = KnownFolders.WabbajackAppLocal.Combine("image_cache") - }); - - - services.AddSingleton(); - services.AddSingleton(s => Wabbajack.Services.OSIntegrated.ServiceExtensions.GetAppSettings(s, MainSettings.SettingsFileName)); - - services.AddResponseCompression(options => - { - options.Providers.Add(); - options.Providers.Add(); - options.MimeTypes = new[] {"application/json"}; - }); - - // CouchDB - services.AddSingleton(s => - { - var settings = s.GetRequiredService().CesiDB; - var client = new CouchClient(settings.Endpoint, b => - { - b.UseBasicAuthentication(settings.Username, settings.Password); - b.SetPropertyCase(PropertyCaseType.None); - b.SetJsonNullValueHandling(NullValueHandling.Ignore); - }); - return client.GetDatabase(settings.Database); - }); - - services.AddSingleton(s => - { - var settings = s.GetRequiredService().MetricsDB; - var client = new CouchClient(settings.Endpoint, b => - { - b.UseBasicAuthentication(settings.Username, settings.Password); - b.SetPropertyCase(PropertyCaseType.None); - b.SetJsonNullValueHandling(NullValueHandling.Ignore); - }); - return client.GetDatabase(settings.Database); - }); - - services.AddMvc(); - services - .AddControllers() - .AddJsonOptions(j => - { - j.JsonSerializerOptions.PropertyNamingPolicy = null; - j.JsonSerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault; - }); - - NettleEngine.GetCompiler().RegisterWJFunctions(); - } - - // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. - public void Configure(IApplicationBuilder app, IWebHostEnvironment env) - { - if (env.IsDevelopment()) app.UseDeveloperExceptionPage(); - - app.UseDeveloperExceptionPage(); - - var provider = new FileExtensionContentTypeProvider(); - provider.Mappings[".rar"] = "application/x-rar-compressed"; - provider.Mappings[".7z"] = "application/x-7z-compressed"; - provider.Mappings[".zip"] = "application/zip"; - provider.Mappings[".wabbajack"] = "application/zip"; - app.UseStaticFiles(); - - app.UseRouting(); - - app.UseAuthentication(); - app.UseAuthorization(); - app.UseResponseCompression(); - - app.UseService(); - - app.UseResponseCaching(); - - app.Use(next => - { - return async context => - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - context.Response.OnStarting(() => - { - stopWatch.Stop(); - var headers = context.Response.Headers; - headers.Add("Access-Control-Allow-Origin", "*"); - headers.Add("Access-Control-Allow-Methods", "POST, GET"); - headers.Add("Access-Control-Allow-Headers", "Accept, Origin, Content-type"); - headers.Add("X-ResponseTime-Ms", stopWatch.ElapsedMilliseconds.ToString()); - if (!headers.ContainsKey("Cache-Control")) - headers.Add("Cache-Control", "no-cache"); - return Task.CompletedTask; - }); - await next(context); - }; - }); - - app.UseFileServer(new FileServerOptions - { - FileProvider = new PhysicalFileProvider( - Path.Combine(Directory.GetCurrentDirectory(), "public")), - StaticFileOptions = {ServeUnknownFileTypes = true} - }); - - app.UseEndpoints(endpoints => { endpoints.MapControllers(); }); - - // Trigger the internal update code - app.ApplicationServices.GetRequiredService(); - app.ApplicationServices.GetRequiredService(); - - app.ApplicationServices.GetRequiredService(); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/TokenProviders/IDiscordToken.cs b/Wabbajack.Server/TokenProviders/IDiscordToken.cs deleted file mode 100644 index 153cb8dd8..000000000 --- a/Wabbajack.Server/TokenProviders/IDiscordToken.cs +++ /dev/null @@ -1,7 +0,0 @@ -using Wabbajack.Networking.Http.Interfaces; - -namespace Wabbajack.Server.TokenProviders; - -public interface IDiscordToken : ITokenProvider -{ -} \ No newline at end of file diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj deleted file mode 100644 index 7f7cffad1..000000000 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ /dev/null @@ -1,78 +0,0 @@ - - - - net8.0 - enable - enable - Exe - - - - CS8600,CS8601,CS8618,CS8604,CS1998 - - - - - - - - - - - - - - - - - - - - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Wabbajack.Server/appsettings.json b/Wabbajack.Server/appsettings.json deleted file mode 100644 index 1acb7a810..000000000 --- a/Wabbajack.Server/appsettings.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft": "Warning", - "Microsoft.Hosting.Lifetime": "Information" - } - }, - "WabbajackSettings": { - "RunBackendNexusRoutines": false, - "TempFolder": "c:\\tmp\\server_temp", - "MetricsFolder": "c:\\tmp\\server_metrics", - "AuthoredFilesFolder": "c:\\tmp\\server_authored_files", - "AuthorAPIKeyFile": "c:\\tmp\\author_keys.txt", - "PatchesFilesFolder": "c:\\tmp\\patches", - "MirrorFilesFolder": "c:\\tmp\\mirrors", - "NexusCacheFolder": "c:\\tmp\\nexus-cache", - "ProxyFolder": "c:\\tmp\\proxy", - "GitHubKey": "", - "CesiDB": { - "Endpoint": "http://localhost:15984", - "Database": "cesi", - "Username": "cesi", - "Password": "password" - }, - "MetricsDB": { - "Endpoint": "http://localhost:15984", - "Database": "metrics", - "Username": "wabbajack", - "Password": "password" - }, - "S3": { - "AccessKey": "<>", - "SecretKey": "<>", - "ServiceUrl": "<>", - "ProxyFilesBucket": "proxy-files", - "AuthoredFilesBucket": "authored-files", - "AuthoredFilesBucketCache": "c:\\tmp\\bucket-cache.txt" - } - }, - "AllowedHosts": "*" -} diff --git a/Wabbajack.Server/public/WABBAJACK_TEST_FILE.txt b/Wabbajack.Server/public/WABBAJACK_TEST_FILE.txt deleted file mode 100644 index 217a36f91..000000000 --- a/Wabbajack.Server/public/WABBAJACK_TEST_FILE.txt +++ /dev/null @@ -1 +0,0 @@ -Cheese for Everyone! \ No newline at end of file diff --git a/Wabbajack.Server/public/metrics.html b/Wabbajack.Server/public/metrics.html deleted file mode 100644 index eb2b5feb3..000000000 --- a/Wabbajack.Server/public/metrics.html +++ /dev/null @@ -1,117 +0,0 @@ - - - - - Wabbajack Metrics - - - - - - - -

Finished Install Counts

- -
- -

Begin Download

- -
- -

Begin Install

- -
- -

Finished Install

- -
- -

Started Wabbajack

- -
- -

Exceptions

- -
- - - - - \ No newline at end of file diff --git a/Wabbajack.Server/sheo_quotes.txt b/Wabbajack.Server/sheo_quotes.txt deleted file mode 100644 index 917d1bc3d..000000000 --- a/Wabbajack.Server/sheo_quotes.txt +++ /dev/null @@ -1,55 +0,0 @@ -I see you have completed my little errand. Well done. Perhaps you’ve gotten a taste of madness aswell? Do not believe madness to be a curse, mortal. For some it is the greatest of blessings. A bitter mercy perhaps, but mercy non the less. Give me the Fork of Horripilation, I believe I have something more suitable for your needs. Go now. Remember what you have seen. -Use the fork wisely, mortal. Few have wield to have not come away changed. Use the fork to strike a deathblow to the giant Bullnetch that resides near the hermit. Do this, return the Fork of Horripilation to me, and Sheogorath will reward you well. -What is it, mortal? Have you come to be of the service to Sheogorath? That in and of itself speaks toward your madness. This pleases me. Fetch the Fork of Horripliation from the mad hermit near Ald Redaynia. Take care with him. He’s not the most... stable man. -Unworthy, unworthy, unworthy! Useless mortal meat. Walking bag of dung! -Bring me a champion! Rend the flesh of my foes! A mortal champion to wade through the entrails of my enemies! -Really, do come in. It’s lovely in the Isles right now. Perfect time for a visit. -Greetings! Salutations! Welcome! Now go away. Leave. Run. Or die. -Isn't that a hoot? I love it, myself. Best part of being a Daedric Prince, really. Go ahead, try it again. He loves it! -Marvellous, marvellous! Self-immolation is a wonderful thing, isn't it? But now that we've had our fun, off to the Sacellum with you. -I suppose an introduction is in order. I'm Sheogorath, Prince of Madness. And other things. I'm not talking about them. -You should be off like the wind, solving problems and doing good deeds! -Time. Time is an artificial construct. An arbitrary system based on the idea that events occur in a linear direction at all times. -Always forward, never back. Is the concept of time correct? Is time relevant? It matters not. One way or another, I fear that our time has run out. -A new Gatekeeper! Excellent. We might be onto something with you, after all. That should keep out the stragglers. -A little busy here! I'm trying to decide what to have for dinner. Oh, how I love eating. One of my favorite things to do. -It's Jyggalag's time, and not a good time at all. You're going to help me stop it. First, though, you need to get your feet wet. -Another Daedric Prince. Not a nice one. I don't think ANY of the other Princes like him, actually. I mean, Malacath is more popular at parties. -The Daedric Prince of Order. Or biscuits... No. Order. And not in a good way. Bleak. Colorless. Dead. Boring, boring, boring. -The Greymarch comes, and Jyggalag walks. Or runs. Never skips, sidles, or struts. Mostly, he just destroys everything around him. -Once you understand what My Realm is, you might understand why it's important to keep it intact. -Two halves, two rulers, two places. Meet and greet. Do what they will, so you know what they're about. -Ask? ASK? I don't ask. I tell. This is My Realm, remember? My creation, My place, My rules. -Wonderful! Time for a celebration... Cheese for everyone! -Makes all of my subjects uneasy. Tense. Homicidal. Some of them, at least. We need to get that Torch relit, before the place falls apart. -You're going to stop the Greymarch by becoming Me. Or a version of Me. You'll be powerful. Powerful enough to stop Jyggalag. -You know what would be a good sign? "Free Sweetrolls!" Who wouldn't like that? -You'll be my champion. You'll grow powerful. You'll grow to be me. Prince of Madness, a new Sheogorath. Or you'll die trying. I love that about you. -Oh, don't forget to make use of dear Haskill. Between you and me, if he's not summoned three or four times a day, I don't think he feels appreciated. -I hate indecision! Or maybe I don't. Make up your mind, or I'll have your skin made into a hat -- one of those arrowcatchers. I love those hats! -So, which is it? What will it be? Mania? Dementia? The suspense is killing me. Or you, if I have to keep waiting. -Except where the backbone is an actual backbone. Ever been to Malacath's realm...? Nasty stuff. But, back to the business at hand. -Happens every time. The Greymarch starts, Order appears, and I become Jyggalag and wipe out My whole Realm. -Flee while you can, mortal. When we next meet I will not know you, and I will slay you like the others. -Ah... New Sheoth. My home away from places that aren't my home. The current location is much better than some of the prior ones. Don't you think? -The Isles, the Isles. A wonderful place! Except when it's horrible. Then it's horribly wonderful. Good for a visit. Or for an eternity. -Time to save the Realm! Rescue the damsel! Slay the beast! Or die trying. Your help is required. -Daedra are the embodiment of change. Change and permanency. I'm no different, except in the ways that I am. -Was it Molag? No, no... Little Tim, the toymaker's son? The ghost of King Lysandus? Or was it... Yes! Stanley, that talking grapefruit from Passwall. -Reeaaaallllyyyy? -Well? Spit it out, mortal. I haven't got an eternity! Oh, wait! I do. -I am a part of you, little mortal. I am a shadow in your subconscious, a blemish on your fragile little psyche. You know me. You just don't know it. -Sheogorath, Daedric Prince of Madness. At your service. -Yaaawwwwnn.... -Oh, pardon me. Were you saying something? I do apologize, it's just that I find myself suddenly and irrevocably... -Bored! -I mean, really. Here you stand, before Sheogorath himself, Daedric Prince of Madness, and all you deem fit to do is... deliver a message? How sad. -Now you. You can call me Ann Marie. -Oh... lovely. Now all my dear Pelagius has to worry about are the several hundred legitimate threats... -Ah, wonderful, wonderful! Why waste all that hatred on yourself when it can so easily be directed at others! -Mortal? Insufferable. -Yes, yes, you're entirely brilliant. Conquering madness and all that. Blah blah blah. -Ah, so now my dear Pelagius can hate himself for being legitimately afraid of things that actually threaten his existence... -Conquering paranoia should be a snap after that ordeal, hmm? -Welcome to the deceptively verdant mind of the Emperor Pelagius III. That's right! You're in the head of a dead, homicidally insane monarch. -The Wabbajack! Huh? Huh? Didn't see that coming, did you? \ No newline at end of file diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index 29873e83a..14647411e 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -13,6 +13,7 @@ using Wabbajack.Configuration; using Wabbajack.Downloaders; using Wabbajack.Downloaders.GameFile; +using Wabbajack.Downloaders.ModDB; using Wabbajack.Downloaders.VerificationCache; using Wabbajack.DTOs; using Wabbajack.DTOs.Interventions; @@ -166,6 +167,8 @@ public static IServiceCollection AddOSIntegrated(this IServiceCollection service // Token Providers service.AddAllSingleton, EncryptedJsonTokenProvider, NexusApiTokenProvider>(); + service.AddAllSingleton, EncryptedJsonTokenProvider, MegaTokenProvider>(); + service.AddAllSingleton, EncryptedJsonTokenProvider, BethesdaNetTokenProvider>(); service .AddAllSingleton, EncryptedJsonTokenProvider, diff --git a/Wabbajack.Services.OSIntegrated/TokenProviders/MegaTokenProvider.cs b/Wabbajack.Services.OSIntegrated/TokenProviders/MegaTokenProvider.cs new file mode 100644 index 000000000..2eee5f3ea --- /dev/null +++ b/Wabbajack.Services.OSIntegrated/TokenProviders/MegaTokenProvider.cs @@ -0,0 +1,12 @@ +using Microsoft.Extensions.Logging; +using Wabbajack.Downloaders.ModDB; +using Wabbajack.DTOs.JsonConverters; + +namespace Wabbajack.Services.OSIntegrated.TokenProviders; + +public class MegaTokenProvider : EncryptedJsonTokenProvider +{ + public MegaTokenProvider(ILogger logger, DTOSerializer dtos) : base(logger, dtos, "mega-login") + { + } +} \ No newline at end of file diff --git a/Wabbajack.sln b/Wabbajack.sln index 1800043e0..bedde649f 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -75,8 +75,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Compiler.Test", " EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Downloaders.WabbajackCDN", "Wabbajack.Downloaders.WabbajackCDN\Wabbajack.Downloaders.WabbajackCDN.csproj", "{0210A092-4A69-479F-8FF4-120921B5758E}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Server", "Wabbajack.Server\Wabbajack.Server.csproj", "{8F7FB1BE-4E1E-4798-AC9E-AA2E18CCDD42}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Networking.GitHub", "Wabbajack.Networking.GitHub\Wabbajack.Networking.GitHub.csproj", "{549D01FD-8E26-42C7-ABEA-4BAF6E24E754}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Services.OSIntegrated", "Wabbajack.Services.OSIntegrated\Wabbajack.Services.OSIntegrated.csproj", "{45E48012-6C58-4C3D-843F-C6EED02868B7}" @@ -299,10 +297,6 @@ Global {0210A092-4A69-479F-8FF4-120921B5758E}.Debug|Any CPU.Build.0 = Debug|Any CPU {0210A092-4A69-479F-8FF4-120921B5758E}.Release|Any CPU.ActiveCfg = Release|Any CPU {0210A092-4A69-479F-8FF4-120921B5758E}.Release|Any CPU.Build.0 = Release|Any CPU - {8F7FB1BE-4E1E-4798-AC9E-AA2E18CCDD42}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {8F7FB1BE-4E1E-4798-AC9E-AA2E18CCDD42}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8F7FB1BE-4E1E-4798-AC9E-AA2E18CCDD42}.Release|Any CPU.ActiveCfg = Release|Any CPU - {8F7FB1BE-4E1E-4798-AC9E-AA2E18CCDD42}.Release|Any CPU.Build.0 = Release|Any CPU {549D01FD-8E26-42C7-ABEA-4BAF6E24E754}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {549D01FD-8E26-42C7-ABEA-4BAF6E24E754}.Debug|Any CPU.Build.0 = Debug|Any CPU {549D01FD-8E26-42C7-ABEA-4BAF6E24E754}.Release|Any CPU.ActiveCfg = Release|Any CPU