diff --git a/Launcher.props b/Launcher.props index 9c20db7..4557273 100644 --- a/Launcher.props +++ b/Launcher.props @@ -8,6 +8,6 @@ 3. Local dev SS14.Loader launching code. --> net8.0 - 0.25.1 + 0.26.0 diff --git a/Marsey/Config/MarseyVars.cs b/Marsey/Config/MarseyVars.cs index 47bf018..d3ec481 100644 --- a/Marsey/Config/MarseyVars.cs +++ b/Marsey/Config/MarseyVars.cs @@ -13,7 +13,7 @@ public static class MarseyVars /// Due to the nature of how Marseyloader is compiled (its not) we cannot check its version /// Almost as if this variable is a consequence of having the project included rather than referenced /// - public static readonly Version MarseyVersion = new Version("2.12.4"); + public static readonly Version MarseyVersion = new Version("2.12.5"); /// /// Default MarseyAPI endpoint url diff --git a/SS14.Launcher/ConfigConstants.cs b/SS14.Launcher/ConfigConstants.cs index ea8f51b..92af73c 100644 --- a/SS14.Launcher/ConfigConstants.cs +++ b/SS14.Launcher/ConfigConstants.cs @@ -38,6 +38,10 @@ public static class ConfigConstants public const string RobustBuildsManifest = "https://central.spacestation14.io/builds/robust/manifest.json"; public const string RobustModulesManifest = "https://central.spacestation14.io/builds/robust/modules.json"; + // How long to keep cached copies of Robust manifests. + // TODO: Take this from Cache-Control header responses instead. + public static readonly TimeSpan RobustManifestCacheTime = TimeSpan.FromMinutes(15); + public const string UrlOverrideAssets = "https://central.spacestation14.io/launcher/override_assets.json"; public const string UrlAssetsBase = "https://central.spacestation14.io/launcher/assets/"; diff --git a/SS14.Launcher/Models/Data/CVars.cs b/SS14.Launcher/Models/Data/CVars.cs index 3ae7743..1fb7144 100644 --- a/SS14.Launcher/Models/Data/CVars.cs +++ b/SS14.Launcher/Models/Data/CVars.cs @@ -89,6 +89,21 @@ public static readonly CVarDef HasDismissedEarlyAccessWarning /// public static readonly CVarDef FilterPlayerCountMaxValue = CVarDef.Create("FilterPlayerCountMaxValue", 0); + /// + /// Enable local overriding of engine versions. + /// + /// + /// If enabled and on a development build, + /// the launcher will pull all engine versions and modules from . + /// This can be set to RobustToolbox/release/ to instantly pull in packaged engine builds. + /// + public static readonly CVarDef EngineOverrideEnabled = CVarDef.Create("EngineOverrideEnabled", false); + + /// + /// Path to load engines from when using . + /// + public static readonly CVarDef EngineOverridePath = CVarDef.Create("EngineOverridePath", ""); + // MarseyCVars start here // Stealthsey diff --git a/SS14.Launcher/Models/EngineManager/EngineManagerDynamic.Manifest.cs b/SS14.Launcher/Models/EngineManager/EngineManagerDynamic.Manifest.cs new file mode 100644 index 0000000..20cf83b --- /dev/null +++ b/SS14.Launcher/Models/EngineManager/EngineManagerDynamic.Manifest.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Net.Http.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Serilog; + +namespace SS14.Launcher.Models.EngineManager; + +public sealed partial class EngineManagerDynamic +{ + // This part of the code is responsible for downloading and caching the Robust build manifest. + + private readonly SemaphoreSlim _manifestSemaphore = new(1); + private readonly Stopwatch _manifestStopwatch = Stopwatch.StartNew(); + + private Dictionary? _cachedRobustVersionInfo; + private TimeSpan _robustCacheValidUntil; + + /// + /// Look up information about an engine version. + /// + /// The version number to look up. + /// Follow redirections in version info. + /// Cancellation token. + /// + /// Information about the version, or null if it could not be found. + /// The returned version may be different than what was requested if redirects were followed. + /// + private async ValueTask GetVersionInfo( + string version, + bool followRedirects = true, + CancellationToken cancel = default) + { + await _manifestSemaphore.WaitAsync(cancel); + try + { + return await GetVersionInfoCore(version, followRedirects, cancel); + } + finally + { + _manifestSemaphore.Release(); + } + } + + private async ValueTask GetVersionInfoCore( + string version, + bool followRedirects, + CancellationToken cancel) + { + // If we have a cached copy, and it's not expired, we check it. + if (_cachedRobustVersionInfo != null && _robustCacheValidUntil > _manifestStopwatch.Elapsed) + { + // Check the version. If this fails, we immediately re-request the manifest as it may have changed. + // (Connecting to a freshly-updated server with a new Robust version, within the cache window.) + if (FindVersionInfoInCached(version, followRedirects) is { } foundVersionInfo) + return foundVersionInfo; + } + + await UpdateBuildManifest(cancel); + + return FindVersionInfoInCached(version, followRedirects); + } + + private async Task UpdateBuildManifest(CancellationToken cancel) + { + // TODO: If-Modified-Since and If-None-Match request conditions. + + Log.Debug("Loading manifest from {manifestUrl}...", ConfigConstants.RobustBuildsManifest); + _cachedRobustVersionInfo = + await _http.GetFromJsonAsync>( + ConfigConstants.RobustBuildsManifest, cancellationToken: cancel); + + _robustCacheValidUntil = _manifestStopwatch.Elapsed + ConfigConstants.RobustManifestCacheTime; + } + + private FoundVersionInfo? FindVersionInfoInCached(string version, bool followRedirects) + { + Debug.Assert(_cachedRobustVersionInfo != null); + + if (!_cachedRobustVersionInfo.TryGetValue(version, out var versionInfo)) + return null; + + if (followRedirects) + { + while (versionInfo.RedirectVersion != null) + { + version = versionInfo.RedirectVersion; + versionInfo = _cachedRobustVersionInfo[versionInfo.RedirectVersion]; + } + } + + return new FoundVersionInfo(version, versionInfo); + } + + private sealed record FoundVersionInfo(string Version, VersionInfo Info); + + private sealed record VersionInfo( + bool Insecure, + [property: JsonPropertyName("redirect")] + string? RedirectVersion, + Dictionary Platforms); + + private sealed class BuildInfo + { + [JsonInclude] [JsonPropertyName("url")] + public string Url = default!; + + [JsonInclude] [JsonPropertyName("sha256")] + public string Sha256 = default!; + + [JsonInclude] [JsonPropertyName("sig")] + public string Signature = default!; + } +} diff --git a/SS14.Launcher/Models/EngineManager/EngineManagerDynamic.cs b/SS14.Launcher/Models/EngineManager/EngineManagerDynamic.cs index 471ba10..aeeb074 100644 --- a/SS14.Launcher/Models/EngineManager/EngineManagerDynamic.cs +++ b/SS14.Launcher/Models/EngineManager/EngineManagerDynamic.cs @@ -5,7 +5,7 @@ using System.Linq; using System.Net.Http; using System.Net.Http.Json; -using System.Text.Json.Serialization; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Dapper; @@ -21,8 +21,10 @@ namespace SS14.Launcher.Models.EngineManager; /// /// Downloads engine versions from the website. /// -public sealed class EngineManagerDynamic : IEngineManager +public sealed partial class EngineManagerDynamic : IEngineManager { + public const string OverrideVersionName = "_OVERRIDE_"; + private readonly DataManager _cfg; private readonly HttpClient _http; @@ -34,6 +36,13 @@ public EngineManagerDynamic() public string GetEnginePath(string engineVersion) { +#if DEVELOPMENT + if (_cfg.GetCVar(CVars.EngineOverrideEnabled)) + { + return FindOverrideZip("Robust.Client", _cfg.GetCVar(CVars.EngineOverridePath)); + } +#endif + if (!_cfg.EngineInstallations.Lookup(engineVersion).HasValue) { throw new ArgumentException("We do not have that engine version!"); @@ -44,43 +53,59 @@ public string GetEnginePath(string engineVersion) public string GetEngineModule(string moduleName, string moduleVersion) { +#if DEVELOPMENT + if (_cfg.GetCVar(CVars.EngineOverrideEnabled)) + moduleVersion = OverrideVersionName; +#endif + return Path.Combine(LauncherPaths.DirModuleInstallations, moduleName, moduleVersion); } public string GetEngineSignature(string engineVersion) { +#if DEVELOPMENT + if (_cfg.GetCVar(CVars.EngineOverrideEnabled)) + return "DEADBEEF"; +#endif + return _cfg.EngineInstallations.Lookup(engineVersion).Value.Signature; } - public async Task DownloadEngineIfNecessary( + public async Task DownloadEngineIfNecessary( string engineVersion, Helpers.DownloadProgressCallback? progress = null, CancellationToken cancel = default) { - if (_cfg.EngineInstallations.Lookup(engineVersion).HasValue) +#if DEVELOPMENT + if (_cfg.GetCVar(CVars.EngineOverrideEnabled)) { - // Already have the engine version, we're good. - return false; + // Engine override means we don't need to download anything, we have it locally! + // At least, if we don't, we'll just blame the developer that enabled it. + return new EngineInstallationResult(engineVersion, false); } +#endif - Log.Information("Installing engine version {version}...", engineVersion); + var foundVersion = await GetVersionInfo(engineVersion, cancel: cancel); + if (foundVersion == null) + throw new UpdateException("Unable to find engine version in manifest!"); - Log.Debug("Loading manifest from {manifestUrl}...", ConfigConstants.RobustBuildsManifest); - var manifest = - await _http.GetFromJsonAsync>( - ConfigConstants.RobustBuildsManifest, cancellationToken: cancel); + if (foundVersion.Info.Insecure) + throw new UpdateException("Specified engine version is insecure!"); - if (!manifest!.TryGetValue(engineVersion, out var versionInfo)) - { - throw new UpdateException("Unable to find engine version in manifest!"); - } + Log.Debug( + "Requested engine version was {RequestedEngien}, redirected to {FoundVersion}", + engineVersion, + foundVersion.Version); - if (versionInfo.Insecure) + if (_cfg.EngineInstallations.Lookup(foundVersion.Version).HasValue) { - throw new UpdateException("Specified engine version is insecure!"); + // Already have the engine version, we're good. + return new EngineInstallationResult(foundVersion.Version, false); } - var bestRid = RidUtility.FindBestRid(versionInfo.Platforms.Keys); + Log.Information("Installing engine version {version}...", foundVersion.Version); + + var bestRid = RidUtility.FindBestRid(foundVersion.Info.Platforms.Keys); if (bestRid == null) { throw new UpdateException("No engine version available for our platform!"); @@ -88,13 +113,13 @@ await _http.GetFromJsonAsync>( Log.Debug("Selecting RID {rid}", bestRid); - var buildInfo = versionInfo.Platforms[bestRid]; + var buildInfo = foundVersion.Info.Platforms[bestRid]; Log.Debug("Downloading engine: {EngineDownloadUrl}", buildInfo.Url); Helpers.EnsureDirectoryExists(LauncherPaths.DirEngineInstallations); - var downloadTarget = Path.Combine(LauncherPaths.DirEngineInstallations, $"{engineVersion}.zip"); + var downloadTarget = Path.Combine(LauncherPaths.DirEngineInstallations, $"{foundVersion.Version}.zip"); await using var file = File.Create(downloadTarget, 4096, FileOptions.Asynchronous); try @@ -110,9 +135,9 @@ await _http.GetFromJsonAsync>( throw; } - _cfg.AddEngineInstallation(new InstalledEngineVersion(engineVersion, buildInfo.Signature)); + _cfg.AddEngineInstallation(new InstalledEngineVersion(foundVersion.Version, buildInfo.Signature)); _cfg.CommitConfig(); - return true; + return new EngineInstallationResult(foundVersion.Version, true); } public async Task DownloadModuleIfNecessary( @@ -122,6 +147,16 @@ public async Task DownloadModuleIfNecessary( Helpers.DownloadProgressCallback? progress = null, CancellationToken cancel = default) { +#if DEVELOPMENT + if (_cfg.GetCVar(CVars.EngineOverrideEnabled)) + { + // For modules we have to extract them from the zip to disk first. + // So it's a little more involved than just giving a different zip path to the launch code. + await CopyOverrideModule(moduleName); + return true; + } +#endif + // Currently the module handling code assumes all modules need straight extract to disk. // This works for CEF, but who knows what the future might hold? @@ -151,16 +186,13 @@ public async Task DownloadModuleIfNecessary( Log.Debug("Downloading module: {EngineDownloadUrl}", platformData.Url); - var moduleDiskPath = Path.Combine(LauncherPaths.DirModuleInstallations, moduleName); - var moduleVersionDiskPath = Path.Combine(moduleDiskPath, moduleVersion); + GetModulePaths( + moduleName, + moduleVersion, + out var moduleDiskPath, + out var moduleVersionDiskPath); - await Task.Run(() => - { - // Avoid disk IO hang. - Helpers.EnsureDirectoryExists(moduleDiskPath); - Helpers.EnsureDirectoryExists(moduleVersionDiskPath); - Helpers.ClearDirectory(moduleVersionDiskPath); - }, CancellationToken.None); + await ClearModuleDir(moduleDiskPath, moduleVersionDiskPath); { await using var tempFile = TempFile.CreateTempFile(); @@ -193,18 +225,7 @@ await Task.Run(() => // CEF is so horrifically huge I'm enabling disk compression on it. Helpers.MarkDirectoryCompress(moduleVersionDiskPath); - Helpers.ExtractZipToDirectory(moduleVersionDiskPath, tempFile); - - // Chmod required files. - if (OperatingSystem.IsLinux()) - { - switch (moduleName) - { - case "Robust.Client.WebView": - Helpers.ChmodPlusX(Path.Combine(moduleVersionDiskPath, "Robust.Client.WebView")); - break; - } - } + ExtractModule(moduleName, moduleVersionDiskPath, tempFile); } _cfg.AddEngineModule(new InstalledEngineModule(moduleName, moduleVersion)); @@ -213,6 +234,61 @@ await Task.Run(() => Log.Debug("Done installing module!"); return true; + + } + + private async Task CopyOverrideModule(string name) + { + GetModulePaths( + name, + OverrideVersionName, + out var modPath, + out var modVersionPath); + + await ClearModuleDir(modPath, modVersionPath); + + var zipPath = FindOverrideZip(name, _cfg.GetCVar(CVars.EngineOverridePath)); + using var zip = File.OpenRead(zipPath); + + // Note: not marking directory as compressed since it would take a while to start. + ExtractModule(name, modVersionPath, zip); + } + + private static void GetModulePaths( + string module, + string version, + out string moduleDiskPath, + out string moduleVersionDiskPath) + { + moduleDiskPath = Path.Combine(LauncherPaths.DirModuleInstallations, module); + moduleVersionDiskPath = Path.Combine(moduleDiskPath, version); + } + + private static async Task ClearModuleDir(string modDiskPath, string modVersionDiskPath) + { + await Task.Run(() => + { + // Avoid disk IO hang. + Helpers.EnsureDirectoryExists(modDiskPath); + Helpers.EnsureDirectoryExists(modVersionDiskPath); + Helpers.ClearDirectory(modVersionDiskPath); + }, CancellationToken.None); + } + + private static void ExtractModule(string moduleName, string moduleVersionDiskPath, FileStream tempFile) + { + Helpers.ExtractZipToDirectory(moduleVersionDiskPath, tempFile); + + // Chmod required files. + if (OperatingSystem.IsLinux()) + { + switch (moduleName) + { + case "Robust.Client.WebView": + Helpers.ChmodPlusX(Path.Combine(moduleVersionDiskPath, "Robust.Client.WebView")); + break; + } + } } private static unsafe bool VerifyModuleSignature(FileStream stream, string signature) @@ -264,9 +340,25 @@ public async Task DoEngineCullMaybeAsync(SqliteConnection contenCon) // Cull main engine installations. - var modulesUsed = contenCon + var origModulesUsed = contenCon .Query<(string, string)>("SELECT DISTINCT ModuleName, ModuleVersion FROM ContentEngineDependency") - .ToHashSet(); + .ToList(); + + // GOD DAMNIT more bodging everything together. + // The code sucks. + // My shitty hacks to do engine version redirection fall apart here as well. + var modulesUsed = new HashSet<(string, string)>(); + foreach (var (name, version) in origModulesUsed) + { + if (name == "Robust" && await GetVersionInfo(version) is { } redirect) + { + modulesUsed.Add(("Robust", redirect.Version)); + } + else + { + modulesUsed.Add((name, version)); + } + } var toCull = _cfg.EngineInstallations.Items.Where(i => !modulesUsed.Contains(("Robust", i.Version))).ToArray(); @@ -321,26 +413,27 @@ public void ClearAllEngines() _cfg.CommitConfig(); } - private sealed class VersionInfo + private static string FindOverrideZip(string name, string dir) { - [JsonInclude] [JsonPropertyName("insecure")] -#pragma warning disable CS0649 - public bool Insecure; -#pragma warning restore CS0649 + var foundRids = new List(); - [JsonInclude] [JsonPropertyName("platforms")] - public Dictionary Platforms = default!; - } + var regex = new Regex(@$"^{Regex.Escape(name)}_([a-z\-\d]+)\.zip$"); + foreach (var item in Directory.EnumerateFiles(dir)) + { + var fileName = Path.GetFileName(item); + var match = regex.Match(fileName); + if (!match.Success) + continue; - private sealed class BuildInfo - { - [JsonInclude] [JsonPropertyName("url")] - public string Url = default!; + foundRids.Add(match.Groups[1].Value); + } - [JsonInclude] [JsonPropertyName("sha256")] - public string Sha256 = default!; + var rid = RidUtility.FindBestRid(foundRids); + if (rid == null) + throw new UpdateException($"Unable to find overriden {name} for current platform"); - [JsonInclude] [JsonPropertyName("sig")] - public string Signature = default!; + var path = Path.Combine(dir, $"{name}_{rid}.zip"); + Log.Warning("Using override for {Name}: {Path}", name, path); + return path; } } diff --git a/SS14.Launcher/Models/EngineManager/IEngineManager.cs b/SS14.Launcher/Models/EngineManager/IEngineManager.cs index 9202330..2de8250 100644 --- a/SS14.Launcher/Models/EngineManager/IEngineManager.cs +++ b/SS14.Launcher/Models/EngineManager/IEngineManager.cs @@ -19,8 +19,7 @@ public interface IEngineManager Task GetEngineModuleManifest(CancellationToken cancel = default); - /// True if something new had to be installed. - Task DownloadEngineIfNecessary( + Task DownloadEngineIfNecessary( string engineVersion, Helpers.DownloadProgressCallback? progress = null, CancellationToken cancel = default); @@ -57,6 +56,8 @@ static string ResolveEngineModuleVersion(EngineModuleManifest manifest, string m } } +public record struct EngineInstallationResult(string Version, bool Changed); + public sealed record EngineModuleManifest( Dictionary Modules ); diff --git a/SS14.Launcher/Models/Updater.cs b/SS14.Launcher/Models/Updater.cs index aa5193f..50d6b96 100644 --- a/SS14.Launcher/Models/Updater.cs +++ b/SS14.Launcher/Models/Updater.cs @@ -136,43 +136,7 @@ private async Task RunUpdate( await Task.Run(() => { CullOldContentVersions(con); }, CancellationToken.None); - (string, string)[] modules; - - { - Status = UpdateStatus.CheckingClientUpdate; - modules = con.Query<(string, string)>( - "SELECT ModuleName, moduleVersion FROM ContentEngineDependency WHERE VersionId = @Version", - new { Version = versionRowId }).ToArray(); - - foreach (var (name, version) in modules) - { - if (name == "Robust") - { - await InstallEngineVersionIfMissing(version, cancel); - } - else - { - Status = UpdateStatus.DownloadingEngineModules; - - var manifest = await moduleManifest.Value; - await _engineManager.DownloadModuleIfNecessary( - name, - version, - manifest, - DownloadProgressCallback, - cancel); - } - } - } - - Status = UpdateStatus.CullingEngine; - await CullEngineVersionsMaybe(con); - - Status = UpdateStatus.CommittingDownload; - _cfg.CommitConfig(); - - Log.Information("Update done!"); - return new ContentLaunchInfo(versionRowId, modules); + return await InstallEnginesForVersion(con, moduleManifest, versionRowId, cancel); } private async Task InstallContentBundle( @@ -304,19 +268,31 @@ FROM ContentManifest return versionId; }, CancellationToken.None); + return await InstallEnginesForVersion(con, moduleManifest, versionId, cancel); + } + + private async Task InstallEnginesForVersion( + SqliteConnection con, + Lazy> moduleManifest, + long versionRowId, + CancellationToken cancel) + { (string, string)[] modules; { Status = UpdateStatus.CheckingClientUpdate; modules = con.Query<(string, string)>( "SELECT ModuleName, moduleVersion FROM ContentEngineDependency WHERE VersionId = @Version", - new { Version = versionId }).ToArray(); + new { Version = versionRowId }).ToArray(); - foreach (var (name, version) in modules) + for (var index = 0; index < modules.Length; index++) { + var (name, version) = modules[index]; if (name == "Robust") { - await InstallEngineVersionIfMissing(version, cancel); + // Engine version may change here due to manifest version redirects. + var newEngineVersion = await InstallEngineVersionIfMissing(version, cancel); + modules[index] = (name, newEngineVersion); } else { @@ -340,7 +316,7 @@ await _engineManager.DownloadModuleIfNecessary( _cfg.CommitConfig(); Log.Information("Update done!"); - return new ContentLaunchInfo(versionId, modules); + return new ContentLaunchInfo(versionRowId, modules); } @@ -1421,13 +1397,13 @@ private async Task CullEngineVersionsMaybe(SqliteConnection contentConnection) await _engineManager.DoEngineCullMaybeAsync(contentConnection); } - private async Task InstallEngineVersionIfMissing(string engineVer, CancellationToken cancel) + private async Task InstallEngineVersionIfMissing(string engineVer, CancellationToken cancel) { Status = UpdateStatus.DownloadingEngineVersion; - var change = await _engineManager.DownloadEngineIfNecessary(engineVer, DownloadProgressCallback, cancel); + var (changedVersion, _) = await _engineManager.DownloadEngineIfNecessary(engineVer, DownloadProgressCallback, cancel); Progress = null; - return change; + return changedVersion; } private void DownloadProgressCallback(long downloaded, long total)