From 71194c2a5dab84ab81c01d98de3618385ff19663 Mon Sep 17 00:00:00 2001 From: Liam Bigelow <40188355+bglw@users.noreply.github.com> Date: Fri, 24 Jan 2025 13:03:32 +1300 Subject: [PATCH 1/5] Add config option `supported_versions` --- docs/content/docs/installation.md | 11 +++++++++++ toolproof/Cargo.toml | 2 ++ toolproof/src/main.rs | 18 ++++++++++++++++++ toolproof/src/options.rs | 13 +++++++++---- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/docs/content/docs/installation.md b/docs/content/docs/installation.md index ebccf27..35611de 100644 --- a/docs/content/docs/installation.md +++ b/docs/content/docs/installation.md @@ -7,6 +7,17 @@ weight: 5 Toolproof is a static binary with no dynamic dependencies, so in most cases will be simple to install and run. Toolproof is currently supported on Windows, macOS, and Linux distributions. +## Ensuring Toolproof is running a supported version + +For all installation methods, your Toolproof configuration can specify the supported Toolproof versions. + +```yml +# In toolproof.yml +supported_versions: ">=0.10.3" +``` + +This can also be set in a `TOOLPROOF_SUPPORTED_VERSIONS` environment variable. + ## Running via npx ```bash diff --git a/toolproof/Cargo.toml b/toolproof/Cargo.toml index 6881589..07d2943 100644 --- a/toolproof/Cargo.toml +++ b/toolproof/Cargo.toml @@ -33,6 +33,8 @@ schematic = { version = "0.12.0", features = ["yaml"] } strip-ansi-escapes = "0.2.0" path-slash = "0.2.1" normalize-path = "0.2.1" +miette = { version = "7", features = ["fancy"] } +semver = "1.0.25" [profile.dev.package.similar] opt-level = 3 diff --git a/toolproof/src/main.rs b/toolproof/src/main.rs index 44ce420..6e5de83 100644 --- a/toolproof/src/main.rs +++ b/toolproof/src/main.rs @@ -7,10 +7,12 @@ use std::{collections::HashMap, time::Instant}; use console::{style, Term}; use futures::future::join_all; +use miette::IntoDiagnostic; use normalize_path::NormalizePath; use parser::{parse_macro, ToolproofFileType, ToolproofPlatform}; use schematic::color::owo::OwoColorize; use segments::ToolproofSegments; +use semver::{Version, VersionReq}; use similar_string::compare_similarity; use tokio::fs::read_to_string; use tokio::process::Command; @@ -204,6 +206,22 @@ fn closest_strings<'o>(target: &String, options: &'o Vec) -> Vec<(&'o St async fn main_inner() -> Result<(), ()> { let ctx = configure(); + if let Some(versions) = &ctx.params.supported_versions { + let req = VersionReq::parse(versions).into_diagnostic().map_err(|e| { + eprintln!("Failed to parse supported versions: {e:?}"); + })?; + let active = Version::parse(&ctx.version).expect("Crate version should be valid"); + let is_local = ctx.version == "0.0.0"; + + if !req.matches(&active) && !is_local { + eprintln!( + "Toolproof is running version {}, but your configuration requires Toolproof {}", + ctx.version, versions + ); + return Err(()); + } + } + if ctx.params.skip_hooks { println!("{}", "Skipping before_all commands".yellow().bold()); } else { diff --git a/toolproof/src/options.rs b/toolproof/src/options.rs index 3cc8aca..60ef507 100644 --- a/toolproof/src/options.rs +++ b/toolproof/src/options.rs @@ -1,6 +1,7 @@ use clap::{ arg, builder::PossibleValuesParser, command, value_parser, Arg, ArgAction, ArgMatches, Command, }; +use miette::IntoDiagnostic; use schematic::{derive_enum, Config, ConfigEnum, ConfigLoader}; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, env, path::PathBuf}; @@ -31,15 +32,15 @@ pub fn configure() -> ToolproofContext { let mut loader = ConfigLoader::::new(); for config in configs { - if let Err(e) = loader.file(config) { - eprintln!("Failed to load {config}:\n{e}"); + if let Err(e) = loader.file(config).into_diagnostic() { + eprintln!("Failed to load {config}:\n{e:?}"); std::process::exit(1); } } - match loader.load() { + match loader.load().into_diagnostic() { Err(e) => { - eprintln!("Failed to initialize configuration: {e}"); + eprintln!("Failed to initialize configuration: {e:?}"); std::process::exit(1); } Ok(mut result) => { @@ -214,6 +215,10 @@ pub struct ToolproofParams { /// Skip running any of the before_all hooks #[setting(env = "TOOLPROOF_SKIPHOOKS")] pub skip_hooks: bool, + + /// Error if Toolproof is below this version + #[setting(env = "TOOLPROOF_SUPPORTED_VERSIONS")] + pub supported_versions: Option, } // The configuration object used internally From 1119e8e7460507c1fd1dbefb81f9aedb710a8870 Mon Sep 17 00:00:00 2001 From: Liam Bigelow <40188355+bglw@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:08:57 +1300 Subject: [PATCH 2/5] Add option to screenshot the browser on any test failure --- toolproof/src/definitions/browser/mod.rs | 2 +- toolproof/src/main.rs | 12 +++++++ toolproof/src/options.rs | 17 ++++++++++ toolproof/src/parser.rs | 1 + toolproof/src/runner.rs | 41 ++++++++++++++++++++++-- toolproof/src/segments.rs | 8 +++++ 6 files changed, 78 insertions(+), 3 deletions(-) diff --git a/toolproof/src/definitions/browser/mod.rs b/toolproof/src/definitions/browser/mod.rs index 976e8af..9d61c6a 100644 --- a/toolproof/src/definitions/browser/mod.rs +++ b/toolproof/src/definitions/browser/mod.rs @@ -645,7 +645,7 @@ mod eval_js { } } -mod screenshots { +pub mod screenshots { use crate::errors::{ToolproofInternalError, ToolproofTestFailure}; use super::*; diff --git a/toolproof/src/main.rs b/toolproof/src/main.rs index 6e5de83..d353e06 100644 --- a/toolproof/src/main.rs +++ b/toolproof/src/main.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; use std::fmt::Display; +use std::path::PathBuf; use std::process::Stdio; use std::sync::Arc; use std::time::Duration; @@ -54,6 +55,7 @@ pub struct ToolproofTestFile { pub original_source: String, pub file_path: String, pub file_directory: String, + pub failure_screenshot: Option, } #[derive(Debug, Clone)] @@ -679,6 +681,16 @@ async fn main_inner() -> Result<(), ()> { log_err(); } } + + if let Some(failure_screenshot) = &file.failure_screenshot { + println!("{}", "--- FAILURE SCREENSHOT ---".on_yellow().bold()); + println!( + "{} {}", + "Browser state at failure was screenshot to".red(), + failure_screenshot.to_string_lossy().cyan().bold() + ); + } + Err(HoldingError::TestFailure) } } diff --git a/toolproof/src/options.rs b/toolproof/src/options.rs index 60ef507..ec15065 100644 --- a/toolproof/src/options.rs +++ b/toolproof/src/options.rs @@ -140,6 +140,13 @@ fn get_cli_matches() -> ArgMatches { .required(false) .value_parser(PossibleValuesParser::new(["chrome", "pagebrowse"])), ) + .arg( + arg!( + --"failure-screenshot-location" "If set, Toolproof will screenshot the browser to this location when a test fails (if applicable)" + ) + .required(false) + .value_parser(value_parser!(PathBuf)), + ) .get_matches() } @@ -219,6 +226,10 @@ pub struct ToolproofParams { /// Error if Toolproof is below this version #[setting(env = "TOOLPROOF_SUPPORTED_VERSIONS")] pub supported_versions: Option, + + /// If set, Toolproof will screenshot the browser to this location when a test fails (if applicable) + #[setting(env = "TOOLPROOF_FAILURE_SCREENSHOT_LOCATION")] + pub failure_screenshot_location: Option, } // The configuration object used internally @@ -302,5 +313,11 @@ impl ToolproofParams { self.placeholders.insert(key.into(), value.into()); } } + + if let Some(failure_screenshot_location) = + cli_matches.get_one::("failure-screenshot-location") + { + self.failure_screenshot_location = Some(failure_screenshot_location.clone()); + } } } diff --git a/toolproof/src/parser.rs b/toolproof/src/parser.rs index c8922e8..dd6e985 100644 --- a/toolproof/src/parser.rs +++ b/toolproof/src/parser.rs @@ -105,6 +105,7 @@ impl TryFrom for ToolproofTestFile { original_source: value.original_source, file_path: value.file_path, file_directory: value.file_directory, + failure_screenshot: None, }) } } diff --git a/toolproof/src/runner.rs b/toolproof/src/runner.rs index a7a900d..c8c0b45 100644 --- a/toolproof/src/runner.rs +++ b/toolproof/src/runner.rs @@ -2,14 +2,19 @@ use async_recursion::async_recursion; use futures::FutureExt; use normalize_path::NormalizePath; use similar_string::find_best_similarity; -use std::{collections::HashMap, path::PathBuf, sync::Arc}; +use std::{ + collections::HashMap, + path::PathBuf, + sync::Arc, + time::{SystemTime, UNIX_EPOCH}, +}; use tokio::time::{self, Duration}; use console::style; use crate::{ civilization::Civilization, - definitions::ToolproofInstruction, + definitions::{browser::screenshots::ScreenshotViewport, ToolproofInstruction}, errors::{ToolproofInputError, ToolproofStepError, ToolproofTestError, ToolproofTestFailure}, platforms::platform_matches, segments::SegmentArgs, @@ -38,6 +43,38 @@ pub async fn run_toolproof_experiment( let res = run_toolproof_steps(&input.file_directory, &mut input.steps, &mut civ, None).await; + if res.is_err() && civ.window.is_some() { + if let Some(screenshot_target) = &civ.universe.ctx.params.failure_screenshot_location { + let instruction = ScreenshotViewport {}; + let filename = format!( + "{}-{}.webp", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Toolproof should be running after the UNIX EPOCH") + .as_secs(), + input.file_path.replace(|c: char| !c.is_alphanumeric(), "-") + ); + let abs_acreenshot_target = civ.universe.ctx.working_directory.join(screenshot_target); + let filepath = abs_acreenshot_target.join(filename); + if instruction + .run( + &SegmentArgs::build_synthetic( + [( + "filepath".to_string(), + &serde_json::Value::String(filepath.to_string_lossy().to_string()), + )] + .into(), + ), + &mut civ, + ) + .await + .is_ok() + { + input.failure_screenshot = Some(filepath) + } + } + } + civ.shutdown().await; res diff --git a/toolproof/src/segments.rs b/toolproof/src/segments.rs index 08149ff..4ea2624 100644 --- a/toolproof/src/segments.rs +++ b/toolproof/src/segments.rs @@ -102,6 +102,14 @@ pub struct SegmentArgs<'a> { } impl<'a> SegmentArgs<'a> { + pub fn build_synthetic(args: HashMap) -> Self { + Self { + args, + placeholder_delim: "INTENTIONALLY_UNSET".to_string(), + placeholders: HashMap::new(), + } + } + pub fn build( reference_instruction: &ToolproofSegments, supplied_instruction: &'a ToolproofSegments, From 7943f8aa02a2812517872a95293b39ac6447f3f6 Mon Sep 17 00:00:00 2001 From: Liam Bigelow <40188355+bglw@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:10:19 +1300 Subject: [PATCH 3/5] Add option elements to the clickable set of elements --- toolproof/src/definitions/browser/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/toolproof/src/definitions/browser/mod.rs b/toolproof/src/definitions/browser/mod.rs index 9d61c6a..8709421 100644 --- a/toolproof/src/definitions/browser/mod.rs +++ b/toolproof/src/definitions/browser/mod.rs @@ -280,7 +280,9 @@ impl BrowserWindow { el_xpath("a"), el_xpath("button"), el_xpath("input"), + el_xpath("option"), el_xpath("*[@role='button']"), + el_xpath("*[@role='option']"), ] .join(" | "); From 157caf3bd624bf9ff4ef7b46a3af8d8c079acc7a Mon Sep 17 00:00:00 2001 From: Liam Bigelow <40188355+bglw@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:13:01 +1300 Subject: [PATCH 4/5] Changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 496deb1..35bf8df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ ## Unreleased +* Allow the generic "I click" action to click `option` elements, and elements with a `role="option"` attribute +* Add a `supported_versions` configuration option to ensure Toolproof isn't running a version older than your tests support +* Add a `failure_screenshot_location` configuration option to enable Toolproof to automatically screenshot the browser on test failure + ## v0.10.2 (December 18, 2024) * Allow the generic "I click" action to click elements with a `role="button"` attribute From e61c4aed1baaea6ed85a6a49ab008e3a3ab16d1e Mon Sep 17 00:00:00 2001 From: Liam Bigelow <40188355+bglw@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:32:25 +1300 Subject: [PATCH 5/5] Add ubuntu patch --- .github/workflows/release.yml | 11 +++++++++++ .github/workflows/test.yml | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5d7a0c9..b5e4b6c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -330,6 +330,17 @@ jobs: default: true components: rustfmt, clippy + - name: Ubuntu AppArmor fix + if: ${{ matrix.os == 'ubuntu-latest' }} + # Ubuntu >= 23 has AppArmor enabled by default, which breaks Chrome. + # See https://github.com/puppeteer/puppeteer/issues/12818 "No usable sandbox!" + # this is taken from the solution used in Puppeteer's CI: https://github.com/puppeteer/puppeteer/pull/13196 + # The alternative is to pin Ubuntu 22 or to use aa-exec to disable AppArmor for commands that need Puppeteer. + # This is also suggested by Chromium https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md + run: | + echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns + shell: bash + - name: Prepare Git run: | git config user.email "github@github.com" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 666f9cc..347ba32 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,6 +52,17 @@ jobs: default: true components: rustfmt, clippy + - name: Ubuntu AppArmor fix + if: ${{ matrix.os == 'ubuntu-latest' }} + # Ubuntu >= 23 has AppArmor enabled by default, which breaks Chrome. + # See https://github.com/puppeteer/puppeteer/issues/12818 "No usable sandbox!" + # this is taken from the solution used in Puppeteer's CI: https://github.com/puppeteer/puppeteer/pull/13196 + # The alternative is to pin Ubuntu 22 or to use aa-exec to disable AppArmor for commands that need Puppeteer. + # This is also suggested by Chromium https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md + run: | + echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns + shell: bash + - name: Build Lib working-directory: ./toolproof run: cargo build