Merge pull request #13 from CloudCannon/feat/minv

Version tagging, failure screenshots, option clicking

.github/workflows/release.yml

@@ -330,6 +330,17 @@ jobs:
           default: true
           components: rustfmt, clippy
 
+      - name: Ubuntu AppArmor fix
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        # Ubuntu >= 23 has AppArmor enabled by default, which breaks Chrome.
+        # See https://github.com/puppeteer/puppeteer/issues/12818 "No usable sandbox!"
+        # this is taken from the solution used in Puppeteer's CI: https://github.com/puppeteer/puppeteer/pull/13196
+        # The alternative is to pin Ubuntu 22 or to use aa-exec to disable AppArmor for commands that need Puppeteer.
+        # This is also suggested by Chromium https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md
+        run: |
+          echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns
+        shell: bash
+
       - name: Prepare Git
         run: |
           git config user.email "[email protected]"

.github/workflows/test.yml

@@ -52,6 +52,17 @@ jobs:
           default: true
           components: rustfmt, clippy
 
+      - name: Ubuntu AppArmor fix
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        # Ubuntu >= 23 has AppArmor enabled by default, which breaks Chrome.
+        # See https://github.com/puppeteer/puppeteer/issues/12818 "No usable sandbox!"
+        # this is taken from the solution used in Puppeteer's CI: https://github.com/puppeteer/puppeteer/pull/13196
+        # The alternative is to pin Ubuntu 22 or to use aa-exec to disable AppArmor for commands that need Puppeteer.
+        # This is also suggested by Chromium https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md
+        run: |
+          echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns
+        shell: bash
+
       - name: Build Lib
         working-directory: ./toolproof
         run: cargo build

CHANGELOG.md

@@ -9,6 +9,10 @@
 
 ## Unreleased
 
+* Allow the generic "I click" action to click `option` elements, and elements with a `role="option"` attribute
+* Add a `supported_versions` configuration option to ensure Toolproof isn't running a version older than your tests support
+* Add a `failure_screenshot_location` configuration option to enable Toolproof to automatically screenshot the browser on test failure
+
 ## v0.10.2 (December 18, 2024)
 
 * Allow the generic "I click" action to click elements with a `role="button"` attribute

docs/content/docs/installation.md

@@ -7,6 +7,17 @@ weight: 5
 
 Toolproof is a static binary with no dynamic dependencies, so in most cases will be simple to install and run. Toolproof is currently supported on Windows, macOS, and Linux distributions.
 
+## Ensuring Toolproof is running a supported version
+
+For all installation methods, your Toolproof configuration can specify the supported Toolproof versions.
+
+```yml
+# In toolproof.yml
+supported_versions: ">=0.10.3"
+```
+
+This can also be set in a `TOOLPROOF_SUPPORTED_VERSIONS` environment variable.
+
 ## Running via npx
 
 ```bash

toolproof/Cargo.toml

@@ -33,6 +33,8 @@ schematic = { version = "0.12.0", features = ["yaml"] }
 strip-ansi-escapes = "0.2.0"
 path-slash = "0.2.1"
 normalize-path = "0.2.1"
+miette = { version = "7", features = ["fancy"] }
+semver = "1.0.25"
 
 [profile.dev.package.similar]
 opt-level = 3

toolproof/src/definitions/browser/mod.rs

@@ -280,7 +280,9 @@ impl BrowserWindow {
                     el_xpath("a"),
                     el_xpath("button"),
                     el_xpath("input"),
+                    el_xpath("option"),
                     el_xpath("*[@role='button']"),
+                    el_xpath("*[@role='option']"),
                 ]
                 .join(" | ");
 
@@ -645,7 +647,7 @@ mod eval_js {
     }
 }
 
-mod screenshots {
+pub mod screenshots {
     use crate::errors::{ToolproofInternalError, ToolproofTestFailure};
 
     use super::*;

toolproof/src/main.rs

@@ -1,16 +1,19 @@
 use std::collections::BTreeMap;
 use std::fmt::Display;
+use std::path::PathBuf;
 use std::process::Stdio;
 use std::sync::Arc;
 use std::time::Duration;
 use std::{collections::HashMap, time::Instant};
 
 use console::{style, Term};
 use futures::future::join_all;
+use miette::IntoDiagnostic;
 use normalize_path::NormalizePath;
 use parser::{parse_macro, ToolproofFileType, ToolproofPlatform};
 use schematic::color::owo::OwoColorize;
 use segments::ToolproofSegments;
+use semver::{Version, VersionReq};
 use similar_string::compare_similarity;
 use tokio::fs::read_to_string;
 use tokio::process::Command;
@@ -52,6 +55,7 @@ pub struct ToolproofTestFile {
     pub original_source: String,
     pub file_path: String,
     pub file_directory: String,
+    pub failure_screenshot: Option<PathBuf>,
 }
 
 #[derive(Debug, Clone)]
@@ -204,6 +208,22 @@ fn closest_strings<'o>(target: &String, options: &'o Vec<String>) -> Vec<(&'o St
 async fn main_inner() -> Result<(), ()> {
     let ctx = configure();
 
+    if let Some(versions) = &ctx.params.supported_versions {
+        let req = VersionReq::parse(versions).into_diagnostic().map_err(|e| {
+            eprintln!("Failed to parse supported versions: {e:?}");
+        })?;
+        let active = Version::parse(&ctx.version).expect("Crate version should be valid");
+        let is_local = ctx.version == "0.0.0";
+
+        if !req.matches(&active) && !is_local {
+            eprintln!(
+                "Toolproof is running version {}, but your configuration requires Toolproof {}",
+                ctx.version, versions
+            );
+            return Err(());
+        }
+    }
+
     if ctx.params.skip_hooks {
         println!("{}", "Skipping before_all commands".yellow().bold());
     } else {
@@ -661,6 +681,16 @@ async fn main_inner() -> Result<(), ()> {
                         log_err();
                     }
                 }
+
+                if let Some(failure_screenshot) = &file.failure_screenshot {
+                    println!("{}", "--- FAILURE SCREENSHOT ---".on_yellow().bold());
+                    println!(
+                        "{} {}",
+                        "Browser state at failure was screenshot to".red(),
+                        failure_screenshot.to_string_lossy().cyan().bold()
+                    );
+                }
+
                 Err(HoldingError::TestFailure)
             }
         }

toolproof/src/options.rs

@@ -1,6 +1,7 @@
 use clap::{
     arg, builder::PossibleValuesParser, command, value_parser, Arg, ArgAction, ArgMatches, Command,
 };
+use miette::IntoDiagnostic;
 use schematic::{derive_enum, Config, ConfigEnum, ConfigLoader};
 use serde::{Deserialize, Serialize};
 use std::{collections::HashMap, env, path::PathBuf};
@@ -31,15 +32,15 @@ pub fn configure() -> ToolproofContext {
 
     let mut loader = ConfigLoader::<ToolproofParams>::new();
     for config in configs {
-        if let Err(e) = loader.file(config) {
-            eprintln!("Failed to load {config}:\n{e}");
+        if let Err(e) = loader.file(config).into_diagnostic() {
+            eprintln!("Failed to load {config}:\n{e:?}");
             std::process::exit(1);
         }
     }
 
-    match loader.load() {
+    match loader.load().into_diagnostic() {
         Err(e) => {
-            eprintln!("Failed to initialize configuration: {e}");
+            eprintln!("Failed to initialize configuration: {e:?}");
             std::process::exit(1);
         }
         Ok(mut result) => {
@@ -139,6 +140,13 @@ fn get_cli_matches() -> ArgMatches {
             .required(false)
             .value_parser(PossibleValuesParser::new(["chrome", "pagebrowse"])),
         )
+        .arg(
+            arg!(
+                --"failure-screenshot-location" <DIR> "If set, Toolproof will screenshot the browser to this location when a test fails (if applicable)"
+            )
+            .required(false)
+            .value_parser(value_parser!(PathBuf)),
+        )
         .get_matches()
 }
 
@@ -214,6 +222,14 @@ pub struct ToolproofParams {
     /// Skip running any of the before_all hooks
     #[setting(env = "TOOLPROOF_SKIPHOOKS")]
     pub skip_hooks: bool,
+
+    /// Error if Toolproof is below this version
+    #[setting(env = "TOOLPROOF_SUPPORTED_VERSIONS")]
+    pub supported_versions: Option<String>,
+
+    /// If set, Toolproof will screenshot the browser to this location when a test fails (if applicable)
+    #[setting(env = "TOOLPROOF_FAILURE_SCREENSHOT_LOCATION")]
+    pub failure_screenshot_location: Option<PathBuf>,
 }
 
 // The configuration object used internally
@@ -297,5 +313,11 @@ impl ToolproofParams {
                 self.placeholders.insert(key.into(), value.into());
             }
         }
+
+        if let Some(failure_screenshot_location) =
+            cli_matches.get_one::<PathBuf>("failure-screenshot-location")
+        {
+            self.failure_screenshot_location = Some(failure_screenshot_location.clone());
+        }
     }
 }

toolproof/src/parser.rs

@@ -105,6 +105,7 @@ impl TryFrom<ToolproofTestInput> for ToolproofTestFile {
             original_source: value.original_source,
             file_path: value.file_path,
             file_directory: value.file_directory,
+            failure_screenshot: None,
         })
     }
 }

toolproof/src/runner.rs

@@ -2,14 +2,19 @@ use async_recursion::async_recursion;
 use futures::FutureExt;
 use normalize_path::NormalizePath;
 use similar_string::find_best_similarity;
-use std::{collections::HashMap, path::PathBuf, sync::Arc};
+use std::{
+    collections::HashMap,
+    path::PathBuf,
+    sync::Arc,
+    time::{SystemTime, UNIX_EPOCH},
+};
 use tokio::time::{self, Duration};
 
 use console::style;
 
 use crate::{
     civilization::Civilization,
-    definitions::ToolproofInstruction,
+    definitions::{browser::screenshots::ScreenshotViewport, ToolproofInstruction},
     errors::{ToolproofInputError, ToolproofStepError, ToolproofTestError, ToolproofTestFailure},
     platforms::platform_matches,
     segments::SegmentArgs,
@@ -38,6 +43,38 @@ pub async fn run_toolproof_experiment(
 
     let res = run_toolproof_steps(&input.file_directory, &mut input.steps, &mut civ, None).await;
 
+    if res.is_err() && civ.window.is_some() {
+        if let Some(screenshot_target) = &civ.universe.ctx.params.failure_screenshot_location {
+            let instruction = ScreenshotViewport {};
+            let filename = format!(
+                "{}-{}.webp",
+                SystemTime::now()
+                    .duration_since(UNIX_EPOCH)
+                    .expect("Toolproof should be running after the UNIX EPOCH")
+                    .as_secs(),
+                input.file_path.replace(|c: char| !c.is_alphanumeric(), "-")
+            );
+            let abs_acreenshot_target = civ.universe.ctx.working_directory.join(screenshot_target);
+            let filepath = abs_acreenshot_target.join(filename);
+            if instruction
+                .run(
+                    &SegmentArgs::build_synthetic(
+                        [(
+                            "filepath".to_string(),
+                            &serde_json::Value::String(filepath.to_string_lossy().to_string()),
+                        )]
+                        .into(),
+                    ),
+                    &mut civ,
+                )
+                .await
+                .is_ok()
+            {
+                input.failure_screenshot = Some(filepath)
+            }
+        }
+    }
+
     civ.shutdown().await;
 
     res

toolproof/src/segments.rs

@@ -102,6 +102,14 @@ pub struct SegmentArgs<'a> {
 }
 
 impl<'a> SegmentArgs<'a> {
+    pub fn build_synthetic(args: HashMap<String, &'a serde_json::Value>) -> Self {
+        Self {
+            args,
+            placeholder_delim: "INTENTIONALLY_UNSET".to_string(),
+            placeholders: HashMap::new(),
+        }
+    }
+
     pub fn build(
         reference_instruction: &ToolproofSegments,
         supplied_instruction: &'a ToolproofSegments,