diff --git a/toolproof/Cargo.toml b/toolproof/Cargo.toml index 76f7910..943baaa 100644 --- a/toolproof/Cargo.toml +++ b/toolproof/Cargo.toml @@ -25,6 +25,7 @@ console = "0.15" dialoguer = { version = "0.11", features = ["fuzzy-select"] } async-trait = "0.1.78" pagebrowse = "0.1.0" +chromiumoxide = "0.5" clap = { version = "4", features = ["cargo"] } schematic = { version = "0.12.0", features = ["yaml"] } strip-ansi-escapes = "0.2.0" diff --git a/toolproof/src/civilization.rs b/toolproof/src/civilization.rs index cfe3693..ed884e3 100644 --- a/toolproof/src/civilization.rs +++ b/toolproof/src/civilization.rs @@ -3,19 +3,21 @@ use std::{ fs, io::{Read, Write}, path::PathBuf, - process::{Command, ExitStatus}, + process::{ExitStatus, Stdio}, str::from_utf8, sync::Arc, + time::Duration, }; use actix_web::dev::ServerHandle; -use pagebrowse::{Pagebrowser, PagebrowserWindow}; use portpicker::pick_unused_port; use tempfile::tempdir; -use tokio::task::JoinHandle; +use tokio::{process::Command, task::JoinHandle}; use wax::Glob; -use crate::{errors::ToolproofTestFailure, options::ToolproofParams, universe::Universe}; +use crate::{ + definitions::browser::BrowserWindow, errors::ToolproofTestFailure, universe::Universe, +}; #[derive(Debug)] pub struct CommandOutput { @@ -27,7 +29,7 @@ pub struct Civilization<'u> { pub tmp_dir: Option, pub last_command_output: Option, pub assigned_server_port: Option, - pub window: Option, + pub window: Option, pub threads: Vec>>, pub handles: Vec, pub env_vars: HashMap, @@ -150,7 +152,7 @@ impl<'u> Civilization<'u> { self.env_vars.insert(name, value); } - pub fn run_command(&mut self, cmd: String) -> Result { + pub async fn run_command(&mut self, cmd: String) -> Result { let mut command = Command::new("sh"); command .arg("-c") @@ -161,7 +163,25 @@ impl<'u> Civilization<'u> { command.env(key, val); } - let Ok(output) = command.output() else { + command.stdout(Stdio::piped()); + command.stderr(Stdio::piped()); + let running = command.spawn().map_err(|_| ToolproofTestFailure::Custom { + msg: format!("Failed to run command: {cmd}"), + })?; + + let Ok(output) = (match tokio::time::timeout( + Duration::from_secs(30), + running.wait_with_output(), + ) + .await + { + Ok(out) => out, + Err(_) => { + return Err(ToolproofTestFailure::Custom { + msg: format!("Failed to run command due to timeout: {cmd}"), + }); + } + }) else { return Err(ToolproofTestFailure::Custom { msg: format!("Failed to run command: {cmd}"), }); diff --git a/toolproof/src/definitions/browser/mod.rs b/toolproof/src/definitions/browser/mod.rs index b01154b..7a3a575 100644 --- a/toolproof/src/definitions/browser/mod.rs +++ b/toolproof/src/definitions/browser/mod.rs @@ -1,15 +1,138 @@ -use std::collections::HashMap; +use std::sync::Arc; use async_trait::async_trait; +use chromiumoxide::cdp::browser_protocol::target::CreateTargetParams; +use futures::StreamExt; +use tokio::task::JoinHandle; use crate::civilization::Civilization; use crate::errors::{ToolproofInputError, ToolproofStepError}; +use crate::options::ToolproofParams; use super::{SegmentArgs, ToolproofInstruction, ToolproofRetriever}; +use chromiumoxide::browser::{Browser, BrowserConfig}; use pagebrowse::{PagebrowseBuilder, Pagebrowser, PagebrowserWindow}; const HARNESS: &'static str = include_str!("./harness.js"); +const INIT_SCRIPT: &'static str = include_str!("./init.js"); + +fn harnessed(js: String) -> String { + HARNESS.replace("// insert_toolproof_inner_js", &js) +} + +pub enum BrowserTester { + Pagebrowse(Arc), + Chrome { + browser: Arc, + event_thread: Arc>>, + }, +} + +impl BrowserTester { + async fn initialize(params: &ToolproofParams) -> Self { + match params.browser { + crate::options::ToolproofBrowserImpl::Chrome => { + let (browser, mut handler) = + Browser::launch(BrowserConfig::builder().build().unwrap()) + .await + .unwrap(); + + BrowserTester::Chrome { + browser: Arc::new(browser), + event_thread: Arc::new(tokio::task::spawn(async move { + loop { + let _ = handler.next().await.unwrap(); + } + })), + } + } + crate::options::ToolproofBrowserImpl::Pagebrowse => { + let pagebrowser = PagebrowseBuilder::new(params.concurrency) + .visible(false) + .manager_path(format!( + "{}/../bin/pagebrowse_manager", + env!("CARGO_MANIFEST_DIR") + )) + .init_script(INIT_SCRIPT.to_string()) + .build() + .await + .expect("Can't build the pagebrowser"); + + BrowserTester::Pagebrowse(Arc::new(pagebrowser)) + } + } + } + + async fn get_window(&self) -> BrowserWindow { + match self { + BrowserTester::Pagebrowse(pb) => { + BrowserWindow::Pagebrowse(pb.get_window().await.unwrap()) + } + BrowserTester::Chrome { browser, .. } => { + let page = browser + .new_page(CreateTargetParams { + url: "about:blank".to_string(), + width: None, + height: None, + browser_context_id: None, + enable_begin_frame_control: None, + new_window: None, + background: None, + }) + .await + .unwrap(); + page.evaluate_on_new_document(INIT_SCRIPT.to_string()) + .await + .expect("Could not set initialization js"); + BrowserWindow::Chrome(page) + } + } + } +} + +pub enum BrowserWindow { + Chrome(chromiumoxide::Page), + Pagebrowse(PagebrowserWindow), +} + +impl BrowserWindow { + async fn navigate(&self, url: String, wait_for_load: bool) -> Result<(), ToolproofStepError> { + match self { + BrowserWindow::Chrome(page) => { + // TODO: This is implicitly always wait_for_load: true + page.goto(url) + .await + .map(|_| ()) + .map_err(|inner| ToolproofStepError::Internal(inner.into())) + } + BrowserWindow::Pagebrowse(window) => window + .navigate(url, wait_for_load) + .await + .map_err(|inner| ToolproofStepError::Internal(inner.into())), + } + } + + async fn evaluate_script( + &self, + script: String, + ) -> Result, ToolproofStepError> { + match self { + BrowserWindow::Chrome(page) => { + let res = page + .evaluate_function(format!("async function() {{{}}}", harnessed(script))) + .await + .map_err(|inner| ToolproofStepError::Internal(inner.into()))?; + + Ok(res.object().value.clone()) + } + BrowserWindow::Pagebrowse(window) => window + .evaluate_script(harnessed(script)) + .await + .map_err(|inner| ToolproofStepError::Internal(inner.into())), + } + } +} mod load_page { use super::*; @@ -37,12 +160,15 @@ mod load_page { args.get_string("url")? ); - let window = civ.universe.pagebrowser.get_window().await.unwrap(); + let browser = civ + .universe + .browser + .get_or_init(|| async { BrowserTester::initialize(&civ.universe.ctx.params).await }) + .await; - window - .navigate(url.to_string(), true) - .await - .map_err(|inner| ToolproofStepError::Internal(inner.into()))?; + let window = browser.get_window().await; + + window.navigate(url.to_string(), true).await?; civ.window = Some(window); @@ -61,10 +187,6 @@ mod eval_js { use super::*; - fn harnessed(js: String) -> String { - HARNESS.replace("// insert_toolproof_inner_js", &js) - } - async fn eval_and_return_js( js: String, civ: &mut Civilization<'_>, @@ -77,10 +199,7 @@ mod eval_js { )); }; - let value = window - .evaluate_script(harnessed(js)) - .await - .map_err(|inner| ToolproofStepError::Internal(inner.into()))?; + let value = window.evaluate_script(js).await?; let Some(serde_json::Value::Object(map)) = &value else { return Err(ToolproofStepError::External( diff --git a/toolproof/src/definitions/mod.rs b/toolproof/src/definitions/mod.rs index 75756f4..208b80d 100644 --- a/toolproof/src/definitions/mod.rs +++ b/toolproof/src/definitions/mod.rs @@ -11,7 +11,7 @@ use crate::{ }; mod assertions; -mod browser; +pub mod browser; mod filesystem; mod hosting; mod process; diff --git a/toolproof/src/definitions/process/mod.rs b/toolproof/src/definitions/process/mod.rs index 33cea17..5ec228c 100644 --- a/toolproof/src/definitions/process/mod.rs +++ b/toolproof/src/definitions/process/mod.rs @@ -61,7 +61,7 @@ mod run { ) -> Result<(), ToolproofStepError> { let command = args.get_string("command")?; - let exit_status = civ.run_command(command.to_string())?; + let exit_status = civ.run_command(command.to_string()).await?; if !exit_status.success() { return Err(ToolproofTestFailure::Custom { @@ -97,7 +97,7 @@ mod run { ) -> Result<(), ToolproofStepError> { let command = args.get_string("command")?; - let exit_status = civ.run_command(command.to_string())?; + let exit_status = civ.run_command(command.to_string()).await?; if exit_status.success() { return Err(ToolproofTestFailure::Custom { diff --git a/toolproof/src/errors.rs b/toolproof/src/errors.rs index 6210473..c363306 100644 --- a/toolproof/src/errors.rs +++ b/toolproof/src/errors.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; +use chromiumoxide::error::CdpError; use pagebrowse::PagebrowseError; use thiserror::Error; @@ -45,6 +46,8 @@ pub enum ToolproofInternalError { Custom { msg: String }, #[error("{0}")] PagebrowseError(#[from] PagebrowseError), + #[error("{0}")] + ChromeError(#[from] CdpError), } #[derive(Error, Debug)] diff --git a/toolproof/src/main.rs b/toolproof/src/main.rs index 9a4b0f2..ac735ee 100644 --- a/toolproof/src/main.rs +++ b/toolproof/src/main.rs @@ -1,29 +1,22 @@ use std::collections::BTreeMap; use std::fmt::Display; use std::sync::Arc; -use std::time::Duration; -use std::{collections::HashMap, path::PathBuf, time::Instant}; +use std::{collections::HashMap, time::Instant}; use console::{style, Term}; -use futures::stream::StreamExt; -use futures::{future::join_all, stream::FuturesUnordered}; +use futures::future::join_all; use normalize_path::NormalizePath; -use pagebrowse::PagebrowseBuilder; use parser::ToolproofFileType; use schematic::color::owo::OwoColorize; use segments::ToolproofSegments; -use similar_string::{compare_similarity, find_best_similarity}; +use similar_string::compare_similarity; use tokio::fs::read_to_string; -use tokio::time::sleep; +use tokio::sync::OnceCell; use wax::Glob; -use crate::definitions::{ - register_assertions, register_instructions, register_retrievers, ToolproofInstruction, -}; +use crate::definitions::{register_assertions, register_instructions, register_retrievers}; use crate::differ::diff_snapshots; -use crate::errors::{ - ToolproofInputError, ToolproofStepError, ToolproofTestError, ToolproofTestFailure, -}; +use crate::errors::{ToolproofInputError, ToolproofStepError, ToolproofTestError}; use crate::interactive::{confirm_snapshot, get_run_mode, question, RunMode}; use crate::logging::log_step_runs; use crate::options::configure; @@ -158,14 +151,15 @@ fn closest_strings<'o>(target: &String, options: &'o Vec) -> Vec<(&'o St scores } -#[tokio::main] -async fn main() { +async fn main_inner() -> Result<(), ()> { let ctx = configure(); let start = Instant::now(); let glob = Glob::new("**/*.toolproof.yml").expect("Valid glob"); - let walker = glob.walk(".").flatten(); + let walker = glob + .walk(ctx.params.root.clone().unwrap_or(".".into())) + .flatten(); let loaded_files = walker .map(|entry| { @@ -210,7 +204,7 @@ async fn main() { for e in errors { eprintln!(" • {e}"); } - std::process::exit(1); + return Err(()); } let all_instructions = register_instructions(); @@ -231,19 +225,8 @@ async fn main() { .map(|k| k.get_comparison_string()) .collect(); - let pagebrowser = PagebrowseBuilder::new(ctx.params.concurrency) - .visible(false) - .manager_path(format!( - "{}/../bin/pagebrowse_manager", - env!("CARGO_MANIFEST_DIR") - )) - .init_script(include_str!("./definitions/browser/init.js").to_string()) - .build() - .await - .expect("Can't build the pagebrowser"); - let universe = Arc::new(Universe { - pagebrowser: Arc::new(pagebrowser), + browser: OnceCell::new(), tests: all_tests, instructions: all_instructions, instruction_comparisons, @@ -259,7 +242,7 @@ async fn main() { Ok(mode) => mode, Err(e) => { eprintln!("{e}"); - std::process::exit(1); + return Err(()); } } } else { @@ -583,7 +566,7 @@ async fn main() { Ok(b) => b, Err(e) => { eprintln!("{e}"); - std::process::exit(1); + return Err(()); } }; @@ -602,7 +585,7 @@ async fn main() { if let Err(e) = tokio::fs::write(&file.file_path, out).await { eprintln!("Unable to write updates snapshot to disk.\n{e}"); - std::process::exit(1); + return Err(()); } } } @@ -637,11 +620,21 @@ async fn main() { "{}", style(&format!("\nSome tests failed{}", duration)).red() ); - std::process::exit(1); + return Err(()); } else { println!( "{}", style(&format!("\nAll tests passed{}", duration)).green() ); } + + Ok(()) +} + +#[tokio::main] +async fn main() { + match main_inner().await { + Ok(_) => std::process::exit(0), + Err(_) => std::process::exit(1), + } } diff --git a/toolproof/src/options.rs b/toolproof/src/options.rs index 04f77f1..3f95a83 100644 --- a/toolproof/src/options.rs +++ b/toolproof/src/options.rs @@ -1,4 +1,6 @@ -use clap::{arg, command, value_parser, Arg, ArgAction, ArgMatches, Command}; +use clap::{ + arg, builder::PossibleValuesParser, command, value_parser, Arg, ArgAction, ArgMatches, Command, +}; use schematic::{derive_enum, Config, ConfigEnum, ConfigLoader}; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, env, path::PathBuf}; @@ -100,9 +102,24 @@ fn get_cli_matches() -> ArgMatches { ) .action(clap::ArgAction::SetTrue), ) + .arg( + arg!( + --browser ... "Specify which browser to use when running browser automation tests" + ) + .required(false) + .value_parser(PossibleValuesParser::new(["chrome", "pagebrowse"])), + ) .get_matches() } +#[derive(ConfigEnum, Default, Clone, Copy, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolproofBrowserImpl { + #[default] + Chrome, + Pagebrowse, +} + #[derive(Config, Debug, Clone)] #[config(rename_all = "snake_case")] pub struct ToolproofParams { @@ -124,6 +141,10 @@ pub struct ToolproofParams { /// Run all tests when in interactive mode pub all: bool, + /// Specify which browser to use when running browser automation tests + #[setting(env = "TOOLPROOF_BROWSER")] + pub browser: ToolproofBrowserImpl, + /// How many tests should be run concurrently #[setting(env = "TOOLPROOF_CONCURRENCY")] #[setting(default = 10)] diff --git a/toolproof/src/runner.rs b/toolproof/src/runner.rs index 2aa9e0d..263262d 100644 --- a/toolproof/src/runner.rs +++ b/toolproof/src/runner.rs @@ -30,11 +30,11 @@ pub async fn run_toolproof_experiment( universe, }; - run_toolproof_steps(&input.file_directory, &mut input.steps, &mut civ).await?; + let res = run_toolproof_steps(&input.file_directory, &mut input.steps, &mut civ).await; civ.shutdown().await; - Ok(()) + res } #[async_recursion] diff --git a/toolproof/src/universe.rs b/toolproof/src/universe.rs index a211379..4748bc4 100644 --- a/toolproof/src/universe.rs +++ b/toolproof/src/universe.rs @@ -1,19 +1,18 @@ -use pagebrowse::Pagebrowser; -use std::{ - collections::{BTreeMap, HashMap}, - path::PathBuf, - sync::Arc, -}; +use std::collections::{BTreeMap, HashMap}; + +use tokio::sync::OnceCell; use crate::{ - definitions::{ToolproofAssertion, ToolproofInstruction, ToolproofRetriever}, + definitions::{ + browser::BrowserTester, ToolproofAssertion, ToolproofInstruction, ToolproofRetriever, + }, options::ToolproofContext, segments::ToolproofSegments, ToolproofTestFile, }; pub struct Universe<'u> { - pub pagebrowser: Arc, + pub browser: OnceCell, pub tests: BTreeMap, pub instructions: HashMap, pub instruction_comparisons: Vec,