diff --git a/.github/workflows/sub-ci-integration-tests-gcp.yml b/.github/workflows/sub-ci-integration-tests-gcp.yml index fabd90da2a4..76cb168feb9 100644 --- a/.github/workflows/sub-ci-integration-tests-gcp.yml +++ b/.github/workflows/sub-ci-integration-tests-gcp.yml @@ -516,32 +516,31 @@ jobs: zebra_state_dir: "zebrad-cache" secrets: inherit - # TODO: Move this test once we have the new scanner binary. - # # Test that the scanner can continue scanning where it was left when zebrad restarts. - # # - # # Runs: - # # - after every PR is merged to `main` - # # - on every PR update - # # - # # If the state version has changed, waits for the new cached states to be created. - # # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - # scan-start-where-left-test: - # name: Scan starts where left - # needs: [test-full-sync, get-available-disks] - # uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml - # if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - # with: - # app_name: zebrad - # test_id: scan-start-where-left - # test_description: Test that the scanner can continue scanning where it was left when zebrad restarts. - # test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_START_WHERE_LEFT=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" - # needs_zebra_state: true - # needs_lwd_state: false - # saves_to_disk: true - # disk_suffix: tip - # root_state_path: "/var/cache" - # zebra_state_dir: "zebrad-cache" - # secrets: inherit + # Test that the scanner can continue scanning where it was left when zebrad restarts. + # + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached states to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + scan-start-where-left-test: + name: Scan starts where left + needs: [test-full-sync, get-available-disks] + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml + if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + with: + app_name: zebra-scan + test_id: scan-start-where-left + test_description: Test that the scanner can continue scanning where it was left when zebrad restarts. + test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_START_WHERE_LEFT=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" + needs_zebra_state: true + needs_lwd_state: false + saves_to_disk: true + disk_suffix: tip + root_state_path: "/var/cache" + zebra_state_dir: "zebrad-cache" + secrets: inherit # Test that the scan task registers keys, deletes keys, and subscribes to results for keys while running. # @@ -557,7 +556,7 @@ jobs: uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} with: - app_name: zebrad + app_name: zebra-scan test_id: scan-task-commands test_description: Test that the scan task registers keys, deletes keys, and subscribes to results for keys while running. test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_TASK_COMMANDS=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" @@ -589,7 +588,7 @@ jobs: lightwalletd-grpc-test, get-block-template-test, submit-block-test, - # scan-start-where-left-test, + scan-start-where-left-test, scan-task-commands-test, ] # Only open tickets for failed scheduled jobs, manual workflow runs, or `main` branch merges. diff --git a/Cargo.lock b/Cargo.lock index efd20be487a..dc3c4cbeda7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6016,9 +6016,11 @@ dependencies = [ "ff", "futures", "group", + "hex", "indexmap 2.2.6", "insta", "itertools 0.13.0", + "jsonrpc", "jubjub", "lazy_static", "proptest", @@ -6029,7 +6031,10 @@ dependencies = [ "serde", "serde_json", "structopt", + "tempfile", "tokio", + "toml 0.8.14", + "tonic 0.11.0", "tower", "tracing", "tracing-subscriber", @@ -6044,6 +6049,7 @@ dependencies = [ "zebra-rpc", "zebra-state", "zebra-test", + "zebrad", ] [[package]] @@ -6159,7 +6165,6 @@ dependencies = [ "zebra-chain", "zebra-node-services", "zebra-rpc", - "zebra-scan", ] [[package]] @@ -6226,7 +6231,6 @@ dependencies = [ "zebra-network", "zebra-node-services", "zebra-rpc", - "zebra-scan", "zebra-state", "zebra-test", "zebra-utils", diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 8cd61a49bb3..cc4fb5ac82e 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -347,14 +347,14 @@ case "$1" in run_cargo_test "${ENTRYPOINT_FEATURES}" "submit_block" elif [[ "${TEST_SCAN_START_WHERE_LEFT}" -eq "1" ]]; then - # Test that the scanner can continue scanning where it was left when zebrad restarts. + # Test that the scanner can continue scanning where it was left when zebra-scanner restarts. check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "shielded-scan" "scan_start_where_left" - + exec cargo test --locked --release --features "zebra-test" --package zebra-scan -- --nocapture --include-ignored scan_start_where_left + elif [[ "${TEST_SCAN_TASK_COMMANDS}" -eq "1" ]]; then - # Test that the scanner can continue scanning where it was left when zebrad restarts. + # Test that the scan task commands are working. check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "shielded-scan" "scan_task_commands" + exec cargo test --locked --release --features "zebra-test" --package zebra-scan -- --nocapture --include-ignored scan_task_commands else exec "$@" diff --git a/zebra-scan/Cargo.toml b/zebra-scan/Cargo.toml index d4e0f825b4f..d8d44ba8604 100644 --- a/zebra-scan/Cargo.toml +++ b/zebra-scan/Cargo.toml @@ -23,6 +23,15 @@ required-features = ["proptest-impl"] name = "zebra-scanner" path = "src/bin/scanner/main.rs" +[[bin]] +name = "scanning-results-reader" +path = "src/bin/scanning-results-reader/main.rs" +required-features = ["results-reader"] + +[[bin]] # Bin to run zebrad, used in scanner tests +name = "zebrad-for-scanner" +path = "src/bin/zebrad-for-scanner/main.rs" + [features] # Production features that activate extra dependencies, or extra features in dependencies @@ -44,7 +53,10 @@ proptest-impl = [ ] # Needed for the zebra-scanner binary. -shielded-scan = [] +results-reader = [ + "jsonrpc", + "hex" +] [dependencies] @@ -91,6 +103,11 @@ structopt = "0.3.26" lazy_static = "1.4.0" serde_json = "1.0.117" +jsonrpc = { version = "0.18.0", optional = true } +hex = { version = "0.4.3", optional = true } + +zebrad = { path = "../zebrad", version = "1.8.0" } + [dev-dependencies] insta = { version = "1.39.0", features = ["ron", "redactions"] } tokio = { version = "1.37.0", features = ["test-util"] } @@ -102,7 +119,11 @@ ff = "0.13.0" group = "0.13.0" jubjub = "0.10.0" rand = "0.8.5" +tempfile = "3.10.1" zcash_note_encryption = "0.4.0" +toml = "0.8.13" +tonic = "0.11.0" zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38", features = ["proptest-impl"] } zebra-test = { path = "../zebra-test", version = "1.0.0-beta.38" } + diff --git a/zebra-utils/src/bin/scanning-results-reader/main.rs b/zebra-scan/src/bin/scanning-results-reader/main.rs similarity index 100% rename from zebra-utils/src/bin/scanning-results-reader/main.rs rename to zebra-scan/src/bin/scanning-results-reader/main.rs diff --git a/zebra-scan/src/bin/zebrad-for-scanner/main.rs b/zebra-scan/src/bin/zebrad-for-scanner/main.rs new file mode 100644 index 00000000000..0eaa81677ce --- /dev/null +++ b/zebra-scan/src/bin/zebrad-for-scanner/main.rs @@ -0,0 +1,8 @@ +//! Main entry point for Zebrad, to be used in zebra-scan tests + +use zebrad::application::{boot, APPLICATION}; + +/// Process entry point for `zebrad` +fn main() { + boot(&APPLICATION); +} diff --git a/zebrad/tests/common/shielded_scan/scan_task_commands.rs b/zebra-scan/tests/scan_task_commands.rs similarity index 75% rename from zebrad/tests/common/shielded_scan/scan_task_commands.rs rename to zebra-scan/tests/scan_task_commands.rs index a7693b8c812..20bbcca190d 100644 --- a/zebrad/tests/common/shielded_scan/scan_task_commands.rs +++ b/zebra-scan/tests/scan_task_commands.rs @@ -5,12 +5,14 @@ //! //! export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" //! cargo test scan_task_commands --features="shielded-scan" -- --ignored --nocapture +#![allow(dead_code)] use std::{fs, time::Duration}; use color_eyre::{eyre::eyre, Result}; - use tokio::sync::mpsc::error::TryRecvError; +use tower::{util::BoxService, Service}; + use zebra_chain::{ block::Height, chain_tip::ChainTip, @@ -20,13 +22,12 @@ use zebra_chain::{ use zebra_scan::{ service::ScanTask, storage::{db::SCANNER_DATABASE_KIND, Storage}, - tests::ZECPAGES_SAPLING_VIEWING_KEY, }; -use crate::common::{ - cached_state::start_state_service_with_cache_dir, launch::can_spawn_zebrad_for_test_type, - test_type::TestType, -}; +use zebra_state::{ChainTipChange, LatestChainTip}; + +pub type BoxStateService = + BoxService; /// The minimum required tip height for the cached state in this test. const REQUIRED_MIN_TIP_HEIGHT: Height = Height(1_000_000); @@ -38,6 +39,9 @@ const WAIT_FOR_RESULTS_DURATION: Duration = Duration::from_secs(60); /// A block height where a scan result can be found with the [`ZECPAGES_SAPLING_VIEWING_KEY`] const EXPECTED_RESULT_HEIGHT: Height = Height(780_532); +/// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo) +const ZECPAGES_SAPLING_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz"; + /// Initialize Zebra's state service with a cached state, then: /// - Start the scan task, /// - Add a new key, @@ -48,24 +52,23 @@ const EXPECTED_RESULT_HEIGHT: Height = Height(780_532); pub(crate) async fn run() -> Result<()> { let _init_guard = zebra_test::init(); - let test_type = TestType::UpdateZebraCachedStateNoRpc; - let test_name = "scan_task_commands"; let network = Network::Mainnet; - // Skip the test unless the user specifically asked for it and there is a zebrad_state_path - if !can_spawn_zebrad_for_test_type(test_name, test_type, true) { - return Ok(()); - } + // Logs the network as zebrad would as part of the metadata when starting up. + // This is currently needed for the 'Check startup logs' step in CI to pass. + tracing::info!("Zcash network: {network}"); - tracing::info!( - ?network, - ?test_type, - "running scan_subscribe_results test using zebra state service", - ); + let zebrad_state_path = match std::env::var_os("ZEBRA_CACHED_STATE_DIR") { + None => { + tracing::error!("ZEBRA_CACHED_STATE_DIR is not set"); + return Ok(()); + } + Some(path) => std::path::PathBuf::from(path), + }; - let zebrad_state_path = test_type - .zebrad_state_path(test_name) - .expect("already checked that there is a cached state path"); + // Remove the scan directory before starting. + let scan_db_path = zebrad_state_path.join(SCANNER_DATABASE_KIND); + fs::remove_dir_all(std::path::Path::new(&scan_db_path)).ok(); let mut scan_config = zebra_scan::Config::default(); scan_config @@ -73,14 +76,6 @@ pub(crate) async fn run() -> Result<()> { .cache_dir .clone_from(&zebrad_state_path); - // Logs the network as zebrad would as part of the metadata when starting up. - // This is currently needed for the 'Check startup logs' step in CI to pass. - tracing::info!("Zcash network: {network}"); - - // Remove the scan directory before starting. - let scan_db_path = zebrad_state_path.join(SCANNER_DATABASE_KIND); - fs::remove_dir_all(std::path::Path::new(&scan_db_path)).ok(); - let (_state_service, _read_state_service, latest_chain_tip, chain_tip_change) = start_state_service_with_cache_dir(&network, zebrad_state_path.clone()).await?; @@ -88,7 +83,8 @@ pub(crate) async fn run() -> Result<()> { cache_dir: zebrad_state_path.clone(), ..zebra_state::Config::default() }; - let (read_state, _db, _) = zebra_state::init_read_only(state_config, &network); + + let (read_state, _, _) = zebra_state::init_read_only(state_config.clone(), &network); let chain_tip_height = latest_chain_tip .best_tip_height() @@ -158,7 +154,6 @@ pub(crate) async fn run() -> Result<()> { TryRecvError::Disconnected, "any result senders should have been dropped" ); - break; } } @@ -166,3 +161,25 @@ pub(crate) async fn run() -> Result<()> { Ok(()) } + +pub async fn start_state_service_with_cache_dir( + network: &Network, + cache_dir: impl Into, +) -> Result<( + BoxStateService, + impl Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + >, + LatestChainTip, + ChainTipChange, +)> { + let config = zebra_state::Config { + cache_dir: cache_dir.into(), + ..zebra_state::Config::default() + }; + + // These tests don't need UTXOs to be verified efficiently, because they use cached states. + Ok(zebra_state::init(config, network, Height::MAX, 0)) +} diff --git a/zebra-scan/tests/scanner.rs b/zebra-scan/tests/scanner.rs new file mode 100644 index 00000000000..3cd8573d342 --- /dev/null +++ b/zebra-scan/tests/scanner.rs @@ -0,0 +1,283 @@ +//! `zebra-scanner` binary tests. +use tempfile::TempDir; + +use zebra_grpc::scanner::{scanner_client::ScannerClient, Empty}; +use zebra_test::{ + args, + command::{Arguments, TestDirExt}, + prelude::*, +}; + +use std::{io::Write, path::Path}; + +mod scan_task_commands; + +/// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo) +const ZECPAGES_SAPLING_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz"; + +/// Test the scanner binary with the `--help` flag. +#[test] +fn scanner_help() -> eyre::Result<()> { + let _init_guard = zebra_test::init(); + + let testdir = testdir()?; + + let child = testdir.spawn_scanner_child(args!["--help"])?; + let output = child.wait_with_output()?; + let output = output.assert_success()?; + + output.stdout_line_contains("USAGE:")?; + + Ok(()) +} + +/// Test that the scanner binary starts the scan task. +/// +/// This test creates a new zebrad cache dir by using a `zebrad` binary specified in the `CARGO_BIN_EXE_zebrad` env variable. +/// +/// To run it locally, one way is: +/// +/// ``` +/// RUST_LOG=info cargo test scan_binary_starts -- --include-ignored --nocapture +/// ``` +#[tokio::test] +#[cfg(not(target_os = "windows"))] +async fn scan_binary_starts() -> Result<()> { + use std::time::Duration; + + let _init_guard = zebra_test::init(); + + // Create a directory to dump test data into it. + let test_dir = testdir()?; + + // Create a config for zebrad with a cache directory inside the test directory. + let mut config = zebrad::config::ZebradConfig::default(); + config.state.cache_dir = test_dir.path().join("zebra").to_path_buf(); + let config_file = test_dir.path().join("zebrad.toml"); + std::fs::File::create(config_file.clone())?.write_all(toml::to_string(&config)?.as_bytes())?; + + // Start the node just to make sure a cache is created. + let mut zebrad = + test_dir.spawn_zebrad_child(args!["-c", config_file.clone().to_str().unwrap(), "start"])?; + zebrad.expect_stdout_line_matches("Opened Zebra state cache at .*")?; + + // Kill the node now that we have a valid zebra node cache. + zebrad.kill(false)?; + let output = zebrad.wait_with_output()?; + + // Make sure the command was killed + output.assert_was_killed()?; + + // Create a new directory for the scanner + let test_dir = testdir()?; + + // Create arguments for the scanner + let key = serde_json::json!({"key": ZECPAGES_SAPLING_VIEWING_KEY}).to_string(); + let listen_addr = "127.0.0.1:8231"; + let rpc_listen_addr = "127.0.0.1:8232"; + let zebrad_cache_dir = config.state.cache_dir.clone(); + let scanning_cache_dir = test_dir.path().join("scanner").to_path_buf(); + let args = args![ + "--zebrad-cache-dir", + zebrad_cache_dir.to_str().unwrap(), + "--scanning-cache-dir", + scanning_cache_dir.to_str().unwrap(), + "--sapling-keys-to-scan", + key, + "--listen-addr", + listen_addr, + "--zebra-rpc-listen-addr", + rpc_listen_addr + ]; + + // Start the scaner using another test directory. + let mut zebra_scanner = test_dir.spawn_scanner_child(args)?; + + // Check scanner was started. + zebra_scanner.expect_stdout_line_matches("loaded Zebra scanner cache")?; + + // Wait for the scanner's gRPC server to start up + tokio::time::sleep(Duration::from_secs(1)).await; + + // Check if the grpc server is up + let mut client = ScannerClient::connect(format!("http://{listen_addr}")).await?; + let request = tonic::Request::new(Empty {}); + client.get_info(request).await?; + + // Look for 2 scanner notices indicating we are below sapling activation. + zebra_scanner.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?; + zebra_scanner.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?; + + // Kill the scanner. + zebra_scanner.kill(true)?; + + // Check that scan task started and the scanning is done. + let output = zebra_scanner.wait_with_output()?; + + // Make sure the command was killed + output.assert_was_killed()?; + output.assert_failure()?; + + Ok(()) +} + +/// Test that the scanner can continue scanning where it was left when zebrad restarts. +/// +/// Needs a cache state close to the tip. A possible way to run it locally is: +/// +/// export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" +/// RUST_LOG=info cargo test scan_start_where_left -- --ignored --nocapture +/// +/// The test will run zebrad with a key to scan, scan the first few blocks after sapling and then stops. +/// Then it will restart zebrad and check that it resumes scanning where it was left. +#[ignore] +#[tokio::test] +#[cfg(not(target_os = "windows"))] +async fn scan_start_where_left() -> Result<()> { + use ZECPAGES_SAPLING_VIEWING_KEY; + + let _init_guard = zebra_test::init(); + + let Ok(zebrad_cachedir) = std::env::var("ZEBRA_CACHED_STATE_DIR") else { + tracing::info!("skipping scan_start_where_left test due to missing cached state, \ + please set a ZEBRA_CACHED_STATE_DIR env var with a populated and valid path to run this test"); + return Ok(()); + }; + + // Logs the network as zebrad would as part of the metadata when starting up. + // This is currently needed for the 'Check startup logs' step in CI to pass. + let network = zebra_chain::parameters::Network::Mainnet; + tracing::info!("Zcash network: {network}"); + + let scanning_cache_dir = testdir()?.path().join("scanner").to_path_buf(); + + // Create arguments for the scanner + let key = serde_json::json!({"key": ZECPAGES_SAPLING_VIEWING_KEY}).to_string(); + let listen_addr = "127.0.0.1:18231"; + let rpc_listen_addr = "127.0.0.1:18232"; + let args = args![ + "--zebrad-cache-dir", + zebrad_cachedir, + "--scanning-cache-dir", + scanning_cache_dir.to_str().unwrap(), + "--sapling-keys-to-scan", + key, + "--listen-addr", + listen_addr, + "--zebra-rpc-listen-addr", + rpc_listen_addr + ]; + + // Start the scaner using another test directory. + let mut zebra_scanner: TestChild = testdir()?.spawn_scanner_child(args.clone())?; + + // Check scanner was started. + zebra_scanner.expect_stdout_line_matches("loaded Zebra scanner cache")?; + + // The first time + zebra_scanner.expect_stdout_line_matches( + r"Scanning the blockchain for key 0, started at block 419200, now at block 420000", + )?; + + // Make sure scanner scans a few blocks. + zebra_scanner.expect_stdout_line_matches( + r"Scanning the blockchain for key 0, started at block 419200, now at block 430000", + )?; + zebra_scanner.expect_stdout_line_matches( + r"Scanning the blockchain for key 0, started at block 419200, now at block 440000", + )?; + + // Kill the scanner. + zebra_scanner.kill(true)?; + + // Check that scan task started and the first scanning is done. + let output = zebra_scanner.wait_with_output()?; + + // Make sure the command was killed + output.assert_was_killed()?; + output.assert_failure()?; + + // Start the node again with the same arguments. + let mut zebra_scanner: TestChild = testdir()?.spawn_scanner_child(args)?; + + // Resuming message. + zebra_scanner.expect_stdout_line_matches( + "Last scanned height for key number 0 is 439000, resuming at 439001", + )?; + zebra_scanner.expect_stdout_line_matches("loaded Zebra scanner cache")?; + + // Start scanning where it was left. + zebra_scanner.expect_stdout_line_matches( + r"Scanning the blockchain for key 0, started at block 439001, now at block 440000", + )?; + zebra_scanner.expect_stdout_line_matches( + r"Scanning the blockchain for key 0, started at block 439001, now at block 450000", + )?; + + // Kill the scanner. + zebra_scanner.kill(true)?; + + // Check that scan task started and the second scanning is done. + let output = zebra_scanner.wait_with_output()?; + + // Make sure the command was killed + output.assert_was_killed()?; + output.assert_failure()?; + + Ok(()) +} + +/// Tests successful: +/// - Registration of a new key, +/// - Subscription to scan results of new key, and +/// - Deletion of keys +/// in the scan task +/// See [`common::shielded_scan::scan_task_commands`] for more information. +/// +/// Example of how to run the scan_task_commands test locally: +/// +/// ```console +/// RUST_LOG=info ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test scan_task_commands -- --include-ignored --nocapture +/// ``` +#[tokio::test] +#[ignore] +async fn scan_task_commands() -> Result<()> { + scan_task_commands::run().await +} + +/// Create a temporary directory for testing `zebra-scanner`. +pub fn testdir() -> eyre::Result { + tempfile::Builder::new() + .prefix("zebra_scanner_tests") + .tempdir() + .map_err(Into::into) +} + +/// Extension trait for methods on `tempfile::TempDir` for using it as a test +/// directory for `zebra-scanner`. +pub trait ScannerTestDirExt +where + Self: AsRef + Sized, +{ + /// Spawn `zebra-scanner` with `args` as a child process in this test directory. + fn spawn_scanner_child(self, args: Arguments) -> Result>; + + /// Spawn `zebrad` with `args` as a child process in this test directory. + fn spawn_zebrad_child(self, args: Arguments) -> Result>; +} + +impl ScannerTestDirExt for T +where + Self: TestDirExt + AsRef + Sized, +{ + fn spawn_scanner_child(self, extra_args: Arguments) -> Result> { + let mut args = Arguments::new(); + args.merge_with(extra_args); + self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebra-scanner"), args) + } + fn spawn_zebrad_child(self, extra_args: Arguments) -> Result> { + let mut args = Arguments::new(); + args.merge_with(extra_args); + self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebrad-for-scanner"), args) + } +} diff --git a/zebra-utils/Cargo.toml b/zebra-utils/Cargo.toml index 38bc2fc79cb..3c3ddab00d0 100644 --- a/zebra-utils/Cargo.toml +++ b/zebra-utils/Cargo.toml @@ -36,11 +36,6 @@ name = "block-template-to-proposal" path = "src/bin/block-template-to-proposal/main.rs" required-features = ["getblocktemplate-rpcs"] -[[bin]] -name = "scanning-results-reader" -path = "src/bin/scanning-results-reader/main.rs" -required-features = ["shielded-scan"] - [[bin]] name = "openapi-generator" path = "src/bin/openapi-generator/main.rs" @@ -76,7 +71,6 @@ shielded-scan = [ "jsonrpc", "zcash_primitives", "zcash_client_backend", - "zebra-scan" ] openapi-generator = [ @@ -102,7 +96,6 @@ thiserror = "1.0.61" zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.38" } zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.38" } -zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.7", optional = true } # These crates are needed for the block-template-to-proposal binary zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.38", optional = true } diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index 950d7659658..a49bba9968b 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -77,9 +77,6 @@ internal-miner = [ "zebra-rpc/getblocktemplate-rpcs", ] -# Experimental shielded blockchain scanning -shielded-scan = ["zebra-scan"] - # Experimental elasticsearch indexing elasticsearch = [ "zebra-state/elasticsearch", @@ -127,7 +124,6 @@ proptest-impl = [ "zebra-state/proptest-impl", "zebra-network/proptest-impl", "zebra-chain/proptest-impl", - "zebra-scan?/proptest-impl", ] # Build the zebra-checkpoints utility for checkpoint generation tests @@ -165,9 +161,6 @@ zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.3 zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.38" } zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38" } -# Experimental shielded-scan feature -zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.7", optional = true } - # Required for crates.io publishing, but it's only used in tests zebra-utils = { path = "../zebra-utils", version = "1.0.0-beta.38", optional = true } @@ -283,7 +276,6 @@ color-eyre = { version = "0.6.3" } zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.38", features = ["proptest-impl"] } zebra-consensus = { path = "../zebra-consensus", version = "1.0.0-beta.38", features = ["proptest-impl"] } zebra-network = { path = "../zebra-network", version = "1.0.0-beta.38", features = ["proptest-impl"] } -zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.7", features = ["proptest-impl"] } zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38", features = ["proptest-impl"] } zebra-test = { path = "../zebra-test", version = "1.0.0-beta.38" } diff --git a/zebrad/src/config.rs b/zebrad/src/config.rs index c0b22479c36..a6174599ef8 100644 --- a/zebrad/src/config.rs +++ b/zebrad/src/config.rs @@ -54,8 +54,4 @@ pub struct ZebradConfig { #[serde(skip_serializing_if = "zebra_rpc::config::mining::Config::skip_getblocktemplate")] /// Mining configuration pub mining: zebra_rpc::config::mining::Config, - - #[cfg(feature = "shielded-scan")] - /// Scanner configuration - pub shielded_scan: zebra_scan::config::Config, } diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index eee5be6e016..810cf100f31 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -120,14 +120,6 @@ //! //! Please refer to the documentation of each test for more information. //! -//! ## Shielded scanning tests -//! -//! Example of how to run the scan_task_commands test: -//! -//! ```console -//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test scan_task_commands --features shielded-scan --release -- --ignored --nocapture -//! ``` -//! //! ## Checkpoint Generation Tests //! //! Generate checkpoints on mainnet and testnet using a cached state: @@ -2891,213 +2883,6 @@ async fn fully_synced_rpc_z_getsubtreesbyindex_snapshot_test() -> Result<()> { Ok(()) } -// TODO: Move this test once we have the new scanner binary. -// /// Test that the scanner task gets started when the node starts. -// #[test] -// #[cfg(feature = "shielded-scan")] -// fn scan_task_starts() -> Result<()> { -// use indexmap::IndexMap; -// use zebra_scan::tests::ZECPAGES_SAPLING_VIEWING_KEY; - -// let _init_guard = zebra_test::init(); - -// let test_type = TestType::LaunchWithEmptyState { -// launches_lightwalletd: false, -// }; -// let mut config = default_test_config(&Mainnet)?; -// let mut keys = IndexMap::new(); -// keys.insert(ZECPAGES_SAPLING_VIEWING_KEY.to_string(), 1); -// config.shielded_scan.sapling_keys_to_scan = keys; - -// // Start zebra with the config. -// let mut zebrad = testdir()? -// .with_exact_config(&config)? -// .spawn_child(args!["start"])? -// .with_timeout(test_type.zebrad_timeout()); - -// // Check scanner was started. -// zebrad.expect_stdout_line_matches("loaded Zebra scanner cache")?; - -// // Look for 2 scanner notices indicating we are below sapling activation. -// zebrad.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?; -// zebrad.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?; - -// // Kill the node. -// zebrad.kill(false)?; - -// // Check that scan task started and the first scanning is done. -// let output = zebrad.wait_with_output()?; - -// // Make sure the command was killed -// output.assert_was_killed()?; -// output.assert_failure()?; - -// Ok(()) -// } - -// TODO: Move this test once we have the new scanner binary. -// /// Test that the scanner gRPC server starts when the node starts. -// #[tokio::test] -// #[cfg(all(feature = "shielded-scan", not(target_os = "windows")))] -// async fn scan_rpc_server_starts() -> Result<()> { -// use zebra_grpc::scanner::{scanner_client::ScannerClient, Empty}; - -// let _init_guard = zebra_test::init(); - -// let test_type = TestType::LaunchWithEmptyState { -// launches_lightwalletd: false, -// }; - -// let port = random_known_port(); -// let listen_addr = format!("127.0.0.1:{port}"); -// let mut config = default_test_config(&Mainnet)?; -// config.shielded_scan.listen_addr = Some(listen_addr.parse()?); - -// // Start zebra with the config. -// let mut zebrad = testdir()? -// .with_exact_config(&config)? -// .spawn_child(args!["start"])? -// .with_timeout(test_type.zebrad_timeout()); - -// // Wait until gRPC server is starting. -// tokio::time::sleep(LAUNCH_DELAY).await; -// zebrad.expect_stdout_line_matches("starting scan gRPC server")?; -// tokio::time::sleep(Duration::from_secs(1)).await; - -// let mut client = ScannerClient::connect(format!("http://{listen_addr}")).await?; - -// let request = tonic::Request::new(Empty {}); - -// client.get_info(request).await?; - -// // Kill the node. -// zebrad.kill(false)?; - -// // Check that scan task started and the first scanning is done. -// let output = zebrad.wait_with_output()?; - -// // Make sure the command was killed -// output.assert_was_killed()?; -// output.assert_failure()?; - -// Ok(()) -// } - -// TODO: Move this test once we have the new scanner binary. -// /// Test that the scanner can continue scanning where it was left when zebrad restarts. -// /// -// /// Needs a cache state close to the tip. A possible way to run it locally is: -// /// -// /// export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" -// /// cargo test scan_start_where_left --features="shielded-scan" -- --ignored --nocapture -// /// -// /// The test will run zebrad with a key to scan, scan the first few blocks after sapling and then stops. -// /// Then it will restart zebrad and check that it resumes scanning where it was left. -// /// -// /// Note: This test will remove all the contents you may have in the ZEBRA_CACHED_STATE_DIR/private-scan directory -// /// so it can start with an empty scanning state. -// #[ignore] -// #[test] -// #[cfg(feature = "shielded-scan")] -// fn scan_start_where_left() -> Result<()> { -// use indexmap::IndexMap; -// use zebra_scan::{storage::db::SCANNER_DATABASE_KIND, tests::ZECPAGES_SAPLING_VIEWING_KEY}; - -// let _init_guard = zebra_test::init(); - -// // use `UpdateZebraCachedStateNoRpc` as the test type to make sure a zebrad cache state is available. -// let test_type = TestType::UpdateZebraCachedStateNoRpc; -// if let Some(cache_dir) = test_type.zebrad_state_path("scan test") { -// // Add a key to the config -// let mut config = default_test_config(&Mainnet)?; -// let mut keys = IndexMap::new(); -// keys.insert(ZECPAGES_SAPLING_VIEWING_KEY.to_string(), 1); -// config.shielded_scan.sapling_keys_to_scan = keys; - -// // Add the cache dir to shielded scan, make it the same as the zebrad cache state. -// config -// .shielded_scan -// .db_config_mut() -// .cache_dir -// .clone_from(&cache_dir); -// config.shielded_scan.db_config_mut().ephemeral = false; - -// // Add the cache dir to state. -// config.state.cache_dir.clone_from(&cache_dir); -// config.state.ephemeral = false; - -// // Remove the scan directory before starting. -// let scan_db_path = cache_dir.join(SCANNER_DATABASE_KIND); -// fs::remove_dir_all(std::path::Path::new(&scan_db_path)).ok(); - -// // Start zebra with the config. -// let mut zebrad = testdir()? -// .with_exact_config(&config)? -// .spawn_child(args!["start"])? -// .with_timeout(test_type.zebrad_timeout()); - -// // Check scanner was started. -// zebrad.expect_stdout_line_matches("loaded Zebra scanner cache")?; - -// // The first time -// zebrad.expect_stdout_line_matches( -// r"Scanning the blockchain for key 0, started at block 419200, now at block 420000", -// )?; - -// // Make sure scanner scans a few blocks. -// zebrad.expect_stdout_line_matches( -// r"Scanning the blockchain for key 0, started at block 419200, now at block 430000", -// )?; -// zebrad.expect_stdout_line_matches( -// r"Scanning the blockchain for key 0, started at block 419200, now at block 440000", -// )?; - -// // Kill the node. -// zebrad.kill(false)?; -// let output = zebrad.wait_with_output()?; - -// // Make sure the command was killed -// output.assert_was_killed()?; -// output.assert_failure()?; - -// // Start the node again. -// let mut zebrad = testdir()? -// .with_exact_config(&config)? -// .spawn_child(args!["start"])? -// .with_timeout(test_type.zebrad_timeout()); - -// // Resuming message. -// zebrad.expect_stdout_line_matches( -// "Last scanned height for key number 0 is 439000, resuming at 439001", -// )?; -// zebrad.expect_stdout_line_matches("loaded Zebra scanner cache")?; - -// // Start scanning where it was left. -// zebrad.expect_stdout_line_matches( -// r"Scanning the blockchain for key 0, started at block 439001, now at block 440000", -// )?; -// zebrad.expect_stdout_line_matches( -// r"Scanning the blockchain for key 0, started at block 439001, now at block 450000", -// )?; -// } - -// Ok(()) -// } - -// TODO: Add this test to CI (#8236) -/// Tests successful: -/// - Registration of a new key, -/// - Subscription to scan results of new key, and -/// - Deletion of keys -/// in the scan task -/// See [`common::shielded_scan::scan_task_commands`] for more information. -#[tokio::test] -#[ignore] -#[cfg(feature = "shielded-scan")] -async fn scan_task_commands() -> Result<()> { - common::shielded_scan::scan_task_commands::run().await -} - /// Checks that the Regtest genesis block can be validated. #[tokio::test] async fn validate_regtest_genesis_block() { diff --git a/zebrad/tests/common/mod.rs b/zebrad/tests/common/mod.rs index 3ff3f63db3a..6fe1b10f034 100644 --- a/zebrad/tests/common/mod.rs +++ b/zebrad/tests/common/mod.rs @@ -26,6 +26,3 @@ pub mod get_block_template_rpcs; #[cfg(feature = "getblocktemplate-rpcs")] pub mod regtest; - -#[cfg(feature = "shielded-scan")] -pub mod shielded_scan; diff --git a/zebrad/tests/common/shielded_scan.rs b/zebrad/tests/common/shielded_scan.rs deleted file mode 100644 index 106d6a16af3..00000000000 --- a/zebrad/tests/common/shielded_scan.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! Acceptance tests for `shielded-scan`` feature in zebrad. - -pub(crate) mod scan_task_commands;