tests(scanner): Move zebra scanner tests to binary (#8659)

* readd the scanner tests in the new binary context * remove commented out tests from zebrad * add Cargo.lock * add local copy of `ZECPAGES_SAPLING_VIEWING_KEY` to binary scanning tests * moves scanner-results-reader to zebra-scan, adds zebrad bin to zebra-scan, adds a short sleep before trying to connect to gRPC server in test * always include zebrad dependency in zebra-scan * renames zebrad bin in zebra-scan to zebrad-for-scanner * skip test when a cached state is missing * remove scanning config from zebrad * move `scan_task_commands` out of zebrad acceptance tests * remove scanner config from zebrad * do not panic if env var is not there * re enable start_scan_where left, remove shielded-scan feature foir scanning tests in CI * change app names * uncomment test * fix the scanner tests in CI by changing the test command * add log needed by CI? * move test * rename trait, docs fix --------- Co-authored-by: Arya <[email protected]>
ZcashFoundation · Jul 15, 2024 · 1238ec0 · 1238ec0
1 parent f3c2e19
commit 1238ec0
Show file tree

Hide file tree

Showing 14 changed files with 398 additions and 306 deletions.
diff --git a/.github/workflows/sub-ci-integration-tests-gcp.yml b/.github/workflows/sub-ci-integration-tests-gcp.yml
@@ -516,32 +516,31 @@ jobs:
       zebra_state_dir: "zebrad-cache"
     secrets: inherit
 
-  # TODO: Move this test once we have the new scanner binary.
-  # # Test that the scanner can continue scanning where it was left when zebrad restarts.
-  # #
-  # # Runs:
-  # # - after every PR is merged to `main`
-  # # - on every PR update
-  # #
-  # # If the state version has changed, waits for the new cached states to be created.
-  # # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
-  # scan-start-where-left-test:
-  #   name: Scan starts where left
-  #   needs: [test-full-sync, get-available-disks]
-  #   uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
-  #   if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
-  #   with:
-  #     app_name: zebrad
-  #     test_id: scan-start-where-left
-  #     test_description: Test that the scanner can continue scanning where it was left when zebrad restarts.
-  #     test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_START_WHERE_LEFT=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache"
-  #     needs_zebra_state: true
-  #     needs_lwd_state: false
-  #     saves_to_disk: true
-  #     disk_suffix: tip
-  #     root_state_path: "/var/cache"
-  #     zebra_state_dir: "zebrad-cache"
-  #   secrets: inherit
+  # Test that the scanner can continue scanning where it was left when zebrad restarts.
+  #
+  # Runs:
+  # - after every PR is merged to `main`
+  # - on every PR update
+  #
+  # If the state version has changed, waits for the new cached states to be created.
+  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
+  scan-start-where-left-test:
+    name: Scan starts where left
+    needs: [test-full-sync, get-available-disks]
+    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
+    if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
+    with:
+      app_name: zebra-scan
+      test_id: scan-start-where-left
+      test_description: Test that the scanner can continue scanning where it was left when zebrad restarts.
+      test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_START_WHERE_LEFT=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache"
+      needs_zebra_state: true
+      needs_lwd_state: false
+      saves_to_disk: true
+      disk_suffix: tip
+      root_state_path: "/var/cache"
+      zebra_state_dir: "zebrad-cache"
+    secrets: inherit
 
   # Test that the scan task registers keys, deletes keys, and subscribes to results for keys while running.
   #
@@ -557,7 +556,7 @@ jobs:
     uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
     if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
     with:
-      app_name: zebrad
+      app_name: zebra-scan
       test_id: scan-task-commands
       test_description: Test that the scan task registers keys, deletes keys, and subscribes to results for keys while running.
       test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_TASK_COMMANDS=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache"
@@ -589,7 +588,7 @@ jobs:
         lightwalletd-grpc-test,
         get-block-template-test,
         submit-block-test,
-        # scan-start-where-left-test,
+        scan-start-where-left-test,
         scan-task-commands-test,
       ]
     # Only open tickets for failed scheduled jobs, manual workflow runs, or `main` branch merges.

diff --git a/Cargo.lock b/Cargo.lock
@@ -6016,9 +6016,11 @@ dependencies = [
  "ff",
  "futures",
  "group",
+ "hex",
  "indexmap 2.2.6",
  "insta",
  "itertools 0.13.0",
+ "jsonrpc",
  "jubjub",
  "lazy_static",
  "proptest",
@@ -6029,7 +6031,10 @@ dependencies = [
  "serde",
  "serde_json",
  "structopt",
+ "tempfile",
  "tokio",
+ "toml 0.8.14",
+ "tonic 0.11.0",
  "tower",
  "tracing",
  "tracing-subscriber",
@@ -6044,6 +6049,7 @@ dependencies = [
  "zebra-rpc",
  "zebra-state",
  "zebra-test",
+ "zebrad",
 ]
 
 [[package]]
@@ -6159,7 +6165,6 @@ dependencies = [
  "zebra-chain",
  "zebra-node-services",
  "zebra-rpc",
- "zebra-scan",
 ]
 
 [[package]]
@@ -6226,7 +6231,6 @@ dependencies = [
  "zebra-network",
  "zebra-node-services",
  "zebra-rpc",
- "zebra-scan",
  "zebra-state",
  "zebra-test",
  "zebra-utils",

diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
@@ -347,14 +347,14 @@ case "$1" in
         run_cargo_test "${ENTRYPOINT_FEATURES}" "submit_block"
 
       elif [[ "${TEST_SCAN_START_WHERE_LEFT}" -eq "1" ]]; then
-        # Test that the scanner can continue scanning where it was left when zebrad restarts.
+        # Test that the scanner can continue scanning where it was left when zebra-scanner restarts.
         check_directory_files "${ZEBRA_CACHED_STATE_DIR}"
-        run_cargo_test "shielded-scan" "scan_start_where_left"
-      
+        exec cargo test --locked --release --features "zebra-test" --package zebra-scan -- --nocapture --include-ignored scan_start_where_left
+
       elif [[ "${TEST_SCAN_TASK_COMMANDS}" -eq "1" ]]; then
-        # Test that the scanner can continue scanning where it was left when zebrad restarts.
+        # Test that the scan task commands are working.
         check_directory_files "${ZEBRA_CACHED_STATE_DIR}"
-        run_cargo_test "shielded-scan" "scan_task_commands"
+        exec cargo test --locked --release --features "zebra-test" --package zebra-scan -- --nocapture --include-ignored scan_task_commands
 
       else
           exec "$@"

diff --git a/zebra-scan/Cargo.toml b/zebra-scan/Cargo.toml
@@ -23,6 +23,15 @@ required-features = ["proptest-impl"]
 name = "zebra-scanner"
 path = "src/bin/scanner/main.rs"
 
+[[bin]]
+name = "scanning-results-reader"
+path = "src/bin/scanning-results-reader/main.rs"
+required-features = ["results-reader"]
+
+[[bin]] # Bin to run zebrad, used in scanner tests
+name = "zebrad-for-scanner"
+path = "src/bin/zebrad-for-scanner/main.rs"
+
 [features]
 
 # Production features that activate extra dependencies, or extra features in dependencies
@@ -44,7 +53,10 @@ proptest-impl = [
 ]
 
 # Needed for the zebra-scanner binary.
-shielded-scan = []
+results-reader = [
+  "jsonrpc",
+  "hex"
+]
 
 [dependencies]
 
@@ -91,6 +103,11 @@ structopt = "0.3.26"
 lazy_static = "1.4.0"
 serde_json = "1.0.117"
 
+jsonrpc = { version = "0.18.0", optional = true }
+hex = { version = "0.4.3", optional = true }
+
+zebrad = { path = "../zebrad", version = "1.8.0" }
+
 [dev-dependencies]
 insta = { version = "1.39.0", features = ["ron", "redactions"] }
 tokio = { version = "1.37.0", features = ["test-util"] }
@@ -102,7 +119,11 @@ ff = "0.13.0"
 group = "0.13.0"
 jubjub = "0.10.0"
 rand = "0.8.5"
+tempfile = "3.10.1"
 zcash_note_encryption = "0.4.0"
+toml = "0.8.13"
+tonic = "0.11.0"
 
 zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38", features = ["proptest-impl"] }
 zebra-test = { path = "../zebra-test", version = "1.0.0-beta.38" }
+
diff --git a/...s/src/bin/scanning-results-reader/main.rs → ...n/src/bin/scanning-results-reader/main.rs b/...s/src/bin/scanning-results-reader/main.rs → ...n/src/bin/scanning-results-reader/main.rs
diff --git a/zebra-scan/src/bin/zebrad-for-scanner/main.rs b/zebra-scan/src/bin/zebrad-for-scanner/main.rs
@@ -0,0 +1,8 @@
+//! Main entry point for Zebrad, to be used in zebra-scan tests
+
+use zebrad::application::{boot, APPLICATION};
+
+/// Process entry point for `zebrad`
+fn main() {
+    boot(&APPLICATION);
+}
diff --git a/...ommon/shielded_scan/scan_task_commands.rs → zebra-scan/tests/scan_task_commands.rs b/...ommon/shielded_scan/scan_task_commands.rs → zebra-scan/tests/scan_task_commands.rs
@@ -5,12 +5,14 @@
 //!
 //! export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state"
 //! cargo test scan_task_commands --features="shielded-scan" -- --ignored --nocapture
+#![allow(dead_code)]
 
 use std::{fs, time::Duration};
 
 use color_eyre::{eyre::eyre, Result};
-
 use tokio::sync::mpsc::error::TryRecvError;
+use tower::{util::BoxService, Service};
+
 use zebra_chain::{
     block::Height,
     chain_tip::ChainTip,
@@ -20,13 +22,12 @@ use zebra_chain::{
 use zebra_scan::{
     service::ScanTask,
     storage::{db::SCANNER_DATABASE_KIND, Storage},
-    tests::ZECPAGES_SAPLING_VIEWING_KEY,
 };
 
-use crate::common::{
-    cached_state::start_state_service_with_cache_dir, launch::can_spawn_zebrad_for_test_type,
-    test_type::TestType,
-};
+use zebra_state::{ChainTipChange, LatestChainTip};
+
+pub type BoxStateService =
+    BoxService<zebra_state::Request, zebra_state::Response, zebra_state::BoxError>;
 
 /// The minimum required tip height for the cached state in this test.
 const REQUIRED_MIN_TIP_HEIGHT: Height = Height(1_000_000);
@@ -38,6 +39,9 @@ const WAIT_FOR_RESULTS_DURATION: Duration = Duration::from_secs(60);
 /// A block height where a scan result can be found with the [`ZECPAGES_SAPLING_VIEWING_KEY`]
 const EXPECTED_RESULT_HEIGHT: Height = Height(780_532);
 
+/// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo)
+const ZECPAGES_SAPLING_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz";
+
 /// Initialize Zebra's state service with a cached state, then:
 /// - Start the scan task,
 /// - Add a new key,
@@ -48,47 +52,39 @@ const EXPECTED_RESULT_HEIGHT: Height = Height(780_532);
 pub(crate) async fn run() -> Result<()> {
     let _init_guard = zebra_test::init();
 
-    let test_type = TestType::UpdateZebraCachedStateNoRpc;
-    let test_name = "scan_task_commands";
     let network = Network::Mainnet;
 
-    // Skip the test unless the user specifically asked for it and there is a zebrad_state_path
-    if !can_spawn_zebrad_for_test_type(test_name, test_type, true) {
-        return Ok(());
-    }
+    // Logs the network as zebrad would as part of the metadata when starting up.
+    // This is currently needed for the 'Check startup logs' step in CI to pass.
+    tracing::info!("Zcash network: {network}");
 
-    tracing::info!(
-        ?network,
-        ?test_type,
-        "running scan_subscribe_results test using zebra state service",
-    );
+    let zebrad_state_path = match std::env::var_os("ZEBRA_CACHED_STATE_DIR") {
+        None => {
+            tracing::error!("ZEBRA_CACHED_STATE_DIR is not set");
+            return Ok(());
+        }
+        Some(path) => std::path::PathBuf::from(path),
+    };
 
-    let zebrad_state_path = test_type
-        .zebrad_state_path(test_name)
-        .expect("already checked that there is a cached state path");
+    // Remove the scan directory before starting.
+    let scan_db_path = zebrad_state_path.join(SCANNER_DATABASE_KIND);
+    fs::remove_dir_all(std::path::Path::new(&scan_db_path)).ok();
 
     let mut scan_config = zebra_scan::Config::default();
     scan_config
         .db_config_mut()
         .cache_dir
         .clone_from(&zebrad_state_path);
 
-    // Logs the network as zebrad would as part of the metadata when starting up.
-    // This is currently needed for the 'Check startup logs' step in CI to pass.
-    tracing::info!("Zcash network: {network}");
-
-    // Remove the scan directory before starting.
-    let scan_db_path = zebrad_state_path.join(SCANNER_DATABASE_KIND);
-    fs::remove_dir_all(std::path::Path::new(&scan_db_path)).ok();
-
     let (_state_service, _read_state_service, latest_chain_tip, chain_tip_change) =
         start_state_service_with_cache_dir(&network, zebrad_state_path.clone()).await?;
 
     let state_config = zebra_state::Config {
         cache_dir: zebrad_state_path.clone(),
         ..zebra_state::Config::default()
     };
-    let (read_state, _db, _) = zebra_state::init_read_only(state_config, &network);
+
+    let (read_state, _, _) = zebra_state::init_read_only(state_config.clone(), &network);
 
     let chain_tip_height = latest_chain_tip
         .best_tip_height()
@@ -158,11 +154,32 @@ pub(crate) async fn run() -> Result<()> {
                     TryRecvError::Disconnected,
                     "any result senders should have been dropped"
                 );
-
                 break;
             }
         }
     }
 
     Ok(())
 }
+
+pub async fn start_state_service_with_cache_dir(
+    network: &Network,
+    cache_dir: impl Into<std::path::PathBuf>,
+) -> Result<(
+    BoxStateService,
+    impl Service<
+        zebra_state::ReadRequest,
+        Response = zebra_state::ReadResponse,
+        Error = zebra_state::BoxError,
+    >,
+    LatestChainTip,
+    ChainTipChange,
+)> {
+    let config = zebra_state::Config {
+        cache_dir: cache_dir.into(),
+        ..zebra_state::Config::default()
+    };
+
+    // These tests don't need UTXOs to be verified efficiently, because they use cached states.
+    Ok(zebra_state::init(config, network, Height::MAX, 0))
+}