From 2594a8ec88f193841d7eace7722ed3ad6e8de253 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 08:54:59 +1000 Subject: [PATCH 01/55] Patch equihash to use the solver branch --- Cargo.lock | 3 +-- Cargo.toml | 6 ++++++ deny.toml | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 667eb8b8d84..38a0897f1f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1339,8 +1339,7 @@ dependencies = [ [[package]] name = "equihash" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab579d7cf78477773b03e80bc2f89702ef02d7112c711d54ca93dcdce68533d5" +source = "git+https://github.com/ZcashFoundation/librustzcash.git?branch=equihash-solver-tromp#da26c34772f4922eb13b4a1e7d88a969bbcf6a91" dependencies = [ "blake2b_simd", "byteorder", diff --git a/Cargo.toml b/Cargo.toml index e2c5e03372a..7f8ba72f03d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,3 +82,9 @@ panic = "abort" # - add "-flto=thin" to all C/C++ code builds # - see https://doc.rust-lang.org/rustc/linker-plugin-lto.html#cc-code-as-a-dependency-in-rust lto = "thin" + +[patch.crates-io] +# Use the the working solver branch: +equihash = { git = 'https://github.com/ZcashFoundation/librustzcash.git', branch = 'equihash-solver-tromp' } +# or during development, use the locally checked out and modified version of equihash: +#equihash = { path = '../librustzcash/components/equihash' } diff --git a/deny.toml b/deny.toml index 428084835be..42b655a8a60 100644 --- a/deny.toml +++ b/deny.toml @@ -137,6 +137,8 @@ unknown-git = "deny" allow-registry = ["https://github.com/rust-lang/crates.io-index"] # List of URLs for allowed Git repositories allow-git = [ + # TODO: remove this after the equihash solver branch is merged and released + "https://github.com/ZcashFoundation/librustzcash.git" ] [sources.allow-org] From f2b5d9c2fbd60b8506bd0e14494dd339729126e7 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 08:59:40 +1000 Subject: [PATCH 02/55] Add an internal-miner feature and set up its dependencies --- Cargo.lock | 1 + Cargo.toml | 6 ++++++ zebra-chain/Cargo.toml | 8 ++++++++ zebrad/Cargo.toml | 7 +++++++ 4 files changed, 22 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 38a0897f1f3..2f114f5cd53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1343,6 +1343,7 @@ source = "git+https://github.com/ZcashFoundation/librustzcash.git?branch=equihas dependencies = [ "blake2b_simd", "byteorder", + "cc", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7f8ba72f03d..92ef0d072ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,12 @@ opt-level = 3 [profile.dev.package.bls12_381] opt-level = 3 +[profile.dev.package.byteorder] +opt-level = 3 + +[profile.dev.package.equihash] +opt-level = 3 + [profile.dev.package.zcash_proofs] opt-level = 3 diff --git a/zebra-chain/Cargo.toml b/zebra-chain/Cargo.toml index a68ab42581c..e1a0e8b8c20 100644 --- a/zebra-chain/Cargo.toml +++ b/zebra-chain/Cargo.toml @@ -34,6 +34,14 @@ getblocktemplate-rpcs = [ "zcash_address", ] +# Experimental internal miner support +internal-miner = [ + "equihash/solver", + # TODO: enable common code using either "internal-miner" or "getblocktemplate-rpcs", + # and remove this feature dependency + "getblocktemplate-rpcs", +] + # Experimental elasticsearch support elasticsearch = [] diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index 9df4307ad80..ec05f5413f7 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -68,6 +68,13 @@ getblocktemplate-rpcs = [ "zebra-chain/getblocktemplate-rpcs", ] +# Experimental internal miner support +internal-miner = [ + "zebra-chain/internal-miner", + # TODO: move common code into zebra-chain or zebra-node-services and remove the RPC dependency + "getblocktemplate-rpcs", +] + # Experimental shielded blockchain scanning shielded-scan = ["zebra-scan"] From de5091d8831079a1b623b49db42cd1159987603c Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 09:00:04 +1000 Subject: [PATCH 03/55] Remove 'Experimental' from mining RPC docs --- zebra-chain/Cargo.toml | 2 +- zebra-consensus/Cargo.toml | 2 +- zebra-node-services/Cargo.toml | 2 +- zebra-rpc/Cargo.toml | 2 +- zebra-state/Cargo.toml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/zebra-chain/Cargo.toml b/zebra-chain/Cargo.toml index e1a0e8b8c20..0529a3db13c 100644 --- a/zebra-chain/Cargo.toml +++ b/zebra-chain/Cargo.toml @@ -29,7 +29,7 @@ async-error = [ "tokio", ] -# Experimental mining RPC support +# Mining RPC support getblocktemplate-rpcs = [ "zcash_address", ] diff --git a/zebra-consensus/Cargo.toml b/zebra-consensus/Cargo.toml index c230921c34e..9782b35e896 100644 --- a/zebra-consensus/Cargo.toml +++ b/zebra-consensus/Cargo.toml @@ -24,7 +24,7 @@ progress-bar = [ "zebra-state/progress-bar", ] -# Experimental mining RPC support +# Mining RPC support getblocktemplate-rpcs = [ "zebra-state/getblocktemplate-rpcs", "zebra-node-services/getblocktemplate-rpcs", diff --git a/zebra-node-services/Cargo.toml b/zebra-node-services/Cargo.toml index d4fafd956fb..a32fc55a6a2 100644 --- a/zebra-node-services/Cargo.toml +++ b/zebra-node-services/Cargo.toml @@ -19,7 +19,7 @@ default = [] # Production features that activate extra dependencies, or extra features in dependencies -# Experimental mining RPC support +# Mining RPC support getblocktemplate-rpcs = [ "zebra-chain/getblocktemplate-rpcs", ] diff --git a/zebra-rpc/Cargo.toml b/zebra-rpc/Cargo.toml index 95405ec29d3..21159478a1b 100644 --- a/zebra-rpc/Cargo.toml +++ b/zebra-rpc/Cargo.toml @@ -19,7 +19,7 @@ default = [] # Production features that activate extra dependencies, or extra features in dependencies -# Experimental mining RPC support +# Mining RPC support getblocktemplate-rpcs = [ "rand", "zcash_address", diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml index f19b427f5dc..3439e05b5b1 100644 --- a/zebra-state/Cargo.toml +++ b/zebra-state/Cargo.toml @@ -22,7 +22,7 @@ progress-bar = [ "howudoin", ] -# Experimental mining RPC support +# Mining RPC support getblocktemplate-rpcs = [ "zebra-chain/getblocktemplate-rpcs", ] From 1cdb65b550c222d209e2f6fd1d1203d411ce4bf7 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 09:03:27 +1000 Subject: [PATCH 04/55] Fix a nightly clippy::question_mark lint --- zebrad/src/application.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/zebrad/src/application.rs b/zebrad/src/application.rs index 24c8ac9b784..96264f2f051 100644 --- a/zebrad/src/application.rs +++ b/zebrad/src/application.rs @@ -78,9 +78,7 @@ fn vergen_build_version() -> Option { // - optional pre-release: `-`tag[`.`tag ...] // - optional build: `+`tag[`.`tag ...] // change the git describe format to the semver 2.0 format - let Some(vergen_git_describe) = VERGEN_GIT_DESCRIBE else { - return None; - }; + let vergen_git_describe = VERGEN_GIT_DESCRIBE?; // `git describe` uses "dirty" for uncommitted changes, // but users won't understand what that means. @@ -90,10 +88,7 @@ fn vergen_build_version() -> Option { let mut vergen_git_describe = vergen_git_describe.split('-').peekable(); // Check the "version core" part. - let version = vergen_git_describe.next(); - let Some(mut version) = version else { - return None; - }; + let mut version = vergen_git_describe.next()?; // strip the leading "v", if present. version = version.strip_prefix('v').unwrap_or(version); From 5e42efece567db87d546c950f573cbf3fc3a8301 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 09:22:24 +1000 Subject: [PATCH 05/55] Move a byte array utility function to zebra-chain --- zebra-chain/src/primitives.rs | 2 ++ zebra-chain/src/primitives/byte_array.rs | 14 ++++++++++++++ zebra-state/src/service/finalized_state/disk_db.rs | 9 +++------ 3 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 zebra-chain/src/primitives/byte_array.rs diff --git a/zebra-chain/src/primitives.rs b/zebra-chain/src/primitives.rs index 9b5056bc620..d074463286e 100644 --- a/zebra-chain/src/primitives.rs +++ b/zebra-chain/src/primitives.rs @@ -12,6 +12,8 @@ mod address; #[cfg(feature = "getblocktemplate-rpcs")] pub use address::Address; +pub mod byte_array; + pub use ed25519_zebra as ed25519; pub use reddsa; pub use redjubjub; diff --git a/zebra-chain/src/primitives/byte_array.rs b/zebra-chain/src/primitives/byte_array.rs new file mode 100644 index 00000000000..7484864e8d0 --- /dev/null +++ b/zebra-chain/src/primitives/byte_array.rs @@ -0,0 +1,14 @@ +//! Functions for modifying byte arrays. + +/// Increments `byte_array` by 1, interpreting it as a big-endian integer. +/// If the big-endian integer overflowed, sets all the bytes to zero, and returns `true`. +pub fn increment_big_endian(byte_array: &mut [u8]) -> bool { + // Increment the last byte in the array that is less than u8::MAX, and clear any bytes after it + // to increment the next value in big-endian (lexicographic) order. + let is_wrapped_overflow = byte_array.iter_mut().rev().all(|v| { + *v = v.wrapping_add(1); + v == &0 + }); + + is_wrapped_overflow +} diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index 6bcf15d569d..825edbc6657 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -23,7 +23,7 @@ use rlimit::increase_nofile_limit; use rocksdb::ReadOptions; use semver::Version; -use zebra_chain::parameters::Network; +use zebra_chain::{parameters::Network, primitives::byte_array::increment_big_endian}; use crate::{ service::finalized_state::disk_format::{FromDisk, IntoDisk}, @@ -640,11 +640,8 @@ impl DiskDb { Included(mut bound) => { // Increment the last byte in the upper bound that is less than u8::MAX, and // clear any bytes after it to increment the next key in lexicographic order - // (next big-endian number) this Vec represents to RocksDB. - let is_wrapped_overflow = bound.iter_mut().rev().all(|v| { - *v = v.wrapping_add(1); - v == &0 - }); + // (next big-endian number). RocksDB uses lexicographic order for keys. + let is_wrapped_overflow = increment_big_endian(&mut bound); if is_wrapped_overflow { bound.insert(0, 0x01) From 9de4fc31950e7734c0c7ffb84ba4d3bc7e890230 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 09:59:45 +1000 Subject: [PATCH 06/55] fixup! Add an internal-miner feature and set up its dependencies --- zebra-chain/Cargo.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/zebra-chain/Cargo.toml b/zebra-chain/Cargo.toml index 0529a3db13c..328d23bbcb2 100644 --- a/zebra-chain/Cargo.toml +++ b/zebra-chain/Cargo.toml @@ -37,9 +37,6 @@ getblocktemplate-rpcs = [ # Experimental internal miner support internal-miner = [ "equihash/solver", - # TODO: enable common code using either "internal-miner" or "getblocktemplate-rpcs", - # and remove this feature dependency - "getblocktemplate-rpcs", ] # Experimental elasticsearch support From 4b2bc085cb48325398ad0a10dc1ff176f00a0189 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 10:00:14 +1000 Subject: [PATCH 07/55] Add an equihash::Solution::solve() method with difficulty checks --- zebra-chain/src/block/header.rs | 5 ++ zebra-chain/src/work/equihash.rs | 93 +++++++++++++++++++++++++++----- 2 files changed, 84 insertions(+), 14 deletions(-) diff --git a/zebra-chain/src/block/header.rs b/zebra-chain/src/block/header.rs index c59d3972ff5..1bbec3b471c 100644 --- a/zebra-chain/src/block/header.rs +++ b/zebra-chain/src/block/header.rs @@ -123,6 +123,11 @@ impl Header { ))? } } + + /// Compute the hash of this header. + pub fn hash(&self) -> Hash { + Hash::from(self) + } } /// A header with a count of the number of transactions in its block. diff --git a/zebra-chain/src/work/equihash.rs b/zebra-chain/src/work/equihash.rs index f65438a5314..37c64fadecc 100644 --- a/zebra-chain/src/work/equihash.rs +++ b/zebra-chain/src/work/equihash.rs @@ -21,7 +21,7 @@ pub struct Error(#[from] equihash::Error); /// The size of an Equihash solution in bytes (always 1344). pub(crate) const SOLUTION_SIZE: usize = 1344; -/// Equihash Solution. +/// Equihash Solution in compressed format. /// /// A wrapper around [u8; 1344] because Rust doesn't implement common /// traits like `Debug`, `Clone`, etc for collections like array @@ -60,11 +60,87 @@ impl Solution { Ok(()) } - #[cfg(feature = "getblocktemplate-rpcs")] + /// Returns a [`Solution`] containing the bytes from `solution`. + /// Returns an error if `solution` is the wrong length. + pub fn from_bytes(solution: Vec) -> Result { + if solution.len() != SOLUTION_SIZE { + return Err(SerializationError::Parse( + "incorrect equihash solution size", + )); + } + + let mut bytes = [0; SOLUTION_SIZE]; + // Won't panic, because we just checked the length. + bytes.copy_from_slice(&solution); + + Ok(Self(bytes)) + } + /// Returns a [`Solution`] of `[0; SOLUTION_SIZE]` to be used in block proposals. + #[cfg(feature = "getblocktemplate-rpcs")] pub fn for_proposal() -> Self { Self([0; SOLUTION_SIZE]) } + + /// Mines and returns one or more [`Solution`]s based on a template `header`. + /// The returned header contains a valid `nonce` and `solution`. + /// + /// The `nonce` in the header template is taken as the starting nonce. If you are running multiple + /// solvers at the same time, start them with different nonces. + /// The `solution` in the header template is ignored. + /// + /// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core while running. + /// It can run for minutes or hours if the network difficulty is high. + #[cfg(feature = "internal-miner")] + pub fn solve(mut header: Header) -> Header { + let mut input = Vec::new(); + header + .zcash_serialize(&mut input) + .expect("serialization into a vec can't fail"); + let input = &input[0..Solution::INPUT_LENGTH]; + + loop { + let solutions = equihash::tromp::solve_200_9_compressed(input, || { + // This skips the first nonce, which doesn't matter in practice. + Self::next_nonce(&mut header.nonce); + Some(*header.nonce) + }); + + for solution in solutions { + header.solution = Self::from_bytes(solution) + .expect("unexpected invalid solution: incorrect length"); + if Self::difficulty_is_valid(&header) { + return header; + } + } + } + } + + /// Modifies `nonce` to be the next integer in big-endian order. + /// Wraps to zero if the next nonce would overflow. + #[cfg(feature = "internal-miner")] + fn next_nonce(nonce: &mut [u8; 32]) { + let _ignore_overflow = crate::primitives::byte_array::increment_big_endian(&mut nonce[..]); + } + + /// Returns `true` if the `nonce` and `solution` in `header` meet the difficulty threshold. + /// + /// Assumes that the difficulty threshold in the header is valid. + #[cfg(feature = "internal-miner")] + fn difficulty_is_valid(header: &Header) -> bool { + // Simplified from zebra_consensus::block::check::difficulty_is_valid(). + let difficulty_threshold = header + .difficulty_threshold + .to_expanded() + .expect("unexpected invalid header template: invalid difficulty threshold"); + + // TODO: avoid calculating this hash multiple times + let hash = header.hash(); + + // Note: this comparison is a u256 integer comparison, like zcashd and bitcoin. Greater + // values represent *less* work. + hash <= difficulty_threshold + } } impl PartialEq for Solution { @@ -109,17 +185,6 @@ impl ZcashSerialize for Solution { impl ZcashDeserialize for Solution { fn zcash_deserialize(mut reader: R) -> Result { let solution: Vec = (&mut reader).zcash_deserialize_into()?; - - if solution.len() != SOLUTION_SIZE { - return Err(SerializationError::Parse( - "incorrect equihash solution size", - )); - } - - let mut bytes = [0; SOLUTION_SIZE]; - // Won't panic, because we just checked the length. - bytes.copy_from_slice(&solution); - - Ok(Self(bytes)) + Self::from_bytes(solution) } } From 6cbe8ec52125c762aa25a45d88f361744f432a84 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 10:26:19 +1000 Subject: [PATCH 08/55] Check solution is valid before returning it --- zebra-chain/src/work/equihash.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zebra-chain/src/work/equihash.rs b/zebra-chain/src/work/equihash.rs index 37c64fadecc..ff70e87c267 100644 --- a/zebra-chain/src/work/equihash.rs +++ b/zebra-chain/src/work/equihash.rs @@ -109,6 +109,12 @@ impl Solution { for solution in solutions { header.solution = Self::from_bytes(solution) .expect("unexpected invalid solution: incorrect length"); + // TODO: only run this redundant check in tests + header + .solution + .check(&header) + .expect("unexpected invalid solution: invalid solution for header"); + if Self::difficulty_is_valid(&header) { return header; } From 2ddc058eda34069a01ca0225f9b305ea39c05d50 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 10:33:00 +1000 Subject: [PATCH 09/55] Add a TODO to check for peers before mining --- zebra-rpc/src/methods/get_block_template_rpcs.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index e99258ad78f..198c4b6dfd0 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -498,6 +498,10 @@ where // - add `async changed()` method to ChainSyncStatus (like `ChainTip`) check_synced_to_tip(network, latest_chain_tip.clone(), sync_status.clone())?; + // TODO: return an error if we have no peers, like `zcashd` does, + // and add a developer config that mines regardless of how many peers we have. + // https://github.com/zcash/zcash/blob/6fdd9f1b81d3b228326c9826fa10696fc516444b/src/miner.cpp#L865-L880 + // We're just about to fetch state data, then maybe wait for any changes. // Mark all the changes before the fetch as seen. // Changes are also ignored in any clones made after the mark. From a2a49b56d2c863702b341e2857e181843175f184 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 10:42:20 +1000 Subject: [PATCH 10/55] Move config validation into GetBlockTemplateRpcImpl::new() --- zebra-rpc/src/methods/get_block_template_rpcs.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index 198c4b6dfd0..a27affe9b9b 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -329,6 +329,17 @@ where sync_status: SyncStatus, address_book: AddressBook, ) -> Self { + // Prevent loss of miner funds due to an unsupported or incorrect address type. + if let Some(miner_address) = mining_config.miner_address { + assert_eq!( + miner_address.network(), + network, + "incorrect miner address config: {miner_address} \ + network.network {network} and miner address network {} must match", + miner_address.network(), + ); + } + // A limit on the configured extra coinbase data, regardless of the current block height. // This is different from the consensus rule, which limits the total height + data. const EXTRA_COINBASE_DATA_LIMIT: usize = From fe2b65f0b95c13302b0f18caab8a2c03b1cba9f4 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 10:42:39 +1000 Subject: [PATCH 11/55] fixup! fixup! Add an internal-miner feature and set up its dependencies --- zebrad/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index ec05f5413f7..da8bde694d3 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -72,7 +72,7 @@ getblocktemplate-rpcs = [ internal-miner = [ "zebra-chain/internal-miner", # TODO: move common code into zebra-chain or zebra-node-services and remove the RPC dependency - "getblocktemplate-rpcs", + "zebra-rpc/getblocktemplate-rpcs", ] # Experimental shielded blockchain scanning From 92f4783765fb455ad7fb48153531094b848300d4 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 11:04:58 +1000 Subject: [PATCH 12/55] Use the same generic constraints for GetBlockTemplateRpcImpl struct and impls --- .../src/methods/get_block_template_rpcs.rs | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index a27affe9b9b..7211a4e42df 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -232,22 +232,29 @@ pub struct GetBlockTemplateRpcImpl< AddressBook, > where Mempool: Service< - mempool::Request, - Response = mempool::Response, - Error = zebra_node_services::BoxError, - >, + mempool::Request, + Response = mempool::Response, + Error = zebra_node_services::BoxError, + > + 'static, + Mempool::Future: Send, State: Service< - zebra_state::ReadRequest, - Response = zebra_state::ReadResponse, - Error = zebra_state::BoxError, - >, + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + Sync + + 'static, + >::Future: Send, + Tip: ChainTip + Clone + Send + Sync + 'static, BlockVerifierRouter: Service + Clone + Send + Sync + 'static, + >::Future: Send, SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, - AddressBook: AddressBookPeers, + AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, { // Configuration // @@ -296,6 +303,7 @@ where Response = mempool::Response, Error = zebra_node_services::BoxError, > + 'static, + Mempool::Future: Send, State: Service< zebra_state::ReadRequest, Response = zebra_state::ReadResponse, @@ -304,12 +312,14 @@ where + Send + Sync + 'static, + >::Future: Send, Tip: ChainTip + Clone + Send + Sync + 'static, BlockVerifierRouter: Service + Clone + Send + Sync + 'static, + >::Future: Send, SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, { From 76de9fec12ba0e9db99414d0beaadba233ba955e Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 11:05:22 +1000 Subject: [PATCH 13/55] Start adding an internal miner component --- zebrad/src/components.rs | 3 ++ zebrad/src/components/miner.rs | 67 ++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 zebrad/src/components/miner.rs diff --git a/zebrad/src/components.rs b/zebrad/src/components.rs index 5daf65a4a79..43b051f1209 100644 --- a/zebrad/src/components.rs +++ b/zebrad/src/components.rs @@ -16,5 +16,8 @@ pub mod tokio; #[allow(missing_docs)] pub mod tracing; +#[cfg(feature = "internal-miner")] +pub mod miner; + pub use inbound::Inbound; pub use sync::ChainSync; diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs new file mode 100644 index 00000000000..8dbfb87c8ab --- /dev/null +++ b/zebrad/src/components/miner.rs @@ -0,0 +1,67 @@ +//! Internal mining in Zebra. +//! +//! # TODO +//! - pause mining if we have no peers, like `zcashd` does, +//! and add a developer config that mines regardless of how many peers we have. +//! +//! - move common code into zebra-chain or zebra-node-services and remove the RPC dependency. + +use color_eyre::Report; +use tower::Service; + +use zebra_chain::{block, chain_sync_status::ChainSyncStatus, chain_tip::ChainTip}; +use zebra_network::AddressBookPeers; +use zebra_node_services::mempool; +use zebra_rpc::methods::{ + get_block_template_rpcs::get_block_template::{ + self, GetBlockTemplateCapability::*, GetBlockTemplateRequestMode::*, + }, + GetBlockTemplateRpc, GetBlockTemplateRpcImpl, +}; + +/// Runs a single mining thread to generate blocks, calculate equihash solutions, and submit valid +/// blocks to Zebra's block validator. +#[instrument(skip(rpc))] +pub async fn run_mining_solver( + rpc: GetBlockTemplateRpcImpl, +) -> Result<(), Report> +where + Mempool: Service< + mempool::Request, + Response = mempool::Response, + Error = zebra_node_services::BoxError, + > + 'static, + Mempool::Future: Send, + State: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + Sync + + 'static, + >::Future: Send, + Tip: ChainTip + Clone + Send + Sync + 'static, + BlockVerifierRouter: Service + + Clone + + Send + + Sync + + 'static, + >::Future: Send, + SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, + AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, +{ + // Pass the correct arguments, even if Zebra currently ignores them. + let mut long_poll_id = None; + let mut parameters = get_block_template::JsonParameters { + mode: Template, + data: None, + capabilities: vec![LongPoll, CoinbaseTxn], + long_poll_id, + _work_id: None, + }; + + let template = rpc.get_block_template(Some(parameters)).await; + + Ok(()) +} From ba5cd316830958b736f682fd1ae37893e097b527 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 11:23:55 +1000 Subject: [PATCH 14/55] Add the miner task to the start command --- zebrad/src/commands/start.rs | 42 ++++++++++++++-- zebrad/src/components/miner.rs | 92 ++++++++++++++++++++++++++++++++-- 2 files changed, 125 insertions(+), 9 deletions(-) diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index ec59f409c5e..4d5f33b1b37 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -220,10 +220,10 @@ impl StartCmd { build_version(), user_agent(), mempool.clone(), - read_only_state_service, - block_verifier_router, + read_only_state_service.clone(), + block_verifier_router.clone(), sync_status.clone(), - address_book, + address_book.clone(), latest_chain_tip.clone(), config.network.network, ); @@ -267,7 +267,8 @@ impl StartCmd { // Spawn never ending end of support task. info!("spawning end of support checking task"); let end_of_support_task_handle = tokio::spawn( - sync::end_of_support::start(config.network.network, latest_chain_tip).in_current_span(), + sync::end_of_support::start(config.network.network, latest_chain_tip.clone()) + .in_current_span(), ); // Give the inbound service more time to clear its queue, @@ -281,7 +282,7 @@ impl StartCmd { &config.mempool, peer_set, mempool.clone(), - sync_status, + sync_status.clone(), chain_tip_change.clone(), ); @@ -308,6 +309,31 @@ impl StartCmd { let scan_task_handle: tokio::task::JoinHandle> = tokio::spawn(std::future::pending().in_current_span()); + // And finally, spawn the internal Zcash miner, if it is enabled. + // + // TODO: add a config to enable the miner rather than a feature. + #[cfg(feature = "internal-miner")] + let miner_task_handle = { + info!("spawning Zcash miner"); + let rpc = zebra_rpc::methods::get_block_template_rpcs::GetBlockTemplateRpcImpl::new( + config.network.network, + config.mining.clone(), + mempool, + read_only_state_service, + latest_chain_tip, + block_verifier_router, + sync_status, + address_book, + ); + + crate::components::miner::spawn_init(&config.mining, rpc) + }; + + #[cfg(not(feature = "internal-miner"))] + // Spawn a dummy miner task which doesn't do anything and never finishes. + let miner_task_handle: tokio::task::JoinHandle> = + tokio::spawn(std::future::pending().in_current_span()); + info!("spawned initial Zebra tasks"); // TODO: put tasks into an ongoing FuturesUnordered and a startup FuturesUnordered? @@ -323,6 +349,7 @@ impl StartCmd { pin!(progress_task_handle); pin!(end_of_support_task_handle); pin!(scan_task_handle); + pin!(miner_task_handle); // startup tasks let BackgroundTaskHandles { @@ -413,6 +440,10 @@ impl StartCmd { scan_result = &mut scan_task_handle => scan_result .expect("unexpected panic in the scan task") .map(|_| info!("scan task exited")), + + miner_result = &mut miner_task_handle => miner_result + .expect("unexpected panic in the miner task") + .map(|_| info!("miner task exited")), }; // Stop Zebra if a task finished and returned an error, @@ -439,6 +470,7 @@ impl StartCmd { progress_task_handle.abort(); end_of_support_task_handle.abort(); scan_task_handle.abort(); + miner_task_handle.abort(); // startup tasks state_checkpoint_verify_handle.abort(); diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index 8dbfb87c8ab..d1b29f0de72 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -7,18 +7,102 @@ //! - move common code into zebra-chain or zebra-node-services and remove the RPC dependency. use color_eyre::Report; +use tokio::task::JoinHandle; use tower::Service; +use tracing::Instrument; use zebra_chain::{block, chain_sync_status::ChainSyncStatus, chain_tip::ChainTip}; use zebra_network::AddressBookPeers; use zebra_node_services::mempool; -use zebra_rpc::methods::{ - get_block_template_rpcs::get_block_template::{ - self, GetBlockTemplateCapability::*, GetBlockTemplateRequestMode::*, +use zebra_rpc::{ + config::mining::Config, + methods::{ + get_block_template_rpcs::get_block_template::{ + self, GetBlockTemplateCapability::*, GetBlockTemplateRequestMode::*, + }, + GetBlockTemplateRpc, GetBlockTemplateRpcImpl, }, - GetBlockTemplateRpc, GetBlockTemplateRpcImpl, }; +/// Initialize the miner based on its config, and spawn a task for it. +/// +/// TODO: +/// - add a miner config for the number of solver threads. +/// - add a test for this function. +pub fn spawn_init( + config: &Config, + rpc: GetBlockTemplateRpcImpl, +) -> JoinHandle> +where + Mempool: Service< + mempool::Request, + Response = mempool::Response, + Error = zebra_node_services::BoxError, + > + 'static, + Mempool::Future: Send, + State: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + Sync + + 'static, + >::Future: Send, + Tip: ChainTip + Clone + Send + Sync + 'static, + BlockVerifierRouter: Service + + Clone + + Send + + Sync + + 'static, + >::Future: Send, + SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, + AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, +{ + let config = config.clone(); + + // TODO: spawn an entirely new executor here, so mining is isolated from higher priority tasks. + tokio::spawn(init(config, rpc).in_current_span()) +} + +/// Initialize the miner based on its config. +/// +/// TODO: add a test for this function. +pub async fn init( + _config: Config, + rpc: GetBlockTemplateRpcImpl, +) -> Result<(), Report> +where + Mempool: Service< + mempool::Request, + Response = mempool::Response, + Error = zebra_node_services::BoxError, + > + 'static, + Mempool::Future: Send, + State: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + Sync + + 'static, + >::Future: Send, + Tip: ChainTip + Clone + Send + Sync + 'static, + BlockVerifierRouter: Service + + Clone + + Send + + Sync + + 'static, + >::Future: Send, + SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, + AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, +{ + // TODO: launch the configured number of solvers + // make mining threads lower priority than other threads + run_mining_solver(rpc).await +} + /// Runs a single mining thread to generate blocks, calculate equihash solutions, and submit valid /// blocks to Zebra's block validator. #[instrument(skip(rpc))] From ba00c288ce73a067c0183322abd9b1539b60427b Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 12:08:38 +1000 Subject: [PATCH 15/55] Add basic miner code --- .../types/get_block_template.rs | 18 ++++++ zebrad/src/components/miner.rs | 59 ++++++++++++++++--- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs index 617b80080c2..92ab9d00593 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs @@ -346,3 +346,21 @@ pub enum Response { /// `getblocktemplate` RPC request in proposal mode. ProposalMode(ProposalResponse), } + +impl Response { + /// Returns the inner template, if the response is in template mode. + pub fn try_into_template(self) -> Option { + match self { + Response::TemplateMode(template) => Some(*template), + Response::ProposalMode(_) => None, + } + } + + /// Returns the inner proposal, if the response is in proposal mode. + pub fn try_into_proposal(self) -> Option { + match self { + Response::TemplateMode(_) => None, + Response::ProposalMode(proposal) => Some(proposal), + } + } +} diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index d1b29f0de72..72ef6e69f13 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -6,19 +6,28 @@ //! //! - move common code into zebra-chain or zebra-node-services and remove the RPC dependency. +use std::sync::Arc; + use color_eyre::Report; use tokio::task::JoinHandle; use tower::Service; use tracing::Instrument; -use zebra_chain::{block, chain_sync_status::ChainSyncStatus, chain_tip::ChainTip}; +use zebra_chain::{ + block, chain_sync_status::ChainSyncStatus, chain_tip::ChainTip, serialization::ZcashSerialize, + work::equihash::Solution, +}; use zebra_network::AddressBookPeers; use zebra_node_services::mempool; use zebra_rpc::{ config::mining::Config, methods::{ - get_block_template_rpcs::get_block_template::{ - self, GetBlockTemplateCapability::*, GetBlockTemplateRequestMode::*, + get_block_template_rpcs::{ + get_block_template::{ + self, proposal::TimeSource, proposal_block_from_template, + GetBlockTemplateCapability::*, GetBlockTemplateRequestMode::*, + }, + types::hex_data::HexData, }, GetBlockTemplateRpc, GetBlockTemplateRpcImpl, }, @@ -26,9 +35,7 @@ use zebra_rpc::{ /// Initialize the miner based on its config, and spawn a task for it. /// -/// TODO: -/// - add a miner config for the number of solver threads. -/// - add a test for this function. +/// TODO: add a test for this function. pub fn spawn_init( config: &Config, rpc: GetBlockTemplateRpcImpl, @@ -100,7 +107,7 @@ where { // TODO: launch the configured number of solvers // make mining threads lower priority than other threads - run_mining_solver(rpc).await + run_mining_solver(rpc, 0).await } /// Runs a single mining thread to generate blocks, calculate equihash solutions, and submit valid @@ -108,6 +115,7 @@ where #[instrument(skip(rpc))] pub async fn run_mining_solver( rpc: GetBlockTemplateRpcImpl, + thread_id: usize, ) -> Result<(), Report> where Mempool: Service< @@ -145,7 +153,42 @@ where _work_id: None, }; - let template = rpc.get_block_template(Some(parameters)).await; + let template = rpc.get_block_template(Some(parameters)).await?; + let template = template + .try_into_template() + .expect("invalid RPC response: proposal in response to a template request"); + + let mut block = proposal_block_from_template(&template, TimeSource::CurTime)?; + // TODO: spawn blocking in a low priority thread + // select!{} on either a solved header or a new block template using long_poll_id + // cancel the solver if there's a new template + // use a different nonce for each solver thread + // + // TODO: Replace with Arc::unwrap_or_clone() when it stabilises: + // https://github.com/rust-lang/rust/issues/93610 + let solved_header = Solution::solve(*block.header); + + block.header = Arc::new(solved_header); + let data = block + .zcash_serialize_to_vec() + .expect("serializing to Vec never fails"); + + match rpc.submit_block(HexData(data), None).await { + Ok(success) => info!( + height = ?block.coinbase_height().expect("height checked by verifier"), + hash = ?block.hash(), + ?thread_id, + ?success, + "successfully mined a new block", + ), + Err(error) => info!( + height = ?block.coinbase_height().expect("height valid in template"), + hash = ?block.hash(), + ?thread_id, + ?error, + "validating a newly mined block failed, trying again", + ), + } Ok(()) } From 2543affae3b4aed6a906156802cd97d3bea998b6 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 12:15:07 +1000 Subject: [PATCH 16/55] Split out a method to mine one block --- zebrad/src/components/miner.rs | 51 +++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index 72ef6e69f13..75a1602e364 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -14,7 +14,10 @@ use tower::Service; use tracing::Instrument; use zebra_chain::{ - block, chain_sync_status::ChainSyncStatus, chain_tip::ChainTip, serialization::ZcashSerialize, + block::{self, Block}, + chain_sync_status::ChainSyncStatus, + chain_tip::ChainTip, + serialization::ZcashSerialize, work::equihash::Solution, }; use zebra_network::AddressBookPeers; @@ -24,7 +27,7 @@ use zebra_rpc::{ methods::{ get_block_template_rpcs::{ get_block_template::{ - self, proposal::TimeSource, proposal_block_from_template, + self, proposal::TimeSource, proposal_block_from_template, GetBlockTemplate, GetBlockTemplateCapability::*, GetBlockTemplateRequestMode::*, }, types::hex_data::HexData, @@ -35,6 +38,9 @@ use zebra_rpc::{ /// Initialize the miner based on its config, and spawn a task for it. /// +/// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core per configured +/// mining thread. +/// /// TODO: add a test for this function. pub fn spawn_init( config: &Config, @@ -74,6 +80,9 @@ where /// Initialize the miner based on its config. /// +/// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core per configured +/// mining thread. +/// /// TODO: add a test for this function. pub async fn init( _config: Config, @@ -112,6 +121,9 @@ where /// Runs a single mining thread to generate blocks, calculate equihash solutions, and submit valid /// blocks to Zebra's block validator. +/// +/// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core while running. +/// It can run for minutes or hours if the network difficulty is high. #[instrument(skip(rpc))] pub async fn run_mining_solver( rpc: GetBlockTemplateRpcImpl, @@ -158,17 +170,7 @@ where .try_into_template() .expect("invalid RPC response: proposal in response to a template request"); - let mut block = proposal_block_from_template(&template, TimeSource::CurTime)?; - // TODO: spawn blocking in a low priority thread - // select!{} on either a solved header or a new block template using long_poll_id - // cancel the solver if there's a new template - // use a different nonce for each solver thread - // - // TODO: Replace with Arc::unwrap_or_clone() when it stabilises: - // https://github.com/rust-lang/rust/issues/93610 - let solved_header = Solution::solve(*block.header); - - block.header = Arc::new(solved_header); + let block = mine_one_block(template, thread_id).await?; let data = block .zcash_serialize_to_vec() .expect("serializing to Vec never fails"); @@ -192,3 +194,26 @@ where Ok(()) } + +/// Mines a single block, calculating its equihash solutions, and submitting it to Zebra's block +/// validator. +/// +/// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core while running. +/// It can run for minutes or hours if the network difficulty is high. +#[instrument(skip(template))] +pub async fn mine_one_block(template: GetBlockTemplate, thread_id: usize) -> Result { + let mut block = proposal_block_from_template(&template, TimeSource::CurTime)?; + + // TODO: spawn blocking in a low priority thread + // select!{} on either a solved header or a new block template using long_poll_id + // cancel the solver if there's a new template + // use a different nonce for each solver thread + // + // TODO: Replace with Arc::unwrap_or_clone() when it stabilises: + // https://github.com/rust-lang/rust/issues/93610 + let solved_header = Solution::solve(*block.header); + + block.header = Arc::new(solved_header); + + Ok(block) +} From d73fcccd8dfe889e1f5a4fb354d71cf7bd4fe809 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 12:20:23 +1000 Subject: [PATCH 17/55] Spawn to a blocking thread --- zebrad/src/components/miner.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index 75a1602e364..c2cef8d3d05 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -11,12 +11,13 @@ use std::sync::Arc; use color_eyre::Report; use tokio::task::JoinHandle; use tower::Service; -use tracing::Instrument; +use tracing::{Instrument, Span}; use zebra_chain::{ block::{self, Block}, chain_sync_status::ChainSyncStatus, chain_tip::ChainTip, + diagnostic::task::WaitForPanics, serialization::ZcashSerialize, work::equihash::Solution, }; @@ -170,6 +171,9 @@ where .try_into_template() .expect("invalid RPC response: proposal in response to a template request"); + // TODO: select!{} on either a solved header or a new block template using long_poll_id + // cancel the solver if there's a new template + // use a different nonce for each solver thread let block = mine_one_block(template, thread_id).await?; let data = block .zcash_serialize_to_vec() @@ -204,14 +208,15 @@ where pub async fn mine_one_block(template: GetBlockTemplate, thread_id: usize) -> Result { let mut block = proposal_block_from_template(&template, TimeSource::CurTime)?; - // TODO: spawn blocking in a low priority thread - // select!{} on either a solved header or a new block template using long_poll_id - // cancel the solver if there's a new template - // use a different nonce for each solver thread + // TODO: spawn to a low priority thread // // TODO: Replace with Arc::unwrap_or_clone() when it stabilises: // https://github.com/rust-lang/rust/issues/93610 - let solved_header = Solution::solve(*block.header); + let span = Span::current(); + let solved_header = + tokio::task::spawn_blocking(move || span.in_scope(move || Solution::solve(*block.header))) + .wait_for_panics() + .await; block.header = Arc::new(solved_header); From 62e7670025acaed0afeb55bdeee3e3eb43460812 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 12:30:49 +1000 Subject: [PATCH 18/55] Wait until a valid template is available --- zebrad/src/components/miner.rs | 90 ++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index c2cef8d3d05..c848f25208c 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -6,10 +6,10 @@ //! //! - move common code into zebra-chain or zebra-node-services and remove the RPC dependency. -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use color_eyre::Report; -use tokio::task::JoinHandle; +use tokio::{task::JoinHandle, time::sleep}; use tower::Service; use tracing::{Instrument, Span}; @@ -37,6 +37,9 @@ use zebra_rpc::{ }, }; +/// The amount of time we wait between block template retries. +pub const BLOCK_TEMPLATE_WAIT_TIME: Duration = Duration::from_secs(20); + /// Initialize the miner based on its config, and spawn a task for it. /// /// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core per configured @@ -115,9 +118,7 @@ where SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, { - // TODO: launch the configured number of solvers - // make mining threads lower priority than other threads - run_mining_solver(rpc, 0).await + run_mining_solver(rpc).await } /// Runs a single mining thread to generate blocks, calculate equihash solutions, and submit valid @@ -128,7 +129,6 @@ where #[instrument(skip(rpc))] pub async fn run_mining_solver( rpc: GetBlockTemplateRpcImpl, - thread_id: usize, ) -> Result<(), Report> where Mempool: Service< @@ -166,49 +166,65 @@ where _work_id: None, }; - let template = rpc.get_block_template(Some(parameters)).await?; - let template = template - .try_into_template() - .expect("invalid RPC response: proposal in response to a template request"); + loop { + let template = rpc.get_block_template(Some(parameters.clone())).await; - // TODO: select!{} on either a solved header or a new block template using long_poll_id - // cancel the solver if there's a new template - // use a different nonce for each solver thread - let block = mine_one_block(template, thread_id).await?; - let data = block - .zcash_serialize_to_vec() - .expect("serializing to Vec never fails"); - - match rpc.submit_block(HexData(data), None).await { - Ok(success) => info!( - height = ?block.coinbase_height().expect("height checked by verifier"), - hash = ?block.hash(), - ?thread_id, - ?success, - "successfully mined a new block", - ), - Err(error) => info!( - height = ?block.coinbase_height().expect("height valid in template"), - hash = ?block.hash(), - ?thread_id, - ?error, - "validating a newly mined block failed, trying again", - ), - } + // Wait for the chain to sync so we get a valid template. + let Ok(template) = template else { + info!( + ?BLOCK_TEMPLATE_WAIT_TIME, + "waiting for a valid block template", + ); + sleep(BLOCK_TEMPLATE_WAIT_TIME).await; + continue; + }; + + let template = template + .try_into_template() + .expect("invalid RPC response: proposal in response to a template request"); - Ok(()) + // TODO: select!{} on either a solved header or a new block template using long_poll_id + // cancel the solver if there's a new template + // launch the configured number of solvers + let solver_thread = 0; + let block = mine_one_block(template, solver_thread).await?; + let data = block + .zcash_serialize_to_vec() + .expect("serializing to Vec never fails"); + + match rpc.submit_block(HexData(data), None).await { + Ok(success) => info!( + height = ?block.coinbase_height().expect("height checked by verifier"), + hash = ?block.hash(), + ?solver_thread, + ?success, + "successfully mined a new block", + ), + Err(error) => info!( + height = ?block.coinbase_height().expect("height valid in template"), + hash = ?block.hash(), + ?solver_thread, + ?error, + "validating a newly mined block failed, trying again", + ), + } + } } /// Mines a single block, calculating its equihash solutions, and submitting it to Zebra's block -/// validator. +/// validator. Uses a different nonce range for each `solver_thread`. /// /// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core while running. /// It can run for minutes or hours if the network difficulty is high. #[instrument(skip(template))] -pub async fn mine_one_block(template: GetBlockTemplate, thread_id: usize) -> Result { +pub async fn mine_one_block( + template: GetBlockTemplate, + solver_thread: usize, +) -> Result { let mut block = proposal_block_from_template(&template, TimeSource::CurTime)?; // TODO: spawn to a low priority thread + // use a different nonce for each solver thread // // TODO: Replace with Arc::unwrap_or_clone() when it stabilises: // https://github.com/rust-lang/rust/issues/93610 From a3d45fe8ac8f1c487d69728bc831195c7f8b863d Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 12:55:12 +1000 Subject: [PATCH 19/55] Handle shutdown --- zebra-chain/src/work/equihash.rs | 33 ++++++++++++++++----- zebrad/src/components/miner.rs | 51 +++++++++++++++++++++++--------- 2 files changed, 62 insertions(+), 22 deletions(-) diff --git a/zebra-chain/src/work/equihash.rs b/zebra-chain/src/work/equihash.rs index ff70e87c267..a58c6fe4259 100644 --- a/zebra-chain/src/work/equihash.rs +++ b/zebra-chain/src/work/equihash.rs @@ -12,12 +12,18 @@ use crate::{ }, }; -/// The error type for Equihash +/// The error type for Equihash validation. #[non_exhaustive] #[derive(Debug, thiserror::Error)] #[error("invalid equihash solution for BlockHeader")] pub struct Error(#[from] equihash::Error); +/// The error type for Equihash solving. +#[non_exhaustive] +#[derive(Copy, Clone, Debug, Eq, PartialEq, thiserror::Error)] +#[error("solver was cancelled")] +pub struct SolverCancelled; + /// The size of an Equihash solution in bytes (always 1344). pub(crate) const SOLUTION_SIZE: usize = 1344; @@ -62,7 +68,7 @@ impl Solution { /// Returns a [`Solution`] containing the bytes from `solution`. /// Returns an error if `solution` is the wrong length. - pub fn from_bytes(solution: Vec) -> Result { + pub fn from_bytes(solution: &[u8]) -> Result { if solution.len() != SOLUTION_SIZE { return Err(SerializationError::Parse( "incorrect equihash solution size", @@ -71,7 +77,7 @@ impl Solution { let mut bytes = [0; SOLUTION_SIZE]; // Won't panic, because we just checked the length. - bytes.copy_from_slice(&solution); + bytes.copy_from_slice(solution); Ok(Self(bytes)) } @@ -92,21 +98,24 @@ impl Solution { /// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core while running. /// It can run for minutes or hours if the network difficulty is high. #[cfg(feature = "internal-miner")] - pub fn solve(mut header: Header) -> Header { + #[allow(clippy::unwrap_in_result)] + pub fn solve(mut header: Header) -> Result { + use crate::shutdown::is_shutting_down; + let mut input = Vec::new(); header .zcash_serialize(&mut input) .expect("serialization into a vec can't fail"); let input = &input[0..Solution::INPUT_LENGTH]; - loop { + while !is_shutting_down() { let solutions = equihash::tromp::solve_200_9_compressed(input, || { // This skips the first nonce, which doesn't matter in practice. Self::next_nonce(&mut header.nonce); Some(*header.nonce) }); - for solution in solutions { + for solution in &solutions { header.solution = Self::from_bytes(solution) .expect("unexpected invalid solution: incorrect length"); // TODO: only run this redundant check in tests @@ -116,10 +125,18 @@ impl Solution { .expect("unexpected invalid solution: invalid solution for header"); if Self::difficulty_is_valid(&header) { - return header; + info!("found valid solution and difficulty"); + return Ok(header); } } + + info!( + solutions = ?solutions.len(), + "found valid solutions which did not pass the difficulty check" + ); } + + Err(SolverCancelled) } /// Modifies `nonce` to be the next integer in big-endian order. @@ -191,6 +208,6 @@ impl ZcashSerialize for Solution { impl ZcashDeserialize for Solution { fn zcash_deserialize(mut reader: R) -> Result { let solution: Vec = (&mut reader).zcash_deserialize_into()?; - Self::from_bytes(solution) + Self::from_bytes(&solution) } } diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index c848f25208c..29cf7c03f78 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -19,7 +19,8 @@ use zebra_chain::{ chain_tip::ChainTip, diagnostic::task::WaitForPanics, serialization::ZcashSerialize, - work::equihash::Solution, + shutdown::is_shutting_down, + work::equihash::{Solution, SolverCancelled}, }; use zebra_network::AddressBookPeers; use zebra_node_services::mempool; @@ -166,7 +167,7 @@ where _work_id: None, }; - loop { + while !is_shutting_down() { let template = rpc.get_block_template(Some(parameters.clone())).await; // Wait for the chain to sync so we get a valid template. @@ -184,44 +185,66 @@ where .expect("invalid RPC response: proposal in response to a template request"); // TODO: select!{} on either a solved header or a new block template using long_poll_id - // cancel the solver if there's a new template + // cancel the solver if there's a new template or if Zebra is shutting down // launch the configured number of solvers - let solver_thread = 0; - let block = mine_one_block(template, solver_thread).await?; + let solver_id = 0; + + let height = template.height; + let Ok(block) = mine_one_block(template, solver_id).await else { + // If the solver was cancelled, we're either shutting down, or we have a new template. + if solver_id == 0 { + info!( + ?height, + ?solver_id, + "solver cancelled: getting a new block template or shutting down" + ); + } else { + debug!( + ?height, + ?solver_id, + "solver cancelled: getting a new block template or shutting down" + ); + } + continue; + }; + let data = block .zcash_serialize_to_vec() .expect("serializing to Vec never fails"); match rpc.submit_block(HexData(data), None).await { Ok(success) => info!( - height = ?block.coinbase_height().expect("height checked by verifier"), + ?height, hash = ?block.hash(), - ?solver_thread, + ?solver_id, ?success, "successfully mined a new block", ), Err(error) => info!( - height = ?block.coinbase_height().expect("height valid in template"), + ?height, hash = ?block.hash(), - ?solver_thread, + ?solver_id, ?error, "validating a newly mined block failed, trying again", ), } } + + Ok(()) } /// Mines a single block, calculating its equihash solutions, and submitting it to Zebra's block -/// validator. Uses a different nonce range for each `solver_thread`. +/// validator. Uses a different nonce range for each `solver_id`. /// /// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core while running. /// It can run for minutes or hours if the network difficulty is high. #[instrument(skip(template))] pub async fn mine_one_block( template: GetBlockTemplate, - solver_thread: usize, -) -> Result { - let mut block = proposal_block_from_template(&template, TimeSource::CurTime)?; + solver_id: usize, +) -> Result { + let mut block = proposal_block_from_template(&template, TimeSource::CurTime) + .expect("unexpected invalid block template"); // TODO: spawn to a low priority thread // use a different nonce for each solver thread @@ -232,7 +255,7 @@ pub async fn mine_one_block( let solved_header = tokio::task::spawn_blocking(move || span.in_scope(move || Solution::solve(*block.header))) .wait_for_panics() - .await; + .await?; block.header = Arc::new(solved_header); From d9849fc8a3cac5d180c35a0cb22ef7228dbfa2f5 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 13:10:41 +1000 Subject: [PATCH 20/55] Run mining on low priority threads --- Cargo.lock | 15 +++++++++++++++ zebrad/Cargo.toml | 4 ++++ zebrad/src/components/miner.rs | 20 +++++++++++++++----- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f114f5cd53..49f432c2e83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4353,6 +4353,20 @@ dependencies = [ "syn 2.0.40", ] +[[package]] +name = "thread-priority" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b72cb4958060ee2d9540cef68bb3871fd1e547037772c7fe7650d5d1cbec53b3" +dependencies = [ + "bitflags 1.3.2", + "cfg-if 1.0.0", + "libc", + "log", + "rustversion", + "winapi", +] + [[package]] name = "thread_local" version = "1.1.7" @@ -6072,6 +6086,7 @@ dependencies = [ "serde_json", "tempfile", "thiserror", + "thread-priority", "tinyvec", "tokio", "tokio-stream", diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index da8bde694d3..bd7156970e8 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -70,6 +70,7 @@ getblocktemplate-rpcs = [ # Experimental internal miner support internal-miner = [ + "thread-priority", "zebra-chain/internal-miner", # TODO: move common code into zebra-chain or zebra-node-services and remove the RPC dependency "zebra-rpc/getblocktemplate-rpcs", @@ -207,6 +208,9 @@ atty = "0.2.14" num-integer = "0.1.45" rand = "0.8.5" +# prod feature internal-miner +thread-priority = { version = "0.15.1", optional = true } + # prod feature sentry sentry = { version = "0.32.1", default-features = false, features = ["backtrace", "contexts", "reqwest", "rustls", "tracing"], optional = true } diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index 29cf7c03f78..d11cfd0f4c8 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -9,6 +9,7 @@ use std::{sync::Arc, time::Duration}; use color_eyre::Report; +use thread_priority::{ThreadBuilder, ThreadPriority}; use tokio::{task::JoinHandle, time::sleep}; use tower::Service; use tracing::{Instrument, Span}; @@ -246,16 +247,25 @@ pub async fn mine_one_block( let mut block = proposal_block_from_template(&template, TimeSource::CurTime) .expect("unexpected invalid block template"); - // TODO: spawn to a low priority thread - // use a different nonce for each solver thread + // TODO: use a different nonce for each solver thread // // TODO: Replace with Arc::unwrap_or_clone() when it stabilises: // https://github.com/rust-lang/rust/issues/93610 let span = Span::current(); let solved_header = - tokio::task::spawn_blocking(move || span.in_scope(move || Solution::solve(*block.header))) - .wait_for_panics() - .await?; + tokio::task::spawn_blocking(move || span.in_scope(move || { + let miner_thread_handle = ThreadBuilder::default().name("zebra-miner").priority(ThreadPriority::Min).spawn(move |priority_result| { + if let Err(error) = priority_result { + info!(?error, "could not set miner to run at a low priority: running at default priority"); + } + + Solution::solve(*block.header) + }).expect("unable to spawn miner thread"); + + miner_thread_handle.wait_for_panics() + })) + .wait_for_panics() + .await?; block.header = Arc::new(solved_header); From b51311313461aec66eacaf87f020c7a42007b981 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 13:15:45 +1000 Subject: [PATCH 21/55] Ignore some invalid solutions --- zebra-chain/src/work/equihash.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/zebra-chain/src/work/equihash.rs b/zebra-chain/src/work/equihash.rs index a58c6fe4259..64ba48ca2fd 100644 --- a/zebra-chain/src/work/equihash.rs +++ b/zebra-chain/src/work/equihash.rs @@ -118,11 +118,12 @@ impl Solution { for solution in &solutions { header.solution = Self::from_bytes(solution) .expect("unexpected invalid solution: incorrect length"); - // TODO: only run this redundant check in tests - header - .solution - .check(&header) - .expect("unexpected invalid solution: invalid solution for header"); + + // TODO: work out why we sometimes get invalid solutions here + if let Err(error) = header.solution.check(&header) { + info!(?error, "found invalid solution for header"); + continue; + } if Self::difficulty_is_valid(&header) { info!("found valid solution and difficulty"); @@ -132,7 +133,7 @@ impl Solution { info!( solutions = ?solutions.len(), - "found valid solutions which did not pass the difficulty check" + "found valid solutions which did not pass the validity or difficulty checks" ); } From 9e3ed366009f428e7ec4efd2e46c709c3c0e7a1a Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 13:21:31 +1000 Subject: [PATCH 22/55] Use a difference nonce for each solver thread --- zebrad/src/components/miner.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index d11cfd0f4c8..10c5296b577 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -242,13 +242,17 @@ where #[instrument(skip(template))] pub async fn mine_one_block( template: GetBlockTemplate, - solver_id: usize, + solver_id: u8, ) -> Result { let mut block = proposal_block_from_template(&template, TimeSource::CurTime) .expect("unexpected invalid block template"); - // TODO: use a different nonce for each solver thread - // + // Use a different nonce for each solver thread. + // Change both the first and last bytes, so we don't have to care how the nonces are incremented. + let header = Arc::make_mut(&mut block.header); + *header.nonce.first_mut().unwrap() = solver_id; + *header.nonce.last_mut().unwrap() = solver_id; + // TODO: Replace with Arc::unwrap_or_clone() when it stabilises: // https://github.com/rust-lang/rust/issues/93610 let span = Span::current(); From 37ca967b6a4ba0a769b43abd4c2f6601306e4e5d Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 8 Jan 2024 13:24:05 +1000 Subject: [PATCH 23/55] Update TODOs --- zebrad/src/components/miner.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index 10c5296b577..92383a2edbf 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -186,8 +186,9 @@ where .expect("invalid RPC response: proposal in response to a template request"); // TODO: select!{} on either a solved header or a new block template using long_poll_id - // cancel the solver if there's a new template or if Zebra is shutting down - // launch the configured number of solvers + // cancel the solver if there's a new template + // add a config & launch the configured number of solvers, using available_parallelism() + // by default let solver_id = 0; let height = template.height; From d3c81a12b152337c922f343c5ff3f2d3992d9aa4 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 9 Jan 2024 05:58:14 +1000 Subject: [PATCH 24/55] Change the patch into a renamed dependency to simplify crate releases --- Cargo.lock | 15 +++++++++++++-- Cargo.toml | 6 ------ deny.toml | 9 ++++++++- zebra-chain/Cargo.toml | 17 ++++++++++++++++- zebra-chain/src/work/equihash.rs | 2 +- 5 files changed, 38 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49f432c2e83..ccd0a936fdc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1336,6 +1336,16 @@ dependencies = [ "regex", ] +[[package]] +name = "equihash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab579d7cf78477773b03e80bc2f89702ef02d7112c711d54ca93dcdce68533d5" +dependencies = [ + "blake2b_simd", + "byteorder", +] + [[package]] name = "equihash" version = "0.2.0" @@ -5633,7 +5643,7 @@ dependencies = [ "blake2s_simd", "bls12_381", "byteorder", - "equihash", + "equihash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "ff", "fpe", "group", @@ -5731,7 +5741,8 @@ dependencies = [ "criterion", "displaydoc", "ed25519-zebra", - "equihash", + "equihash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "equihash 0.2.0 (git+https://github.com/ZcashFoundation/librustzcash.git?branch=equihash-solver-tromp)", "futures", "group", "halo2_proofs", diff --git a/Cargo.toml b/Cargo.toml index 92ef0d072ae..05a7d6ece9c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,9 +88,3 @@ panic = "abort" # - add "-flto=thin" to all C/C++ code builds # - see https://doc.rust-lang.org/rustc/linker-plugin-lto.html#cc-code-as-a-dependency-in-rust lto = "thin" - -[patch.crates-io] -# Use the the working solver branch: -equihash = { git = 'https://github.com/ZcashFoundation/librustzcash.git', branch = 'equihash-solver-tromp' } -# or during development, use the locally checked out and modified version of equihash: -#equihash = { path = '../librustzcash/components/equihash' } diff --git a/deny.toml b/deny.toml index 42b655a8a60..4fc06769919 100644 --- a/deny.toml +++ b/deny.toml @@ -86,6 +86,11 @@ skip-tree = [ # wait for hdwallet to upgrade { name = "ring", version = "=0.16.20" }, + # wait for the equihash/solver feature to merge + # https://github.com/zcash/librustzcash/pull/1083 + # https://github.com/zcash/librustzcash/pull/1088 + { name = "equihash", version = "=0.2.0" }, + # zebra-utils dependencies # wait for structopt upgrade (or upgrade to clap 4) @@ -137,7 +142,9 @@ unknown-git = "deny" allow-registry = ["https://github.com/rust-lang/crates.io-index"] # List of URLs for allowed Git repositories allow-git = [ - # TODO: remove this after the equihash solver branch is merged and released + # TODO: remove this after the equihash solver branch is merged and released. + # + # "cargo deny" will log a warning in builds without the internal-miner feature. That's ok. "https://github.com/ZcashFoundation/librustzcash.git" ] diff --git a/zebra-chain/Cargo.toml b/zebra-chain/Cargo.toml index 328d23bbcb2..0c44b524b04 100644 --- a/zebra-chain/Cargo.toml +++ b/zebra-chain/Cargo.toml @@ -36,7 +36,10 @@ getblocktemplate-rpcs = [ # Experimental internal miner support internal-miner = [ - "equihash/solver", + # TODO: replace with "equihash/solver" when that feature is merged and released: + # https://github.com/zcash/librustzcash/pull/1083 + # https://github.com/zcash/librustzcash/pull/1088 + "equihash-solver", ] # Experimental elasticsearch support @@ -66,7 +69,19 @@ blake2s_simd = "1.0.2" bridgetree = "0.4.0" bs58 = { version = "0.5.0", features = ["check"] } byteorder = "1.5.0" + equihash = "0.2.0" +# Experimental internal miner support +# +# TODO: remove "equihash-solver" when the "equihash/solver" feature is merged and released: +# https://github.com/zcash/librustzcash/pull/1083 +# https://github.com/zcash/librustzcash/pull/1088 +# +# Use the the working solver branch: +equihash-solver = { version = "0.2.0", git = "https://github.com/ZcashFoundation/librustzcash.git", branch = "equihash-solver-tromp", features = ["solver"], package = "equihash", optional = true } +# or during development, use the locally checked out and modified version of equihash: +#equihash-solver = { path = "../librustzcash/components/equihash", package = "equihash" } + group = "0.13.0" incrementalmerkletree = "0.5.0" jubjub = "0.10.0" diff --git a/zebra-chain/src/work/equihash.rs b/zebra-chain/src/work/equihash.rs index 64ba48ca2fd..684add39777 100644 --- a/zebra-chain/src/work/equihash.rs +++ b/zebra-chain/src/work/equihash.rs @@ -109,7 +109,7 @@ impl Solution { let input = &input[0..Solution::INPUT_LENGTH]; while !is_shutting_down() { - let solutions = equihash::tromp::solve_200_9_compressed(input, || { + let solutions = equihash_solver::tromp::solve_200_9_compressed(input, || { // This skips the first nonce, which doesn't matter in practice. Self::next_nonce(&mut header.nonce); Some(*header.nonce) From 384d89204561fa3bd925ace4ab9b3bfdc5753932 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 9 Jan 2024 09:23:35 +1000 Subject: [PATCH 25/55] Clean up instrumentation and TODOs --- zebrad/src/components/miner.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index 92383a2edbf..c7b135750ff 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -48,6 +48,7 @@ pub const BLOCK_TEMPLATE_WAIT_TIME: Duration = Duration::from_secs(20); /// mining thread. /// /// TODO: add a test for this function. +#[instrument(skip(rpc))] pub fn spawn_init( config: &Config, rpc: GetBlockTemplateRpcImpl, @@ -128,7 +129,6 @@ where /// /// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core while running. /// It can run for minutes or hours if the network difficulty is high. -#[instrument(skip(rpc))] pub async fn run_mining_solver( rpc: GetBlockTemplateRpcImpl, ) -> Result<(), Report> @@ -249,13 +249,12 @@ pub async fn mine_one_block( .expect("unexpected invalid block template"); // Use a different nonce for each solver thread. - // Change both the first and last bytes, so we don't have to care how the nonces are incremented. + // Change both the first and last bytes, so we don't have to care if the nonces are incremented in + // big-endian or little-endian order. And we can see the thread that mined a block from the nonce. let header = Arc::make_mut(&mut block.header); *header.nonce.first_mut().unwrap() = solver_id; *header.nonce.last_mut().unwrap() = solver_id; - // TODO: Replace with Arc::unwrap_or_clone() when it stabilises: - // https://github.com/rust-lang/rust/issues/93610 let span = Span::current(); let solved_header = tokio::task::spawn_blocking(move || span.in_scope(move || { @@ -264,6 +263,8 @@ pub async fn mine_one_block( info!(?error, "could not set miner to run at a low priority: running at default priority"); } + // TODO: Replace with Arc::unwrap_or_clone() when it stabilises: + // https://github.com/rust-lang/rust/issues/93610 Solution::solve(*block.header) }).expect("unable to spawn miner thread"); From be5fce5a4366c3f8b40645d77b644d78524fa160 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 9 Jan 2024 11:12:35 +1000 Subject: [PATCH 26/55] Make RPC instances cloneable and clean up generics --- zebra-rpc/src/methods.rs | 85 +++++++++++++++---- .../src/methods/get_block_template_rpcs.rs | 66 ++++++++++++-- zebra-rpc/src/server.rs | 10 ++- zebrad/src/components/miner.rs | 15 +++- 4 files changed, 144 insertions(+), 32 deletions(-) diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index ea44310f3ce..9fc190a9a64 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -6,7 +6,7 @@ //! Some parts of the `zcashd` RPC documentation are outdated. //! So this implementation follows the `zcashd` server and `lightwalletd` client implementations. -use std::{collections::HashSet, sync::Arc}; +use std::{collections::HashSet, fmt::Debug, sync::Arc}; use chrono::Utc; use futures::{FutureExt, TryFutureExt}; @@ -15,7 +15,7 @@ use indexmap::IndexMap; use jsonrpc_core::{self, BoxFuture, Error, ErrorCode, Result}; use jsonrpc_derive::rpc; use tokio::{sync::broadcast, task::JoinHandle}; -use tower::{buffer::Buffer, Service, ServiceExt}; +use tower::{Service, ServiceExt}; use tracing::Instrument; use zebra_chain::{ @@ -268,19 +268,28 @@ pub trait Rpc { } /// RPC method implementations. +#[derive(Clone)] pub struct RpcImpl where Mempool: Service< - mempool::Request, - Response = mempool::Response, - Error = zebra_node_services::BoxError, - >, + mempool::Request, + Response = mempool::Response, + Error = zebra_node_services::BoxError, + > + Clone + + Send + + Sync + + 'static, + Mempool::Future: Send, State: Service< - zebra_state::ReadRequest, - Response = zebra_state::ReadResponse, - Error = zebra_state::BoxError, - >, - Tip: ChainTip, + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + Sync + + 'static, + State::Future: Send, + Tip: ChainTip + Clone + Send + Sync + 'static, { // Configuration // @@ -304,7 +313,7 @@ where // Services // /// A handle to the mempool service. - mempool: Buffer, + mempool: Mempool, /// A handle to the state service. state: State, @@ -318,13 +327,51 @@ where queue_sender: broadcast::Sender, } +impl Debug for RpcImpl +where + Mempool: Service< + mempool::Request, + Response = mempool::Response, + Error = zebra_node_services::BoxError, + > + Clone + + Send + + Sync + + 'static, + Mempool::Future: Send, + State: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + Sync + + 'static, + State::Future: Send, + Tip: ChainTip + Clone + Send + Sync + 'static, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Skip fields without Debug impls, and skip channels + f.debug_struct("RpcImpl") + .field("build_version", &self.build_version) + .field("user_agent", &self.user_agent) + .field("network", &self.network) + .field("debug_force_finished_sync", &self.debug_force_finished_sync) + .field("debug_like_zcashd", &self.debug_like_zcashd) + .finish() + } +} + impl RpcImpl where Mempool: Service< mempool::Request, Response = mempool::Response, Error = zebra_node_services::BoxError, - > + 'static, + > + Clone + + Send + + Sync + + 'static, + Mempool::Future: Send, State: Service< zebra_state::ReadRequest, Response = zebra_state::ReadResponse, @@ -333,6 +380,7 @@ where + Send + Sync + 'static, + State::Future: Send, Tip: ChainTip + Clone + Send + Sync + 'static, { /// Create a new instance of the RPC handler. @@ -346,15 +394,13 @@ where network: Network, debug_force_finished_sync: bool, debug_like_zcashd: bool, - mempool: Buffer, + mempool: Mempool, state: State, latest_chain_tip: Tip, ) -> (Self, JoinHandle<()>) where VersionString: ToString + Clone + Send + 'static, UserAgentString: ToString + Clone + Send + 'static, - >::Future: Send, - >::Future: Send, { let (runner, queue_sender) = Queue::start(); @@ -391,11 +437,14 @@ where impl Rpc for RpcImpl where - Mempool: tower::Service< + Mempool: Service< mempool::Request, Response = mempool::Response, Error = zebra_node_services::BoxError, - > + 'static, + > + Clone + + Send + + Sync + + 'static, Mempool::Future: Send, State: Service< zebra_state::ReadRequest, diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index 7211a4e42df..beedf61f26c 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -1,11 +1,11 @@ //! RPC methods related to mining only available with `getblocktemplate-rpcs` rust feature. -use std::{sync::Arc, time::Duration}; +use std::{fmt::Debug, sync::Arc, time::Duration}; use futures::{future::OptionFuture, FutureExt, TryFutureExt}; use jsonrpc_core::{self, BoxFuture, Error, ErrorCode, Result}; use jsonrpc_derive::rpc; -use tower::{buffer::Buffer, Service, ServiceExt}; +use tower::{Service, ServiceExt}; use zcash_address::{self, unified::Encoding, TryFromAddress}; @@ -223,6 +223,7 @@ pub trait GetBlockTemplateRpc { } /// RPC method implementations. +#[derive(Clone)] pub struct GetBlockTemplateRpcImpl< Mempool, State, @@ -235,7 +236,10 @@ pub struct GetBlockTemplateRpcImpl< mempool::Request, Response = mempool::Response, Error = zebra_node_services::BoxError, - > + 'static, + > + Clone + + Send + + Sync + + 'static, Mempool::Future: Send, State: Service< zebra_state::ReadRequest, @@ -277,7 +281,7 @@ pub struct GetBlockTemplateRpcImpl< // Services // /// A handle to the mempool service. - mempool: Buffer, + mempool: Mempool, /// A handle to the state service. state: State, @@ -295,6 +299,48 @@ pub struct GetBlockTemplateRpcImpl< address_book: AddressBook, } +impl Debug + for GetBlockTemplateRpcImpl +where + Mempool: Service< + mempool::Request, + Response = mempool::Response, + Error = zebra_node_services::BoxError, + > + Clone + + Send + + Sync + + 'static, + Mempool::Future: Send, + State: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + Sync + + 'static, + >::Future: Send, + Tip: ChainTip + Clone + Send + Sync + 'static, + BlockVerifierRouter: Service + + Clone + + Send + + Sync + + 'static, + >::Future: Send, + SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, + AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Skip fields without debug impls + f.debug_struct("GetBlockTemplateRpcImpl") + .field("network", &self.network) + .field("miner_address", &self.miner_address) + .field("extra_coinbase_data", &self.extra_coinbase_data) + .field("debug_like_zcashd", &self.debug_like_zcashd) + .finish() + } +} + impl GetBlockTemplateRpcImpl where @@ -302,7 +348,10 @@ where mempool::Request, Response = mempool::Response, Error = zebra_node_services::BoxError, - > + 'static, + > + Clone + + Send + + Sync + + 'static, Mempool::Future: Send, State: Service< zebra_state::ReadRequest, @@ -332,7 +381,7 @@ where pub fn new( network: Network, mining_config: crate::config::mining::Config, - mempool: Buffer, + mempool: Mempool, state: State, latest_chain_tip: Tip, block_verifier_router: BlockVerifierRouter, @@ -399,7 +448,10 @@ where mempool::Request, Response = mempool::Response, Error = zebra_node_services::BoxError, - > + 'static, + > + Clone + + Send + + Sync + + 'static, Mempool::Future: Send, State: Service< zebra_state::ReadRequest, diff --git a/zebra-rpc/src/server.rs b/zebra-rpc/src/server.rs index 44e6afe84f5..477427a62b4 100644 --- a/zebra-rpc/src/server.rs +++ b/zebra-rpc/src/server.rs @@ -12,8 +12,7 @@ use std::{fmt, panic}; use jsonrpc_core::{Compatibility, MetaIoHandler}; use jsonrpc_http_server::{CloseHandle, ServerBuilder}; use tokio::task::JoinHandle; -use tower::{buffer::Buffer, Service}; - +use tower::Service; use tracing::{Instrument, *}; use zebra_chain::{ @@ -99,7 +98,7 @@ impl RpcServer { mining_config: crate::config::mining::Config, build_version: VersionString, user_agent: UserAgentString, - mempool: Buffer, + mempool: Mempool, state: State, #[cfg_attr(not(feature = "getblocktemplate-rpcs"), allow(unused_variables))] block_verifier_router: BlockVerifierRouter, @@ -117,7 +116,10 @@ impl RpcServer { mempool::Request, Response = mempool::Response, Error = zebra_node_services::BoxError, - > + 'static, + > + Clone + + Send + + Sync + + 'static, Mempool::Future: Send, State: Service< zebra_state::ReadRequest, diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index c7b135750ff..c85f426bb24 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -58,7 +58,10 @@ where mempool::Request, Response = mempool::Response, Error = zebra_node_services::BoxError, - > + 'static, + > + Clone + + Send + + Sync + + 'static, Mempool::Future: Send, State: Service< zebra_state::ReadRequest, @@ -100,7 +103,10 @@ where mempool::Request, Response = mempool::Response, Error = zebra_node_services::BoxError, - > + 'static, + > + Clone + + Send + + Sync + + 'static, Mempool::Future: Send, State: Service< zebra_state::ReadRequest, @@ -137,7 +143,10 @@ where mempool::Request, Response = mempool::Response, Error = zebra_node_services::BoxError, - > + 'static, + > + Clone + + Send + + Sync + + 'static, Mempool::Future: Send, State: Service< zebra_state::ReadRequest, From 96d57a7c3d79dc3b9dc3dd0bfee6c27f46084a46 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 9 Jan 2024 11:13:22 +1000 Subject: [PATCH 27/55] Make LongPollId Copy so it's easier to use --- zebra-rpc/src/methods/get_block_template_rpcs.rs | 4 +--- .../src/methods/get_block_template_rpcs/types/long_poll.rs | 2 +- .../common/get_block_template_rpcs/get_block_template.rs | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index beedf61f26c..3a040a6540e 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -546,9 +546,7 @@ where async move { get_block_template::check_parameters(¶meters)?; - let client_long_poll_id = parameters - .as_ref() - .and_then(|params| params.long_poll_id.clone()); + let client_long_poll_id = parameters.as_ref().and_then(|params| params.long_poll_id); // - One-off checks diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs index 73ea1c015af..9f9a316166a 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs @@ -113,7 +113,7 @@ impl LongPollInput { /// /// `zcashd` IDs are currently 69 hex/decimal digits long. /// Since Zebra's IDs are only 46 hex/decimal digits, mining pools should be able to handle them. -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(try_from = "String", into = "String")] pub struct LongPollId { // Fields that invalidate old work: diff --git a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs index e6bd3d3d9c4..56a62ff4346 100644 --- a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs +++ b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs @@ -169,7 +169,7 @@ async fn try_validate_block_template(client: &RpcRequestClient) -> Result<()> { { let client = client.clone(); - let mut long_poll_id = response_json_result.long_poll_id.clone(); + let mut long_poll_id = response_json_result.long_poll_id; tokio::spawn(async move { loop { @@ -196,7 +196,7 @@ async fn try_validate_block_template(client: &RpcRequestClient) -> Result<()> { } long_poll_result = long_poll_request => { - long_poll_id = long_poll_result.long_poll_id.clone(); + long_poll_id = long_poll_result.long_poll_id; if let Some(false) = long_poll_result.submit_old { let _ = long_poll_result_tx.send(long_poll_result); From b8bde145acb534dfb7a7959b0cb77f388d21d241 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 9 Jan 2024 11:14:51 +1000 Subject: [PATCH 28/55] Add API to restart mining if there's a new block template --- zebra-chain/src/work/equihash.rs | 18 ++++++- zebra-state/src/service/watch_receiver.rs | 11 +++- zebrad/src/components/miner.rs | 64 +++++++++++++++++------ 3 files changed, 73 insertions(+), 20 deletions(-) diff --git a/zebra-chain/src/work/equihash.rs b/zebra-chain/src/work/equihash.rs index 684add39777..deaf379b35a 100644 --- a/zebra-chain/src/work/equihash.rs +++ b/zebra-chain/src/work/equihash.rs @@ -19,7 +19,6 @@ use crate::{ pub struct Error(#[from] equihash::Error); /// The error type for Equihash solving. -#[non_exhaustive] #[derive(Copy, Clone, Debug, Eq, PartialEq, thiserror::Error)] #[error("solver was cancelled")] pub struct SolverCancelled; @@ -91,6 +90,8 @@ impl Solution { /// Mines and returns one or more [`Solution`]s based on a template `header`. /// The returned header contains a valid `nonce` and `solution`. /// + /// If `cancel_fn()` returns an error, returns early with `Err(SolverCancelled)`. + /// /// The `nonce` in the header template is taken as the starting nonce. If you are running multiple /// solvers at the same time, start them with different nonces. /// The `solution` in the header template is ignored. @@ -99,7 +100,10 @@ impl Solution { /// It can run for minutes or hours if the network difficulty is high. #[cfg(feature = "internal-miner")] #[allow(clippy::unwrap_in_result)] - pub fn solve(mut header: Header) -> Result { + pub fn solve(mut header: Header, mut cancel_fn: F) -> Result + where + F: FnMut() -> Result<(), SolverCancelled>, + { use crate::shutdown::is_shutting_down; let mut input = Vec::new(); @@ -109,12 +113,22 @@ impl Solution { let input = &input[0..Solution::INPUT_LENGTH]; while !is_shutting_down() { + // Don't run the solver if we'd just cancel it anyway. + cancel_fn()?; + let solutions = equihash_solver::tromp::solve_200_9_compressed(input, || { + // Cancel the solver if we have a new template. + if cancel_fn().is_err() { + return None; + } + // This skips the first nonce, which doesn't matter in practice. Self::next_nonce(&mut header.nonce); Some(*header.nonce) }); + // If we got any solutions, try submitting them, because the new template might just + // contain some extra transactions. Mining extra transactions is optional. for solution in &solutions { header.solution = Self::from_bytes(solution) .expect("unexpected invalid solution: incorrect length"); diff --git a/zebra-state/src/service/watch_receiver.rs b/zebra-state/src/service/watch_receiver.rs index 6540ccf98d4..d2157d91add 100644 --- a/zebra-state/src/service/watch_receiver.rs +++ b/zebra-state/src/service/watch_receiver.rs @@ -99,13 +99,22 @@ where self.receiver.borrow().clone() } - /// Calls [`watch::Receiver::changed`] and returns the result. + /// Calls [`watch::Receiver::changed()`] and returns the result. + /// Returns when the inner value has been updated, even if the old and new values are equal. /// /// Marks the watched data as seen. pub async fn changed(&mut self) -> Result<(), watch::error::RecvError> { self.receiver.changed().await } + /// Calls [`watch::Receiver::has_changed()`] and returns the result. + /// Returns `true` when the inner value has been updated, even if the old and new values are equal. + /// + /// Does not mark the watched data as seen. + pub fn has_changed(&self) -> Result { + self.receiver.has_changed() + } + /// Marks the watched data as seen. pub fn mark_as_seen(&mut self) { self.receiver.borrow_and_update(); diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index c85f426bb24..13cb45ddbf2 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -200,8 +200,25 @@ where // by default let solver_id = 0; - let height = template.height; - let Ok(block) = mine_one_block(template, solver_id).await else { + continue; + }; + + // Mine a block using the equihash solver. + let height = template.coinbase_height().expect("template is valid"); + let cancel_receiver = template_receiver.clone(); + let cancel_fn = move || match cancel_receiver.has_changed() { + Ok(has_changed) => { + if has_changed { + Err(SolverCancelled) + } else { + Ok(()) + } + } + // If the sender was dropped, we're likely shutting down, so cancel the solver. + Err(_sender_dropped) => Err(SolverCancelled), + }; + + let Ok(block) = mine_one_block(solver_id, template, cancel_fn).await else { // If the solver was cancelled, we're either shutting down, or we have a new template. if solver_id == 0 { info!( @@ -219,6 +236,11 @@ where continue; }; + // Submit the newly mined block to the verifiers. + // + // TODO: if there is a new template (`cancel_fn().is_err()`), and + // GetBlockTemplate.submit_old is false, return immediately, and skip submitting the + // block. let data = block .zcash_serialize_to_vec() .expect("serializing to Vec never fails"); @@ -244,27 +266,33 @@ where Ok(()) } -/// Mines a single block, calculating its equihash solutions, and submitting it to Zebra's block -/// validator. Uses a different nonce range for each `solver_id`. +/// Mines a single block based on `template`, calculates its equihash solutions, checks difficulty, +/// and returning the newly mined block. Uses a different nonce range for each `solver_id`. /// -/// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core while running. -/// It can run for minutes or hours if the network difficulty is high. -#[instrument(skip(template))] -pub async fn mine_one_block( - template: GetBlockTemplate, +/// If `cancel_fn()` returns an error, returns early with `Err(SolverCancelled)`. +/// +/// See [`run_mining_solver()`] for more details. +pub async fn mine_one_block( solver_id: u8, -) -> Result { - let mut block = proposal_block_from_template(&template, TimeSource::CurTime) - .expect("unexpected invalid block template"); + mut template: Arc, + cancel_fn: F, +) -> Result, SolverCancelled> +where + F: FnMut() -> Result<(), SolverCancelled> + Send + Sync + 'static, +{ + // TODO: Replace with Arc::unwrap_or_clone() when it stabilises: + // https://github.com/rust-lang/rust/issues/93610 + let mut header = *template.header; // Use a different nonce for each solver thread. // Change both the first and last bytes, so we don't have to care if the nonces are incremented in // big-endian or little-endian order. And we can see the thread that mined a block from the nonce. - let header = Arc::make_mut(&mut block.header); *header.nonce.first_mut().unwrap() = solver_id; *header.nonce.last_mut().unwrap() = solver_id; + // Mine a block using the solver, in a low-priority blocking thread. let span = Span::current(); + // TODO: get and submit all valid headers, not just the first one let solved_header = tokio::task::spawn_blocking(move || span.in_scope(move || { let miner_thread_handle = ThreadBuilder::default().name("zebra-miner").priority(ThreadPriority::Min).spawn(move |priority_result| { @@ -272,9 +300,7 @@ pub async fn mine_one_block( info!(?error, "could not set miner to run at a low priority: running at default priority"); } - // TODO: Replace with Arc::unwrap_or_clone() when it stabilises: - // https://github.com/rust-lang/rust/issues/93610 - Solution::solve(*block.header) + Solution::solve(header, cancel_fn) }).expect("unable to spawn miner thread"); miner_thread_handle.wait_for_panics() @@ -282,7 +308,11 @@ pub async fn mine_one_block( .wait_for_panics() .await?; + // Modify the template into a solved block. + // + // TODO: Replace with Arc::unwrap_or_clone() when it stabilises + let block = Arc::make_mut(&mut template); block.header = Arc::new(solved_header); - Ok(block) + Ok(template) } From 1c7ad5105ee2ce4cef3906df217143fba3ec2c16 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 9 Jan 2024 11:15:26 +1000 Subject: [PATCH 29/55] Actually restart mining if there's a new block template --- zebrad/src/components/miner.rs | 135 ++++++++++++++++++++++++++++----- 1 file changed, 118 insertions(+), 17 deletions(-) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index 13cb45ddbf2..96bccaa782d 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -10,7 +10,7 @@ use std::{sync::Arc, time::Duration}; use color_eyre::Report; use thread_priority::{ThreadBuilder, ThreadPriority}; -use tokio::{task::JoinHandle, time::sleep}; +use tokio::{sync::watch, task::JoinHandle, time::sleep}; use tower::Service; use tracing::{Instrument, Span}; @@ -30,7 +30,7 @@ use zebra_rpc::{ methods::{ get_block_template_rpcs::{ get_block_template::{ - self, proposal::TimeSource, proposal_block_from_template, GetBlockTemplate, + self, proposal::TimeSource, proposal_block_from_template, GetBlockTemplateCapability::*, GetBlockTemplateRequestMode::*, }, types::hex_data::HexData, @@ -38,6 +38,7 @@ use zebra_rpc::{ GetBlockTemplateRpc, GetBlockTemplateRpcImpl, }, }; +use zebra_state::WatchReceiver; /// The amount of time we wait between block template retries. pub const BLOCK_TEMPLATE_WAIT_TIME: Duration = Duration::from_secs(20); @@ -127,16 +128,30 @@ where SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, { - run_mining_solver(rpc).await + let (template_sender, template_receiver) = watch::channel(None); + let template_receiver = WatchReceiver::new(template_receiver); + + // TODO: select!{} on the block generator and all the mining solver threads + // add a config & launch the configured number of solvers, using available_parallelism() + // by default + generate_block_templates(rpc.clone(), template_sender).await?; + run_mining_solver(0, template_receiver, rpc).await?; + + Ok(()) } -/// Runs a single mining thread to generate blocks, calculate equihash solutions, and submit valid -/// blocks to Zebra's block validator. -/// -/// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core while running. -/// It can run for minutes or hours if the network difficulty is high. -pub async fn run_mining_solver( +/// Generates block templates using `rpc`, and sends them to mining threads using `template_sender`. +#[instrument(skip(rpc, template_sender))] +pub async fn generate_block_templates< + Mempool, + State, + Tip, + BlockVerifierRouter, + SyncStatus, + AddressBook, +>( rpc: GetBlockTemplateRpcImpl, + template_sender: watch::Sender>>, ) -> Result<(), Report> where Mempool: Service< @@ -168,12 +183,11 @@ where AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, { // Pass the correct arguments, even if Zebra currently ignores them. - let mut long_poll_id = None; let mut parameters = get_block_template::JsonParameters { mode: Template, data: None, capabilities: vec![LongPoll, CoinbaseTxn], - long_poll_id, + long_poll_id: None, _work_id: None, }; @@ -186,7 +200,12 @@ where ?BLOCK_TEMPLATE_WAIT_TIME, "waiting for a valid block template", ); - sleep(BLOCK_TEMPLATE_WAIT_TIME).await; + + // Skip the wait if we got an error because we are shutting down. + if !is_shutting_down() { + sleep(BLOCK_TEMPLATE_WAIT_TIME).await; + } + continue; }; @@ -194,11 +213,89 @@ where .try_into_template() .expect("invalid RPC response: proposal in response to a template request"); - // TODO: select!{} on either a solved header or a new block template using long_poll_id - // cancel the solver if there's a new template - // add a config & launch the configured number of solvers, using available_parallelism() - // by default - let solver_id = 0; + info!( + height = ?template.height, + transactions = ?template.transactions.len(), + "mining with an updated block template", + ); + + // Tell the next loop iteration to wait until the template has changed before returning. + parameters.long_poll_id = Some(template.long_poll_id); + + let block = proposal_block_from_template(&template, TimeSource::CurTime) + .expect("unexpected invalid block template"); + + // Always send, even if all the receivers have been dropped. + let _prev_template = template_sender.send_replace(Some(Arc::new(block))); + } + + Ok(()) +} + +/// Runs a single mining thread that gets blocks from the `template_receiver`, calculates equihash +/// solutions with nonces based on `solver_id`, and submits valid blocks to Zebra's block validator. +/// +/// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core while running. +/// It can run for minutes or hours if the network difficulty is high. Mining uses a thread with +/// low CPU priority. +#[instrument(skip(template_receiver, rpc))] +pub async fn run_mining_solver( + solver_id: u8, + template_receiver: WatchReceiver>>, + rpc: GetBlockTemplateRpcImpl, +) -> Result<(), Report> +where + Mempool: Service< + mempool::Request, + Response = mempool::Response, + Error = zebra_node_services::BoxError, + > + Clone + + Send + + Sync + + 'static, + Mempool::Future: Send, + State: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + Sync + + 'static, + >::Future: Send, + Tip: ChainTip + Clone + Send + Sync + 'static, + BlockVerifierRouter: Service + + Clone + + Send + + Sync + + 'static, + >::Future: Send, + SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, + AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, +{ + while !is_shutting_down() { + // Get the latest block template. + let template = template_receiver.cloned_watch_data(); + + let Some(template) = template else { + if solver_id == 0 { + info!( + ?solver_id, + ?BLOCK_TEMPLATE_WAIT_TIME, + "solver waiting for initial block template" + ); + } else { + debug!( + ?solver_id, + ?BLOCK_TEMPLATE_WAIT_TIME, + "solver waiting for initial block template" + ); + } + + // Skip the wait if we didn't get a template because we are shutting down. + if !is_shutting_down() { + sleep(BLOCK_TEMPLATE_WAIT_TIME).await; + } continue; }; @@ -224,12 +321,16 @@ where info!( ?height, ?solver_id, + new_template = ?template_receiver.has_changed(), + shutting_down = ?is_shutting_down(), "solver cancelled: getting a new block template or shutting down" ); } else { debug!( ?height, ?solver_id, + new_template = ?template_receiver.has_changed(), + shutting_down = ?is_shutting_down(), "solver cancelled: getting a new block template or shutting down" ); } From 43349d68ff2c39df279a2e0c36e21cb74ded7385 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 9 Jan 2024 11:15:39 +1000 Subject: [PATCH 30/55] Tidy instrumentation --- zebrad/src/components/miner.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index 96bccaa782d..ebb92d24e32 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -48,8 +48,8 @@ pub const BLOCK_TEMPLATE_WAIT_TIME: Duration = Duration::from_secs(20); /// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core per configured /// mining thread. /// -/// TODO: add a test for this function. -#[instrument(skip(rpc))] +/// See [`run_mining_solver()`] for more details. +#[instrument(skip(config, rpc))] pub fn spawn_init( config: &Config, rpc: GetBlockTemplateRpcImpl, @@ -94,7 +94,7 @@ where /// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core per configured /// mining thread. /// -/// TODO: add a test for this function. +/// See [`run_mining_solver()`] for more details. pub async fn init( _config: Config, rpc: GetBlockTemplateRpcImpl, From a92a3fa62b973510d41e2f18aeb6416209113445 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 9 Jan 2024 11:15:58 +1000 Subject: [PATCH 31/55] fixup! Move config validation into GetBlockTemplateRpcImpl::new() --- zebra-rpc/src/server.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/zebra-rpc/src/server.rs b/zebra-rpc/src/server.rs index 477427a62b4..de78c1d6c64 100644 --- a/zebra-rpc/src/server.rs +++ b/zebra-rpc/src/server.rs @@ -152,17 +152,6 @@ impl RpcServer { #[cfg(feature = "getblocktemplate-rpcs")] { - // Prevent loss of miner funds due to an unsupported or incorrect address type. - if let Some(miner_address) = mining_config.miner_address { - assert_eq!( - miner_address.network(), - network, - "incorrect miner address config: {miner_address} \ - network.network {network} and miner address network {} must match", - miner_address.network(), - ); - } - // Initialize the getblocktemplate rpc method handler let get_block_template_rpc_impl = GetBlockTemplateRpcImpl::new( network, From 24079ce6f51f481e293ae7c5ca9f8ce3a53adb2e Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 9 Jan 2024 11:17:25 +1000 Subject: [PATCH 32/55] fixup! Make RPC instances cloneable and clean up generics --- zebra-rpc/src/methods/tests/prop.rs | 28 ++++++++++++------------- zebra-rpc/src/methods/tests/snapshot.rs | 3 ++- zebra-rpc/src/methods/tests/vectors.rs | 10 ++++----- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/zebra-rpc/src/methods/tests/prop.rs b/zebra-rpc/src/methods/tests/prop.rs index cf2cc7c1bd5..c6c4fa69693 100644 --- a/zebra-rpc/src/methods/tests/prop.rs +++ b/zebra-rpc/src/methods/tests/prop.rs @@ -45,7 +45,7 @@ proptest! { Mainnet, false, true, - Buffer::new(mempool.clone(), 1), + mempool.clone(), Buffer::new(state.clone(), 1), NoChainTip, ); @@ -100,7 +100,7 @@ proptest! { Mainnet, false, true, - Buffer::new(mempool.clone(), 1), + mempool.clone(), Buffer::new(state.clone(), 1), NoChainTip, ); @@ -160,7 +160,7 @@ proptest! { Mainnet, false, true, - Buffer::new(mempool.clone(), 1), + mempool.clone(), Buffer::new(state.clone(), 1), NoChainTip, ); @@ -228,7 +228,7 @@ proptest! { Mainnet, false, true, - Buffer::new(mempool.clone(), 1), + mempool.clone(), Buffer::new(state.clone(), 1), NoChainTip, ); @@ -285,7 +285,7 @@ proptest! { Mainnet, false, true, - Buffer::new(mempool.clone(), 1), + mempool.clone(), Buffer::new(state.clone(), 1), NoChainTip, ); @@ -340,7 +340,7 @@ proptest! { Mainnet, false, true, - Buffer::new(mempool.clone(), 1), + mempool.clone(), Buffer::new(state.clone(), 1), NoChainTip, ); @@ -441,7 +441,7 @@ proptest! { Mainnet, false, true, - Buffer::new(mempool.clone(), 1), + mempool.clone(), Buffer::new(state.clone(), 1), NoChainTip, ); @@ -500,7 +500,7 @@ proptest! { Mainnet, false, true, - Buffer::new(mempool.clone(), 1), + mempool.clone(), Buffer::new(state.clone(), 1), NoChainTip, ); @@ -548,7 +548,7 @@ proptest! { network, false, true, - Buffer::new(mempool.clone(), 1), + mempool.clone(), Buffer::new(state.clone(), 1), NoChainTip, ); @@ -599,7 +599,7 @@ proptest! { network, false, true, - Buffer::new(mempool.clone(), 1), + mempool.clone(), Buffer::new(state.clone(), 1), chain_tip, ); @@ -686,7 +686,7 @@ proptest! { network, false, true, - Buffer::new(mempool.clone(), 1), + mempool.clone(), Buffer::new(state.clone(), 1), chain_tip, ); @@ -750,7 +750,7 @@ proptest! { network, false, true, - Buffer::new(mempool.clone(), 1), + mempool.clone(), Buffer::new(state.clone(), 1), chain_tip, ); @@ -802,7 +802,7 @@ proptest! { Mainnet, false, true, - Buffer::new(mempool.clone(), 1), + mempool.clone(), Buffer::new(state.clone(), 1), NoChainTip, ); @@ -892,7 +892,7 @@ proptest! { Mainnet, false, true, - Buffer::new(mempool.clone(), 1), + mempool.clone(), Buffer::new(state.clone(), 1), NoChainTip, ); diff --git a/zebra-rpc/src/methods/tests/snapshot.rs b/zebra-rpc/src/methods/tests/snapshot.rs index 8b74f393e83..2a8e9149f56 100644 --- a/zebra-rpc/src/methods/tests/snapshot.rs +++ b/zebra-rpc/src/methods/tests/snapshot.rs @@ -8,6 +8,7 @@ use std::{collections::BTreeMap, sync::Arc}; use insta::dynamic_redaction; +use tower::buffer::Buffer; use zebra_chain::{ block::Block, @@ -338,7 +339,7 @@ async fn test_mocked_rpc_response_data_for_network(network: Network) { network, false, true, - Buffer::new(mempool, 1), + mempool, state.clone(), latest_chain_tip, ); diff --git a/zebra-rpc/src/methods/tests/vectors.rs b/zebra-rpc/src/methods/tests/vectors.rs index 63e53386d5c..d6e67e27fd3 100644 --- a/zebra-rpc/src/methods/tests/vectors.rs +++ b/zebra-rpc/src/methods/tests/vectors.rs @@ -1099,7 +1099,7 @@ async fn rpc_getmininginfo() { let get_block_template_rpc = get_block_template_rpcs::GetBlockTemplateRpcImpl::new( Mainnet, Default::default(), - Buffer::new(MockService::build().for_unit_tests(), 1), + MockService::build().for_unit_tests(), read_state, latest_chain_tip.clone(), MockService::build().for_unit_tests(), @@ -1135,7 +1135,7 @@ async fn rpc_getnetworksolps() { let get_block_template_rpc = get_block_template_rpcs::GetBlockTemplateRpcImpl::new( Mainnet, Default::default(), - Buffer::new(MockService::build().for_unit_tests(), 1), + MockService::build().for_unit_tests(), read_state, latest_chain_tip.clone(), MockService::build().for_unit_tests(), @@ -1575,7 +1575,7 @@ async fn rpc_validateaddress() { let get_block_template_rpc = get_block_template_rpcs::GetBlockTemplateRpcImpl::new( Mainnet, Default::default(), - Buffer::new(MockService::build().for_unit_tests(), 1), + MockService::build().for_unit_tests(), MockService::build().for_unit_tests(), mock_chain_tip, MockService::build().for_unit_tests(), @@ -1620,7 +1620,7 @@ async fn rpc_z_validateaddress() { let get_block_template_rpc = get_block_template_rpcs::GetBlockTemplateRpcImpl::new( Mainnet, Default::default(), - Buffer::new(MockService::build().for_unit_tests(), 1), + MockService::build().for_unit_tests(), MockService::build().for_unit_tests(), mock_chain_tip, MockService::build().for_unit_tests(), @@ -1825,7 +1825,7 @@ async fn rpc_z_listunifiedreceivers() { let get_block_template_rpc = get_block_template_rpcs::GetBlockTemplateRpcImpl::new( Mainnet, Default::default(), - Buffer::new(MockService::build().for_unit_tests(), 1), + MockService::build().for_unit_tests(), MockService::build().for_unit_tests(), mock_chain_tip, MockService::build().for_unit_tests(), From 669aeadc0164ee787ade36700818c4969292c664 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 9 Jan 2024 11:37:59 +1000 Subject: [PATCH 33/55] Run the template generator and one miner concurrently --- zebra-state/src/service/watch_receiver.rs | 3 +- zebrad/src/components/miner.rs | 37 +++++++++++++++-------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/zebra-state/src/service/watch_receiver.rs b/zebra-state/src/service/watch_receiver.rs index d2157d91add..34a3d5e87ea 100644 --- a/zebra-state/src/service/watch_receiver.rs +++ b/zebra-state/src/service/watch_receiver.rs @@ -116,7 +116,8 @@ where } /// Marks the watched data as seen. + /// Calls [`watch::Receiver::mark_changed()`]. pub fn mark_as_seen(&mut self) { - self.receiver.borrow_and_update(); + self.receiver.mark_changed(); } } diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index ebb92d24e32..abe50c0e017 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -10,7 +10,7 @@ use std::{sync::Arc, time::Duration}; use color_eyre::Report; use thread_priority::{ThreadBuilder, ThreadPriority}; -use tokio::{sync::watch, task::JoinHandle, time::sleep}; +use tokio::{select, sync::watch, task::JoinHandle, time::sleep}; use tower::Service; use tracing::{Instrument, Span}; @@ -131,11 +131,19 @@ where let (template_sender, template_receiver) = watch::channel(None); let template_receiver = WatchReceiver::new(template_receiver); - // TODO: select!{} on the block generator and all the mining solver threads - // add a config & launch the configured number of solvers, using available_parallelism() + // TODO: add a config & launch the configured number of solvers, using available_parallelism() // by default - generate_block_templates(rpc.clone(), template_sender).await?; - run_mining_solver(0, template_receiver, rpc).await?; + let template_generator = generate_block_templates(rpc.clone(), template_sender); + let mining_solver = run_mining_solver(0, template_receiver, rpc); + + // These tasks run forever unless there is a fatal error or shutdown. + // When that happens, the first task to notice returns, and the other tasks are cancelled. + // Then this task returns and drops the `template_sender`, which cancels all the spawned miner + // threads. + select! { + result = template_generator => { result?; } + result = mining_solver => { result?; } + } Ok(()) } @@ -191,7 +199,8 @@ where _work_id: None, }; - while !is_shutting_down() { + // Shut down the task when all the template receivers are dropped, or Zebra shuts down. + while !template_sender.is_closed() && !is_shutting_down() { let template = rpc.get_block_template(Some(parameters.clone())).await; // Wait for the chain to sync so we get a valid template. @@ -209,6 +218,7 @@ where continue; }; + // Convert from RPC GetBlockTemplate to Block let template = template .try_into_template() .expect("invalid RPC response: proposal in response to a template request"); @@ -219,14 +229,14 @@ where "mining with an updated block template", ); - // Tell the next loop iteration to wait until the template has changed before returning. + // Tell the next get_block_template() call to wait until the template has changed. parameters.long_poll_id = Some(template.long_poll_id); let block = proposal_block_from_template(&template, TimeSource::CurTime) .expect("unexpected invalid block template"); - // Always send, even if all the receivers have been dropped. - let _prev_template = template_sender.send_replace(Some(Arc::new(block))); + // If all the receivers have been dropped, stop sending templates and shut down. + template_sender.send(Some(Arc::new(block)))?; } Ok(()) @@ -241,7 +251,7 @@ where #[instrument(skip(template_receiver, rpc))] pub async fn run_mining_solver( solver_id: u8, - template_receiver: WatchReceiver>>, + mut template_receiver: WatchReceiver>>, rpc: GetBlockTemplateRpcImpl, ) -> Result<(), Report> where @@ -273,8 +283,11 @@ where SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, { - while !is_shutting_down() { - // Get the latest block template. + // Shut down the task when the template sender is dropped, or Zebra shuts down. + while template_receiver.has_changed().is_ok() && !is_shutting_down() { + // Get the latest block template, and mark the current value as seen. + // We mark the value first to avoid missed updates. + template_receiver.mark_as_seen(); let template = template_receiver.cloned_watch_data(); let Some(template) = template else { From 29596f90667cdfb647ef66b06044d6071f8bcf52 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 9 Jan 2024 11:49:20 +1000 Subject: [PATCH 34/55] Reduce logging --- zebra-chain/src/work/equihash.rs | 2 +- zebrad/src/components/miner.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zebra-chain/src/work/equihash.rs b/zebra-chain/src/work/equihash.rs index deaf379b35a..50409ff8b6b 100644 --- a/zebra-chain/src/work/equihash.rs +++ b/zebra-chain/src/work/equihash.rs @@ -145,7 +145,7 @@ impl Solution { } } - info!( + debug!( solutions = ?solutions.len(), "found valid solutions which did not pass the validity or difficulty checks" ); diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index abe50c0e017..1d6f0e3860f 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -205,7 +205,7 @@ where // Wait for the chain to sync so we get a valid template. let Ok(template) = template else { - info!( + debug!( ?BLOCK_TEMPLATE_WAIT_TIME, "waiting for a valid block template", ); From dc39853273faed03af0f34c6d5270cb5b8c42fc1 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 9 Jan 2024 12:58:52 +1000 Subject: [PATCH 35/55] Fix a bug in getblocktemplate RPC tip change detection --- .../src/methods/get_block_template_rpcs.rs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index 3a040a6540e..776a39e8c21 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -660,6 +660,7 @@ where )); // Return immediately if the chain tip has changed. + latest_chain_tip.mark_best_tip_seen(); let wait_for_best_tip_change = latest_chain_tip.best_tip_changed(); // Wait for the maximum block time to elapse. This can change the block header @@ -709,6 +710,30 @@ where tip_changed_result = wait_for_best_tip_change => { match tip_changed_result { Ok(()) => { + // Despite the documentation, this future sometimes returns + // spuriously, even when the tip hasn't changed. This could be a + // bug where the state does spurious updates, or where the change + // detection or its future is implemented incorrectly. + let new_tip_hash = latest_chain_tip.best_tip_hash(); + if new_tip_hash == Some(tip_hash) { + tracing::debug!( + ?max_time, + ?cur_time, + ?server_long_poll_id, + ?client_long_poll_id, + ?tip_hash, + ?tip_height, + "ignoring spurious state change notification" + ); + + // Wait for the mempool interval, then check for any changes. + tokio::time::sleep(Duration::from_secs( + GET_BLOCK_TEMPLATE_MEMPOOL_LONG_POLL_INTERVAL, + )).await; + + continue; + } + tracing::info!( ?max_time, ?cur_time, From 9657302cd5c5868dd5db8d02e0bb72e0b6deba2b Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 9 Jan 2024 13:23:42 +1000 Subject: [PATCH 36/55] Work around some watch channel change bugs --- .../src/methods/get_block_template_rpcs.rs | 3 +- zebrad/src/components/miner.rs | 38 ++++++++++++++++--- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index 776a39e8c21..b2c5eb121d2 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -713,7 +713,8 @@ where // Despite the documentation, this future sometimes returns // spuriously, even when the tip hasn't changed. This could be a // bug where the state does spurious updates, or where the change - // detection or its future is implemented incorrectly. + // detection or its future is implemented incorrectly. Since it's a + // bug in both the RPC and miner, it could be a tokio bug. let new_tip_hash = latest_chain_tip.best_tip_hash(); if new_tip_hash == Some(tip_hash) { tracing::debug!( diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index 1d6f0e3860f..b7003a138ba 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -29,6 +29,7 @@ use zebra_rpc::{ config::mining::Config, methods::{ get_block_template_rpcs::{ + constants::GET_BLOCK_TEMPLATE_MEMPOOL_LONG_POLL_INTERVAL, get_block_template::{ self, proposal::TimeSource, proposal_block_from_template, GetBlockTemplateCapability::*, GetBlockTemplateRequestMode::*, @@ -235,8 +236,23 @@ where let block = proposal_block_from_template(&template, TimeSource::CurTime) .expect("unexpected invalid block template"); - // If all the receivers have been dropped, stop sending templates and shut down. - template_sender.send(Some(Arc::new(block)))?; + // If the template has actually changed, send an updated template. + template_sender.send_if_modified(|old_block| { + if old_block.as_ref().map(|b| *b.header) == Some(*block.header) { + return false; + } + *old_block = Some(Arc::new(block)); + true + }); + + // If the blockchain is changing rapidly, limit how often we'll update the template. + // But if we're shutting down, do that immediately. + if !template_sender.is_closed() && !is_shutting_down() { + sleep(Duration::from_secs( + GET_BLOCK_TEMPLATE_MEMPOOL_LONG_POLL_INTERVAL, + )) + .await; + } } Ok(()) @@ -313,12 +329,23 @@ where continue; }; - // Mine a block using the equihash solver. let height = template.coinbase_height().expect("template is valid"); - let cancel_receiver = template_receiver.clone(); + + // Set up the cancellation conditions for the miner. + let mut cancel_receiver = template_receiver.clone(); + let old_header = *template.header; let cancel_fn = move || match cancel_receiver.has_changed() { + // Despite the documentation, has_changed sometimes returns `true` spuriously, even + // when the template hasn't changed. This could be a bug where the RPC or block + // generator does spurious updates, or where the change detection is implemented + // incorrectly. Since it's a bug in both the RPC and miner, it could be a tokio bug. Ok(has_changed) => { - if has_changed { + cancel_receiver.mark_as_seen(); + // We only need to check header equality, because the block data is bound to the + // header. + if has_changed + && Some(old_header) != cancel_receiver.cloned_watch_data().map(|b| *b.header) + { Err(SolverCancelled) } else { Ok(()) @@ -328,6 +355,7 @@ where Err(_sender_dropped) => Err(SolverCancelled), }; + // Mine a block using the equihash solver. let Ok(block) = mine_one_block(solver_id, template, cancel_fn).await else { // If the solver was cancelled, we're either shutting down, or we have a new template. if solver_id == 0 { From 4edc15e3d7539d855d6b3db4d2d9cd3ff89527ee Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 9 Jan 2024 13:26:28 +1000 Subject: [PATCH 37/55] Rate-limit template changes in the receiver --- zebrad/src/components/miner.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index b7003a138ba..ce6b1f02131 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -375,6 +375,13 @@ where "solver cancelled: getting a new block template or shutting down" ); } + + // If the blockchain is changing rapidly, limit how often we'll update the template. + // But if we're shutting down, do that immediately. + if template_receiver.has_changed().is_ok() && !is_shutting_down() { + sleep(Duration::from_secs(1)).await; + } + continue; }; From b312ebb88d3e0999a6f0fc7f15b65b9a45dfe0f1 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 10 Jan 2024 10:48:32 +1000 Subject: [PATCH 38/55] Run one mining solver per available core --- zebrad/src/components/miner.rs | 35 +++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index ce6b1f02131..ce01a6bffda 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -6,9 +6,10 @@ //! //! - move common code into zebra-chain or zebra-node-services and remove the RPC dependency. -use std::{sync::Arc, time::Duration}; +use std::{cmp::min, sync::Arc, thread::available_parallelism, time::Duration}; use color_eyre::Report; +use futures::{stream::FuturesUnordered, TryStreamExt}; use thread_priority::{ThreadBuilder, ThreadPriority}; use tokio::{select, sync::watch, task::JoinHandle, time::sleep}; use tower::Service; @@ -55,6 +56,7 @@ pub fn spawn_init, ) -> JoinHandle> +// TODO: simplify or avoid repeating these generics (how?) where Mempool: Service< mempool::Request, @@ -129,21 +131,44 @@ where SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, { + // If we can't detect the number of cores, assume one core. + let solver_count = available_parallelism().map(usize::from).unwrap_or(1); + + info!( + ?solver_count, + "launching mining tasks with parallel solvers" + ); + let (template_sender, template_receiver) = watch::channel(None); let template_receiver = WatchReceiver::new(template_receiver); // TODO: add a config & launch the configured number of solvers, using available_parallelism() // by default let template_generator = generate_block_templates(rpc.clone(), template_sender); - let mining_solver = run_mining_solver(0, template_receiver, rpc); + + let mut mining_solvers = FuturesUnordered::new(); + for solver_id in 0..solver_count { + // Assume there are less than 256 cores. If there are more, only run 256 tasks. + let solver_id = min(solver_id, usize::from(u8::MAX)) + .try_into() + .expect("just limited to u8::MAX"); + + mining_solvers.push(run_mining_solver( + solver_id, + template_receiver.clone(), + rpc.clone(), + )); + } // These tasks run forever unless there is a fatal error or shutdown. - // When that happens, the first task to notice returns, and the other tasks are cancelled. + // When that happens, the first task to error returns, and the other futures are cancelled. // Then this task returns and drops the `template_sender`, which cancels all the spawned miner - // threads. + // threads. This only works because we run the `template_generator` in this task. If we spawned + // it, then the spawned task would take ownership of `template_sender` and keep running. + // (Mining solvers can be spawned, because they finish when the `template_sender` is dropped.) select! { result = template_generator => { result?; } - result = mining_solver => { result?; } + result = mining_solvers.try_next() => { result?; } } Ok(()) From 29ea11fcd9c4a3317a6a8980a1813a50da8f0eb6 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 10 Jan 2024 12:09:02 +1000 Subject: [PATCH 39/55] Use updated C code with double-free protection --- Cargo.lock | 2 +- zebra-chain/Cargo.toml | 2 +- zebrad/src/components/miner.rs | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ccd0a936fdc..b8cb2a3bb7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1349,7 +1349,7 @@ dependencies = [ [[package]] name = "equihash" version = "0.2.0" -source = "git+https://github.com/ZcashFoundation/librustzcash.git?branch=equihash-solver-tromp#da26c34772f4922eb13b4a1e7d88a969bbcf6a91" +source = "git+https://github.com/ZcashFoundation/librustzcash.git?branch=equihash-solver-tromp#838d1e937e8a6f23e99fa4ea4e1984013d423a6f" dependencies = [ "blake2b_simd", "byteorder", diff --git a/zebra-chain/Cargo.toml b/zebra-chain/Cargo.toml index 0c44b524b04..f5685581e37 100644 --- a/zebra-chain/Cargo.toml +++ b/zebra-chain/Cargo.toml @@ -80,7 +80,7 @@ equihash = "0.2.0" # Use the the working solver branch: equihash-solver = { version = "0.2.0", git = "https://github.com/ZcashFoundation/librustzcash.git", branch = "equihash-solver-tromp", features = ["solver"], package = "equihash", optional = true } # or during development, use the locally checked out and modified version of equihash: -#equihash-solver = { path = "../librustzcash/components/equihash", package = "equihash" } +#equihash-solver = { version = "0.2.0", path = "../../librustzcash/components/equihash", features = ["solver"], package = "equihash", optional = true } group = "0.13.0" incrementalmerkletree = "0.5.0" diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index ce01a6bffda..a4dbab4978b 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -51,7 +51,6 @@ pub const BLOCK_TEMPLATE_WAIT_TIME: Duration = Duration::from_secs(20); /// mining thread. /// /// See [`run_mining_solver()`] for more details. -#[instrument(skip(config, rpc))] pub fn spawn_init( config: &Config, rpc: GetBlockTemplateRpcImpl, From ad022c3d1b29f4ffdd94f162ae365db5eb709252 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 06:04:23 +1000 Subject: [PATCH 40/55] Update to the latest solver branch --- Cargo.lock | 2 +- zebra-chain/Cargo.toml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b8cb2a3bb7d..2fb608bd5b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1349,7 +1349,7 @@ dependencies = [ [[package]] name = "equihash" version = "0.2.0" -source = "git+https://github.com/ZcashFoundation/librustzcash.git?branch=equihash-solver-tromp#838d1e937e8a6f23e99fa4ea4e1984013d423a6f" +source = "git+https://github.com/ZcashFoundation/librustzcash.git?branch=equihash-solver-tromp#ee069c1a7afe29d301f83cbeebf88b075164589e" dependencies = [ "blake2b_simd", "byteorder", diff --git a/zebra-chain/Cargo.toml b/zebra-chain/Cargo.toml index f5685581e37..7b6d716ce78 100644 --- a/zebra-chain/Cargo.toml +++ b/zebra-chain/Cargo.toml @@ -77,7 +77,9 @@ equihash = "0.2.0" # https://github.com/zcash/librustzcash/pull/1083 # https://github.com/zcash/librustzcash/pull/1088 # -# Use the the working solver branch: +# Use the solver PR: +# - latest: branch = "equihash-solver-tromp", +# - crashing with double-frees: rev = "da26c34772f4922eb13b4a1e7d88a969bbcf6a91", equihash-solver = { version = "0.2.0", git = "https://github.com/ZcashFoundation/librustzcash.git", branch = "equihash-solver-tromp", features = ["solver"], package = "equihash", optional = true } # or during development, use the locally checked out and modified version of equihash: #equihash-solver = { version = "0.2.0", path = "../../librustzcash/components/equihash", features = ["solver"], package = "equihash", optional = true } From bccf803f2b67128f7df277d873be4c9f193b779d Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 06:19:28 +1000 Subject: [PATCH 41/55] Return and submit all valid solutions --- zebra-chain/src/work/equihash.rs | 24 ++++++--- zebrad/src/components/miner.rs | 88 ++++++++++++++++++-------------- 2 files changed, 67 insertions(+), 45 deletions(-) diff --git a/zebra-chain/src/work/equihash.rs b/zebra-chain/src/work/equihash.rs index 50409ff8b6b..810317f2207 100644 --- a/zebra-chain/src/work/equihash.rs +++ b/zebra-chain/src/work/equihash.rs @@ -12,6 +12,9 @@ use crate::{ }, }; +#[cfg(feature = "internal-miner")] +use crate::serialization::AtLeastOne; + /// The error type for Equihash validation. #[non_exhaustive] #[derive(Debug, thiserror::Error)] @@ -100,7 +103,10 @@ impl Solution { /// It can run for minutes or hours if the network difficulty is high. #[cfg(feature = "internal-miner")] #[allow(clippy::unwrap_in_result)] - pub fn solve(mut header: Header, mut cancel_fn: F) -> Result + pub fn solve( + mut header: Header, + mut cancel_fn: F, + ) -> Result, SolverCancelled> where F: FnMut() -> Result<(), SolverCancelled>, { @@ -127,6 +133,8 @@ impl Solution { Some(*header.nonce) }); + let mut valid_solutions = Vec::new(); + // If we got any solutions, try submitting them, because the new template might just // contain some extra transactions. Mining extra transactions is optional. for solution in &solutions { @@ -140,15 +148,17 @@ impl Solution { } if Self::difficulty_is_valid(&header) { - info!("found valid solution and difficulty"); - return Ok(header); + valid_solutions.push(header); } } - debug!( - solutions = ?solutions.len(), - "found valid solutions which did not pass the validity or difficulty checks" - ); + match valid_solutions.try_into() { + Ok(at_least_one_solution) => return Ok(at_least_one_solution), + Err(_is_empty_error) => debug!( + solutions = ?solutions.len(), + "found valid solutions which did not pass the validity or difficulty checks" + ), + } } Err(SolverCancelled) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index a4dbab4978b..42da75379b5 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -20,7 +20,7 @@ use zebra_chain::{ chain_sync_status::ChainSyncStatus, chain_tip::ChainTip, diagnostic::task::WaitForPanics, - serialization::ZcashSerialize, + serialization::{AtLeastOne, ZcashSerialize}, shutdown::is_shutting_down, work::equihash::{Solution, SolverCancelled}, }; @@ -379,8 +379,8 @@ where Err(_sender_dropped) => Err(SolverCancelled), }; - // Mine a block using the equihash solver. - let Ok(block) = mine_one_block(solver_id, template, cancel_fn).await else { + // Mine at least one block using the equihash solver. + let Ok(blocks) = mine_a_block(solver_id, template, cancel_fn).await else { // If the solver was cancelled, we're either shutting down, or we have a new template. if solver_id == 0 { info!( @@ -409,47 +409,50 @@ where continue; }; - // Submit the newly mined block to the verifiers. + // Submit the newly mined blocks to the verifiers. // // TODO: if there is a new template (`cancel_fn().is_err()`), and // GetBlockTemplate.submit_old is false, return immediately, and skip submitting the - // block. - let data = block - .zcash_serialize_to_vec() - .expect("serializing to Vec never fails"); - - match rpc.submit_block(HexData(data), None).await { - Ok(success) => info!( - ?height, - hash = ?block.hash(), - ?solver_id, - ?success, - "successfully mined a new block", - ), - Err(error) => info!( - ?height, - hash = ?block.hash(), - ?solver_id, - ?error, - "validating a newly mined block failed, trying again", - ), + // blocks. + for block in blocks { + let data = block + .zcash_serialize_to_vec() + .expect("serializing to Vec never fails"); + + match rpc.submit_block(HexData(data), None).await { + Ok(success) => info!( + ?height, + hash = ?block.hash(), + ?solver_id, + ?success, + "successfully mined a new block", + ), + Err(error) => info!( + ?height, + hash = ?block.hash(), + ?solver_id, + ?error, + "validating a newly mined block failed, trying again", + ), + } } } Ok(()) } -/// Mines a single block based on `template`, calculates its equihash solutions, checks difficulty, -/// and returning the newly mined block. Uses a different nonce range for each `solver_id`. +/// Mines one or more blocks based on `template`. Calculates equihash solutions, checks difficulty, +/// and returns as soon as it has at least one block. Uses a different nonce range for each +/// `solver_id`. /// /// If `cancel_fn()` returns an error, returns early with `Err(SolverCancelled)`. /// /// See [`run_mining_solver()`] for more details. -pub async fn mine_one_block( +pub async fn mine_a_block( solver_id: u8, - mut template: Arc, + template: Arc, cancel_fn: F, -) -> Result, SolverCancelled> +) -> Result, SolverCancelled> where F: FnMut() -> Result<(), SolverCancelled> + Send + Sync + 'static, { @@ -463,10 +466,9 @@ where *header.nonce.first_mut().unwrap() = solver_id; *header.nonce.last_mut().unwrap() = solver_id; - // Mine a block using the solver, in a low-priority blocking thread. + // Mine one or more blocks using the solver, in a low-priority blocking thread. let span = Span::current(); - // TODO: get and submit all valid headers, not just the first one - let solved_header = + let solved_headers = tokio::task::spawn_blocking(move || span.in_scope(move || { let miner_thread_handle = ThreadBuilder::default().name("zebra-miner").priority(ThreadPriority::Min).spawn(move |priority_result| { if let Err(error) = priority_result { @@ -481,11 +483,21 @@ where .wait_for_panics() .await?; - // Modify the template into a solved block. - // - // TODO: Replace with Arc::unwrap_or_clone() when it stabilises - let block = Arc::make_mut(&mut template); - block.header = Arc::new(solved_header); + // Modify the template into solved blocks. - Ok(template) + // TODO: Replace with Arc::unwrap_or_clone() when it stabilises + let block = (*template).clone(); + + let solved_blocks: Vec = solved_headers + .into_iter() + .map(|header| { + let mut block = block.clone(); + block.header = Arc::new(header); + block + }) + .collect(); + + Ok(solved_blocks + .try_into() + .expect("a 1:1 mapping of AtLeastOne produces at least one block")) } From 6e6bc01c19f45c2ecf13316a2b6fc3d3002cc7c3 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 06:20:30 +1000 Subject: [PATCH 42/55] Document what INPUT_LENGTH means --- zebra-chain/src/work/equihash.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zebra-chain/src/work/equihash.rs b/zebra-chain/src/work/equihash.rs index 810317f2207..18903c935f1 100644 --- a/zebra-chain/src/work/equihash.rs +++ b/zebra-chain/src/work/equihash.rs @@ -61,6 +61,8 @@ impl Solution { .zcash_serialize(&mut input) .expect("serialization into a vec can't fail"); + // The part of the header before the nonce and solution. + // This data is kept constant during solver runs, so the verifier API takes it separately. let input = &input[0..Solution::INPUT_LENGTH]; equihash::is_valid_solution(n, k, input, nonce.as_ref(), solution)?; @@ -116,6 +118,8 @@ impl Solution { header .zcash_serialize(&mut input) .expect("serialization into a vec can't fail"); + // Take the part of the header before the nonce and solution. + // This data is kept constant for this solver run. let input = &input[0..Solution::INPUT_LENGTH]; while !is_shutting_down() { From 8656076e39e59c47d9bb6d8e6baef4e1f376d751 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 06:32:41 +1000 Subject: [PATCH 43/55] Fix watch channel change detection --- zebra-rpc/src/methods/get_block_template_rpcs.rs | 15 ++++++++------- zebra-state/src/service/watch_receiver.rs | 7 ++++++- zebrad/src/components/miner.rs | 8 ++++---- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index b2c5eb121d2..eb9eff8d00a 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -660,8 +660,9 @@ where )); // Return immediately if the chain tip has changed. - latest_chain_tip.mark_best_tip_seen(); - let wait_for_best_tip_change = latest_chain_tip.best_tip_changed(); + // The clone preserves the seen status of the chain tip. + let mut wait_for_best_tip_change = latest_chain_tip.clone(); + let wait_for_best_tip_change = wait_for_best_tip_change.best_tip_changed(); // Wait for the maximum block time to elapse. This can change the block header // on testnet. (On mainnet it can happen due to a network disconnection, or a @@ -710,11 +711,11 @@ where tip_changed_result = wait_for_best_tip_change => { match tip_changed_result { Ok(()) => { - // Despite the documentation, this future sometimes returns - // spuriously, even when the tip hasn't changed. This could be a - // bug where the state does spurious updates, or where the change - // detection or its future is implemented incorrectly. Since it's a - // bug in both the RPC and miner, it could be a tokio bug. + // Spurious updates shouldn't happen in the state, because the + // difficulty and hash ordering is a stable total order. But + // since they could cause a busy-loop, guard against them here. + latest_chain_tip.mark_best_tip_seen(); + let new_tip_hash = latest_chain_tip.best_tip_hash(); if new_tip_hash == Some(tip_hash) { tracing::debug!( diff --git a/zebra-state/src/service/watch_receiver.rs b/zebra-state/src/service/watch_receiver.rs index 34a3d5e87ea..6c4aba0b564 100644 --- a/zebra-state/src/service/watch_receiver.rs +++ b/zebra-state/src/service/watch_receiver.rs @@ -116,8 +116,13 @@ where } /// Marks the watched data as seen. - /// Calls [`watch::Receiver::mark_changed()`]. pub fn mark_as_seen(&mut self) { + self.receiver.borrow_and_update(); + } + + /// Marks the watched data as unseen. + /// Calls [`watch::Receiver::mark_changed()`]. + pub fn mark_changed(&mut self) { self.receiver.mark_changed(); } } diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index 42da75379b5..4c2cb6f18b6 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -359,12 +359,12 @@ where let mut cancel_receiver = template_receiver.clone(); let old_header = *template.header; let cancel_fn = move || match cancel_receiver.has_changed() { - // Despite the documentation, has_changed sometimes returns `true` spuriously, even - // when the template hasn't changed. This could be a bug where the RPC or block - // generator does spurious updates, or where the change detection is implemented - // incorrectly. Since it's a bug in both the RPC and miner, it could be a tokio bug. + // Guard against get_block_template() providing an identical header. This could happen + // if something irrelevant to the block data changes, the time was within 1 second, or + // there is a spurious channel change. Ok(has_changed) => { cancel_receiver.mark_as_seen(); + // We only need to check header equality, because the block data is bound to the // header. if has_changed From ced81077ce353371dfb8ccb2367af983292c133c Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 06:42:01 +1000 Subject: [PATCH 44/55] Don't return early when a mining task fails --- zebrad/src/components/miner.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index 4c2cb6f18b6..93e333a0abe 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -131,6 +131,9 @@ where AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, { // If we can't detect the number of cores, assume one core. + // + // TODO: add a config & launch the configured number of solvers, using available_parallelism() + // by default let solver_count = available_parallelism().map(usize::from).unwrap_or(1); info!( @@ -141,8 +144,7 @@ where let (template_sender, template_receiver) = watch::channel(None); let template_receiver = WatchReceiver::new(template_receiver); - // TODO: add a config & launch the configured number of solvers, using available_parallelism() - // by default + // TODO: Spawn these tasks, to avoid blocked cooperative futures, and improve shutdown responsiveness. let template_generator = generate_block_templates(rpc.clone(), template_sender); let mut mining_solvers = FuturesUnordered::new(); @@ -161,16 +163,17 @@ where // These tasks run forever unless there is a fatal error or shutdown. // When that happens, the first task to error returns, and the other futures are cancelled. - // Then this task returns and drops the `template_sender`, which cancels all the spawned miner - // threads. This only works because we run the `template_generator` in this task. If we spawned - // it, then the spawned task would take ownership of `template_sender` and keep running. - // (Mining solvers can be spawned, because they finish when the `template_sender` is dropped.) + let first_result; select! { - result = template_generator => { result?; } - result = mining_solvers.try_next() => { result?; } + result = template_generator => { first_result = result; } + result = mining_solvers.try_next() => { first_result = result.map(|ok| ok.expect("there is at least one solver")); } } - Ok(()) + // When this task returns and drops the `template_sender`, it cancels all the spawned miner + // threads. This only works because we run the `template_generator` in this task. If we spawned + // it, then the spawned task would take ownership of `template_sender` and keep running. + // (Mining solvers can be spawned, because they finish when the `template_sender` is dropped.) + first_result } /// Generates block templates using `rpc`, and sends them to mining threads using `template_sender`. From 5bcc618170df5ed66e039482658158a057f814e0 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 07:42:48 +1000 Subject: [PATCH 45/55] Spawn async miner tasks to avoid cooperative blocking, deadlocks, and improve shutdown responsiveness --- zebrad/src/components/miner.rs | 44 ++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index 93e333a0abe..aeefcec59c0 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -144,8 +144,15 @@ where let (template_sender, template_receiver) = watch::channel(None); let template_receiver = WatchReceiver::new(template_receiver); - // TODO: Spawn these tasks, to avoid blocked cooperative futures, and improve shutdown responsiveness. - let template_generator = generate_block_templates(rpc.clone(), template_sender); + // Spawn these tasks, to avoid blocked cooperative futures, and improve shutdown responsiveness. + // This is particularly important when there are a large number of solver threads. + let mut abort_handles = Vec::new(); + + let template_generator = tokio::task::spawn( + generate_block_templates(rpc.clone(), template_sender).in_current_span(), + ); + abort_handles.push(template_generator.abort_handle()); + let template_generator = template_generator.wait_for_panics(); let mut mining_solvers = FuturesUnordered::new(); for solver_id in 0..solver_count { @@ -154,25 +161,36 @@ where .try_into() .expect("just limited to u8::MAX"); - mining_solvers.push(run_mining_solver( - solver_id, - template_receiver.clone(), - rpc.clone(), - )); + let solver = tokio::task::spawn( + run_mining_solver(solver_id, template_receiver.clone(), rpc.clone()).in_current_span(), + ); + abort_handles.push(solver.abort_handle()); + + mining_solvers.push(solver.wait_for_panics()); } // These tasks run forever unless there is a fatal error or shutdown. - // When that happens, the first task to error returns, and the other futures are cancelled. + // When that happens, the first task to error returns, and the other JoinHandle futures are + // cancelled. let first_result; select! { result = template_generator => { first_result = result; } - result = mining_solvers.try_next() => { first_result = result.map(|ok| ok.expect("there is at least one solver")); } + result = mining_solvers.try_next() => { + first_result = result + .transpose() + .expect("stream never teminates because there is at least one solver task"); + } + } + + // But the spawned async tasks keep running, so we need to abort them here. + for abort_handle in abort_handles { + abort_handle.abort(); } - // When this task returns and drops the `template_sender`, it cancels all the spawned miner - // threads. This only works because we run the `template_generator` in this task. If we spawned - // it, then the spawned task would take ownership of `template_sender` and keep running. - // (Mining solvers can be spawned, because they finish when the `template_sender` is dropped.) + // Any spawned blocking threads will keep running. When this task returns and drops the + // `template_sender`, it cancels all the spawned miner threads. This works because we've + // aborted the `template_generator` task, which owns the `template_sender`. (And it doesn't + // spawn any blocking threads.) first_result } From dad12ad18f2e91ca0517e740f26d3df9c693d051 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 08:10:10 +1000 Subject: [PATCH 46/55] Make existing parallelism docs and configs consistent --- Cargo.lock | 1 - zebra-rpc/Cargo.toml | 1 - zebra-rpc/src/config.rs | 10 +++++----- zebra-rpc/src/server.rs | 4 ++-- zebrad/src/components/sync.rs | 3 ++- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e3169070976..4f0b4a92bb3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5898,7 +5898,6 @@ dependencies = [ "jsonrpc-core", "jsonrpc-derive", "jsonrpc-http-server", - "num_cpus", "proptest", "rand 0.8.5", "serde", diff --git a/zebra-rpc/Cargo.toml b/zebra-rpc/Cargo.toml index 21159478a1b..b7f9e9bab67 100644 --- a/zebra-rpc/Cargo.toml +++ b/zebra-rpc/Cargo.toml @@ -48,7 +48,6 @@ hyper = { version = "0.14.28", features = ["http1", "server"] } jsonrpc-core = "18.0.0" jsonrpc-derive = "18.0.0" jsonrpc-http-server = "18.0.0" -num_cpus = "1.16.0" # zebra-rpc needs the preserve_order feature in serde_json, which is a dependency of jsonrpc-core serde_json = { version = "1.0.108", features = ["preserve_order"] } diff --git a/zebra-rpc/src/config.rs b/zebra-rpc/src/config.rs index b6bbf5196ef..3f74ead07db 100644 --- a/zebra-rpc/src/config.rs +++ b/zebra-rpc/src/config.rs @@ -36,17 +36,17 @@ pub struct Config { /// State queries are run concurrently using the shared thread pool controlled by /// the [`SyncSection.parallel_cpu_threads`](https://docs.rs/zebrad/latest/zebrad/components/sync/struct.Config.html#structfield.parallel_cpu_threads) config. /// - /// We recommend setting both configs to `0` (automatic scaling) for the best performance. - /// This uses one thread per available CPU core. + /// If the number of threads is not configured or zero, Zebra uses the number of logical cores. + /// If the number of logical cores can't be detected, Zebra uses one thread. /// - /// Set to `1` by default, which runs all RPC queries on a single thread, and detects RPC - /// port conflicts from multiple Zebra or `zcashd` instances. + /// Set to `1` to run all RPC queries on a single thread, and detect RPC port conflicts from + /// multiple Zebra or `zcashd` instances. /// /// For details, see [the `jsonrpc_http_server` documentation](https://docs.rs/jsonrpc-http-server/latest/jsonrpc_http_server/struct.ServerBuilder.html#method.threads). /// /// ## Warning /// - /// Changing this config disables RPC port conflict detection. + /// The default config uses multiple threads, which disables RPC port conflict detection. /// This can allow multiple Zebra instances to share the same RPC port. /// /// If some of those instances are outdated or failed, RPC queries can be slow or inconsistent. diff --git a/zebra-rpc/src/server.rs b/zebra-rpc/src/server.rs index de78c1d6c64..c8e13a9a62f 100644 --- a/zebra-rpc/src/server.rs +++ b/zebra-rpc/src/server.rs @@ -7,7 +7,7 @@ //! See the full list of //! [Differences between JSON-RPC 1.0 and 2.0.](https://www.simple-is-better.org/rpc/#differences-between-1-0-and-2-0) -use std::{fmt, panic}; +use std::{fmt, panic, thread::available_parallelism}; use jsonrpc_core::{Compatibility, MetaIoHandler}; use jsonrpc_http_server::{CloseHandle, ServerBuilder}; @@ -187,7 +187,7 @@ impl RpcServer { // If zero, automatically scale threads to the number of CPU cores let mut parallel_cpu_threads = config.parallel_cpu_threads; if parallel_cpu_threads == 0 { - parallel_cpu_threads = num_cpus::get(); + parallel_cpu_threads = available_parallelism().map(usize::from).unwrap_or(1); } // The server is a blocking task, which blocks on executor shutdown. diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs index 160e27777f3..651dacc1f8d 100644 --- a/zebrad/src/components/sync.rs +++ b/zebrad/src/components/sync.rs @@ -261,7 +261,8 @@ pub struct Config { /// The number of threads used to verify signatures, proofs, and other CPU-intensive code. /// - /// Set to `0` by default, which uses one thread per available CPU core. + /// If the number of threads is not configured or zero, Zebra uses the number of logical cores. + /// If the number of logical cores can't be detected, Zebra uses one thread. /// For details, see [the `rayon` documentation](https://docs.rs/rayon/latest/rayon/struct.ThreadPoolBuilder.html#method.num_threads). pub parallel_cpu_threads: usize, } From 54d27cbc9e08d7028970b4c46cee01c21aa95b49 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 08:10:39 +1000 Subject: [PATCH 47/55] Add a mining parallelism config --- zebra-rpc/Cargo.toml | 3 +++ zebra-rpc/src/config/mining.rs | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/zebra-rpc/Cargo.toml b/zebra-rpc/Cargo.toml index b7f9e9bab67..cff2f724c0e 100644 --- a/zebra-rpc/Cargo.toml +++ b/zebra-rpc/Cargo.toml @@ -29,6 +29,9 @@ getblocktemplate-rpcs = [ "zebra-chain/getblocktemplate-rpcs", ] +# Experimental internal miner support +internal-miner = [] + # Test-only features proptest-impl = [ "proptest", diff --git a/zebra-rpc/src/config/mining.rs b/zebra-rpc/src/config/mining.rs index 1a27baa3646..65cea3bbda7 100644 --- a/zebra-rpc/src/config/mining.rs +++ b/zebra-rpc/src/config/mining.rs @@ -15,6 +15,28 @@ pub struct Config { /// `getblocktemplate` RPC coinbase transaction. pub miner_address: Option, + /// Mine blocks using Zebra's internal miner, without an external mining pool or equihash solver. + /// + /// This experimental feature is only supported on testnet. + /// Mainnet miners should use a mining pool with GPUs or ASICs designed for efficient mining. + /// + /// The internal miner is off by default. + #[cfg(feature = "internal-miner")] + pub internal_miner: bool, + + /// The number of internal miner threads used by Zebra. + /// These threads are scheduled at low priority. + /// + /// The number of threads is limited by the available parallelism reported by the OS. + /// If the number of threads isn't configured, or can't be detected, Zebra uses one thread. + /// This is different from Zebra's other parallelism configs, because mining runs constantly and + /// uses a large amount of memory. (144 MB of RAM and 100% of a core per thread.) + /// + /// If the number of threads is set to zero, Zebra disables mining. + /// This matches `zcashd`'s behaviour, but is different from Zebra's other parallelism configs. + #[cfg(feature = "internal-miner")] + pub internal_miner_threads: usize, + /// Extra data to include in coinbase transaction inputs. /// Limited to around 95 bytes by the consensus rules. /// @@ -36,6 +58,12 @@ impl Default for Config { // TODO: do we want to default to v5 transactions and Zebra coinbase data? extra_coinbase_data: None, debug_like_zcashd: true, + // TODO: ignore and warn rather than panicking if these fields are in the config, + // but the feature isn't enabled. + #[cfg(feature = "internal-miner")] + internal_miner: false, + #[cfg(feature = "internal-miner")] + internal_miner_threads: 1, } } } @@ -48,4 +76,10 @@ impl Config { pub fn skip_getblocktemplate(&self) -> bool { !cfg!(feature = "getblocktemplate-rpcs") } + + /// Is the internal miner enabled using at least one thread? + #[cfg(feature = "internal-miner")] + pub fn is_internal_miner_enabled(&self) -> bool { + self.internal_miner && self.internal_miner_threads > 0 + } } From cc1e73748e8d7492e8889156d651fe30504bdc89 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 08:29:55 +1000 Subject: [PATCH 48/55] Use the minimum of the configured or available threads for mining --- zebrad/Cargo.toml | 1 + zebrad/src/commands/start.rs | 4 +++- zebrad/src/components/miner.rs | 15 +++++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index bd7156970e8..a2ef9d82876 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -73,6 +73,7 @@ internal-miner = [ "thread-priority", "zebra-chain/internal-miner", # TODO: move common code into zebra-chain or zebra-node-services and remove the RPC dependency + "zebra-rpc/internal-miner", "zebra-rpc/getblocktemplate-rpcs", ] diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 4d5f33b1b37..01e3a0b88bb 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -313,7 +313,7 @@ impl StartCmd { // // TODO: add a config to enable the miner rather than a feature. #[cfg(feature = "internal-miner")] - let miner_task_handle = { + let miner_task_handle = if config.mining.is_internal_miner_enabled() { info!("spawning Zcash miner"); let rpc = zebra_rpc::methods::get_block_template_rpcs::GetBlockTemplateRpcImpl::new( config.network.network, @@ -327,6 +327,8 @@ impl StartCmd { ); crate::components::miner::spawn_init(&config.mining, rpc) + } else { + tokio::spawn(std::future::pending().in_current_span()) }; #[cfg(not(feature = "internal-miner"))] diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index aeefcec59c0..4970e546c30 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -98,7 +98,7 @@ where /// /// See [`run_mining_solver()`] for more details. pub async fn init( - _config: Config, + config: Config, rpc: GetBlockTemplateRpcImpl, ) -> Result<(), Report> where @@ -130,11 +130,14 @@ where SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, { - // If we can't detect the number of cores, assume one core. - // - // TODO: add a config & launch the configured number of solvers, using available_parallelism() - // by default - let solver_count = available_parallelism().map(usize::from).unwrap_or(1); + let configured_threads = config.internal_miner_threads; + // If we can't detect the number of cores, use the configured number. + let available_threads = available_parallelism() + .map(usize::from) + .unwrap_or(configured_threads); + + // Use the minimum of the configured and available threads. + let solver_count = min(configured_threads, available_threads); info!( ?solver_count, From c70438a9ef406f58d4de8f01cba4900d811efd08 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 08:39:51 +1000 Subject: [PATCH 49/55] Ignore optional feature fields in tests --- .../src/methods/tests/snapshot/get_block_template_rpcs.rs | 3 +++ zebra-rpc/src/methods/tests/vectors.rs | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs index 29dd8c577d6..73e7f43db58 100644 --- a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs @@ -92,10 +92,13 @@ pub async fn test_responses( let mut mock_sync_status = MockSyncStatus::default(); mock_sync_status.set_is_close_to_tip(true); + #[allow(clippy::unnecessary_struct_initialization)] let mining_config = crate::config::mining::Config { miner_address: Some(transparent::Address::from_script_hash(network, [0xad; 20])), extra_coinbase_data: None, debug_like_zcashd: true, + // Use default field values when optional features are enabled in tests + ..Default::default() }; // nu5 block height diff --git a/zebra-rpc/src/methods/tests/vectors.rs b/zebra-rpc/src/methods/tests/vectors.rs index d6e67e27fd3..bfcc2affbef 100644 --- a/zebra-rpc/src/methods/tests/vectors.rs +++ b/zebra-rpc/src/methods/tests/vectors.rs @@ -1231,10 +1231,13 @@ async fn rpc_getblocktemplate_mining_address(use_p2pkh: bool) { true => Some(transparent::Address::from_pub_key_hash(Mainnet, [0x7e; 20])), }; + #[allow(clippy::unnecessary_struct_initialization)] let mining_config = crate::config::mining::Config { miner_address, extra_coinbase_data: None, debug_like_zcashd: true, + // Use default field values when optional features are enabled in tests + ..Default::default() }; // nu5 block height @@ -1677,10 +1680,13 @@ async fn rpc_getdifficulty() { let mut mock_sync_status = MockSyncStatus::default(); mock_sync_status.set_is_close_to_tip(true); + #[allow(clippy::unnecessary_struct_initialization)] let mining_config = Config { miner_address: None, extra_coinbase_data: None, debug_like_zcashd: true, + // Use default field values when optional features are enabled in tests + ..Default::default() }; // nu5 block height From 1c6802fe63cfa76b6d947b8c9f10ec2df95877a0 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 08:48:12 +1000 Subject: [PATCH 50/55] Downgrade some frequent logs to debug --- zebra-rpc/src/methods/get_block_template_rpcs.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index eb9eff8d00a..7d0550c0598 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -689,7 +689,6 @@ where // But the coinbase value depends on the selected transactions, so this needs // further analysis to check if it actually saves us any time. - // TODO: change logging to debug after testing tokio::select! { // Poll the futures in the listed order, for efficiency. // We put the most frequent conditions first. @@ -697,7 +696,7 @@ where // This timer elapses every few seconds _elapsed = wait_for_mempool_request => { - tracing::info!( + tracing::debug!( ?max_time, ?cur_time, ?server_long_poll_id, @@ -736,7 +735,7 @@ where continue; } - tracing::info!( + tracing::debug!( ?max_time, ?cur_time, ?server_long_poll_id, @@ -746,8 +745,7 @@ where } Err(recv_error) => { - // This log should stay at info when the others go to debug, - // it will help with debugging. + // This log is rare and helps with debugging, so it's ok to be info. tracing::info!( ?recv_error, ?max_time, @@ -770,8 +768,7 @@ where // The max time does not elapse during normal operation on mainnet, // and it rarely elapses on testnet. Some(_elapsed) = wait_for_max_time => { - // This log should stay at info when the others go to debug, - // it's very rare. + // This log is very rare so it's ok to be info. tracing::info!( ?max_time, ?cur_time, From df6bac447df83e844d6ac27ae6736eb8f69c9f92 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 08:59:13 +1000 Subject: [PATCH 51/55] Document new zebrad features and tasks --- zebrad/src/commands/start.rs | 10 ++++++++++ zebrad/src/lib.rs | 7 +++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 01e3a0b88bb..e9b98cc523d 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -45,6 +45,16 @@ //! * Progress Task //! * logs progress towards the chain tip //! +//! Shielded Scanning: +//! * Shielded Scanner Task +//! * if the user has configured Zebra with their shielded viewing keys, scans new and existing +//! blocks for transactions that use those keys +//! +//! Block Mining: +//! * Internal Miner Task +//! * if the user has configured Zebra to mine blocks, spawns tasks to generate new blocks, +//! and submits them for verification. This automatically shares these new blocks with peers. +//! //! Mempool Transactions: //! * Mempool Service //! * activates when the syncer is near the chain tip diff --git a/zebrad/src/lib.rs b/zebrad/src/lib.rs index ab4c790e45a..2043c082521 100644 --- a/zebrad/src/lib.rs +++ b/zebrad/src/lib.rs @@ -111,9 +111,12 @@ //! ### Experimental //! //! * `elasticsearch`: save block data into elasticsearch database. Read the [elasticsearch](https://zebra.zfnd.org/user/elasticsearch.html) -//! section of the book for more details. +//! section of the book for more details. //! * `shielded-scan`: enable experimental support for scanning shielded transactions. Read the [shielded-scan](https://zebra.zfnd.org/user/shielded-scan.html) -//! section of the book for more details. +//! section of the book for more details. +//! * `internal-miner`: enable experimental support for mining inside Zebra, without an external +//! mining pool. This feature is only supported on testnet. Use a GPU or ASIC on mainnet for +//! efficient mining. #![doc(html_favicon_url = "https://zfnd.org/wp-content/uploads/2022/03/zebra-favicon-128.png")] #![doc(html_logo_url = "https://zfnd.org/wp-content/uploads/2022/03/zebra-icon.png")] From fa850f5e34a16f6717a826a3a2717b2b0e56952d Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 08:59:32 +1000 Subject: [PATCH 52/55] Describe the internal-miner feature in the CHANGELOG --- CHANGELOG.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dea8996f90f..af4106660e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,20 @@ All notable changes to Zebra are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org). +## [Zebra 1.6.0](https://github.com/ZcashFoundation/zebra/releases/tag/v1.6.0) - TODO: 2024-01-?? + +This release: +- TODO: summary of other important changes +- adds an experimental `internal-miner` feature, which mines blocks within `zebrad`. This feature + is only supported on testnet. Use a more efficient GPU or ASIC for mainnet mining. + +TODO: the rest of the changelog + + ## [Zebra 1.5.0](https://github.com/ZcashFoundation/zebra/releases/tag/v1.5.0) - 2023-11-28 This release: -- fixes a panic that was introduced in Zebra v1.4.0, which happens in rare circumstances when reading cached sprout or history trees. +- fixes a panic that was introduced in Zebra v1.4.0, which happens in rare circumstances when reading cached sprout or history trees. - further improves how Zebra recovers from network interruptions and prevents potential network hangs. - limits the ability of synthetic nodes to spread throughout the network through Zebra to address some of the Ziggurat red team report. From 1b180e1925762c99d094e3faf536f483d5366590 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 09:17:03 +1000 Subject: [PATCH 53/55] Update dependency to de-duplicate equihash solutions --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 4f0b4a92bb3..e23547a956a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1349,7 +1349,7 @@ dependencies = [ [[package]] name = "equihash" version = "0.2.0" -source = "git+https://github.com/ZcashFoundation/librustzcash.git?branch=equihash-solver-tromp#ee069c1a7afe29d301f83cbeebf88b075164589e" +source = "git+https://github.com/ZcashFoundation/librustzcash.git?branch=equihash-solver-tromp#251098313920466958fcd05b25e151d4edd3a1b1" dependencies = [ "blake2b_simd", "byteorder", From 57bf84da344da1282092223745300eed50ebb421 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 09:20:29 +1000 Subject: [PATCH 54/55] Use futures::StreamExt instead of TryStreamExt --- zebrad/src/components/miner.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index 4970e546c30..69d7a8347c1 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -9,7 +9,7 @@ use std::{cmp::min, sync::Arc, thread::available_parallelism, time::Duration}; use color_eyre::Report; -use futures::{stream::FuturesUnordered, TryStreamExt}; +use futures::{stream::FuturesUnordered, StreamExt}; use thread_priority::{ThreadBuilder, ThreadPriority}; use tokio::{select, sync::watch, task::JoinHandle, time::sleep}; use tower::Service; @@ -178,9 +178,8 @@ where let first_result; select! { result = template_generator => { first_result = result; } - result = mining_solvers.try_next() => { + result = mining_solvers.next() => { first_result = result - .transpose() .expect("stream never teminates because there is at least one solver task"); } } From 44298fbc2c5b91f93f69627b71e2e360befe406d Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 11 Jan 2024 09:32:13 +1000 Subject: [PATCH 55/55] Fix a panic message typo --- zebrad/src/components/miner.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index 69d7a8347c1..b337c7669ee 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -180,7 +180,7 @@ where result = template_generator => { first_result = result; } result = mining_solvers.next() => { first_result = result - .expect("stream never teminates because there is at least one solver task"); + .expect("stream never terminates because there is at least one solver task"); } }