From 25b8404e2df8a03eab4b419a4cf97c2323125cf2 Mon Sep 17 00:00:00 2001 From: oscar-pepper <109323234+Oscar-Pepper@users.noreply.github.com> Date: Mon, 1 Apr 2024 21:55:16 +0100 Subject: [PATCH] Add block cache trait (#1192) --- Cargo.lock | 47 ++--- zcash_client_backend/CHANGELOG.md | 2 + zcash_client_backend/Cargo.toml | 6 + zcash_client_backend/src/data_api/chain.rs | 206 ++++++++++++++++++++- zcash_client_backend/src/scanning.rs | 45 +++-- 5 files changed, 264 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9657010b5b..e43ea00643 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -115,18 +115,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] name = "async-trait" -version = "0.1.74" +version = "0.1.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +checksum = "461abc97219de0eaaf81fe3ef974a540158f3d079c2ab200f891f1a2ef201e85" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -1613,7 +1613,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -1708,7 +1708,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -1723,9 +1723,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] @@ -1777,7 +1777,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.39", + "syn 2.0.53", "tempfile", "which", ] @@ -1792,7 +1792,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -1821,9 +1821,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -2220,7 +2220,7 @@ checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -2366,9 +2366,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.39" +version = "2.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032" dependencies = [ "proc-macro2", "quote", @@ -2423,7 +2423,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -2488,6 +2488,7 @@ dependencies = [ "bytes", "libc", "mio", + "num_cpus", "pin-project-lite", "socket2 0.5.5", "windows-sys", @@ -2565,7 +2566,7 @@ dependencies = [ "proc-macro2", "prost-build", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -2619,7 +2620,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -2821,7 +2822,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", "wasm-bindgen-shared", ] @@ -2843,7 +2844,7 @@ checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3012,6 +3013,7 @@ name = "zcash_client_backend" version = "0.12.1" dependencies = [ "assert_matches", + "async-trait", "base64", "bech32", "bls12_381", @@ -3039,6 +3041,7 @@ dependencies = [ "shardtree", "subtle", "time", + "tokio", "tonic", "tonic-build", "tracing", @@ -3277,7 +3280,7 @@ checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -3297,7 +3300,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 009c79ea2b..ac4d35c3d7 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -39,6 +39,7 @@ and this library adheres to Rust's notion of - `WalletSummary::next_orchard_subtree_index` - `chain::ChainState` - `chain::ScanSummary::{spent_orchard_note_count, received_orchard_note_count}` + - `chain::BlockCache` trait - `impl Debug for chain::CommitmentTreeRoot` - `zcash_client_backend::fees`: - `orchard` @@ -54,6 +55,7 @@ and this library adheres to Rust's notion of - `Nullifiers::{orchard, extend_orchard, retain_orchard}` - `TaggedOrchardBatch` - `TaggedOrchardBatchRunner` + - `testing` module - `zcash_client_backend::wallet`: - `Note::Orchard` - `WalletOrchardSpend` diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index c547dc343b..d5347b56c6 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -84,10 +84,14 @@ shardtree.workspace = true # - Test dependencies proptest = { workspace = true, optional = true } +jubjub = { workspace = true, optional = true } # - ZIP 321 nom = "7" +# - Asychronous +async-trait = "0.1.78" + # Dependencies used internally: # (Breaking upgrades to these are usually backwards-compatible, but check MSRVs.) # - Documentation @@ -116,6 +120,7 @@ shardtree = { workspace = true, features = ["test-dependencies"] } zcash_proofs.workspace = true zcash_address = { workspace = true, features = ["test-dependencies"] } zcash_keys = { workspace = true, features = ["test-dependencies"] } +tokio = { version = "1.21.0", features = ["rt-multi-thread"] } time = ">=0.3.22, <0.3.24" # time 0.3.24 has MSRV 1.67 @@ -139,6 +144,7 @@ orchard = ["dep:orchard", "zcash_keys/orchard"] ## Exposes APIs that are useful for testing, such as `proptest` strategies. test-dependencies = [ "dep:proptest", + "dep:jubjub", "orchard?/test-dependencies", "zcash_keys/test-dependencies", "zcash_primitives/test-dependencies", diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 8741fed52a..a0da739cb7 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -151,8 +151,9 @@ //! # } //! ``` -use std::ops::Range; +use std::ops::{Add, Range}; +use async_trait::async_trait; use incrementalmerkletree::frontier::Frontier; use subtle::ConditionallySelectable; use zcash_primitives::{ @@ -161,7 +162,7 @@ use zcash_primitives::{ }; use crate::{ - data_api::{NullifierQuery, WalletWrite}, + data_api::{scanning::ScanRange, NullifierQuery, WalletWrite}, proto::compact_formats::CompactBlock, scanning::{scan_block_with_runners, BatchRunners, Nullifiers, ScanningKeys}, }; @@ -169,7 +170,7 @@ use crate::{ pub mod error; use error::Error; -use super::WalletRead; +use super::{scanning::ScanPriority, WalletRead}; /// A struct containing metadata about a subtree root of the note commitment tree. /// @@ -224,6 +225,205 @@ pub trait BlockSource { F: FnMut(CompactBlock) -> Result<(), error::Error>; } +/// `BlockCache` is a trait that extends `BlockSource` and defines methods for managing +/// a cache of compact blocks. +/// +/// # Examples +/// +/// ``` +/// use async_trait::async_trait; +/// use std::sync::{Arc, Mutex}; +/// use zcash_client_backend::data_api::{ +/// chain::{error, BlockCache, BlockSource}, +/// scanning::{ScanPriority, ScanRange}, +/// }; +/// use zcash_client_backend::proto::compact_formats::CompactBlock; +/// use zcash_primitives::consensus::BlockHeight; +/// +/// struct ExampleBlockCache { +/// cached_blocks: Arc>>, +/// } +/// +/// # impl BlockSource for ExampleBlockCache { +/// # type Error = (); +/// # +/// # fn with_blocks( +/// # &self, +/// # _from_height: Option, +/// # _limit: Option, +/// # _with_block: F, +/// # ) -> Result<(), error::Error> +/// # where +/// # F: FnMut(CompactBlock) -> Result<(), error::Error>, +/// # { +/// # Ok(()) +/// # } +/// # } +/// # +/// #[async_trait] +/// impl BlockCache for ExampleBlockCache { +/// fn get_tip_height(&self, range: Option<&ScanRange>) -> Result, Self::Error> { +/// let cached_blocks = self.cached_blocks.lock().unwrap(); +/// let blocks: Vec<&CompactBlock> = match range { +/// Some(range) => cached_blocks +/// .iter() +/// .filter(|&block| { +/// let block_height = BlockHeight::from_u32(block.height as u32); +/// range.block_range().contains(&block_height) +/// }) +/// .collect(), +/// None => cached_blocks.iter().collect(), +/// }; +/// let highest_block = blocks.iter().max_by_key(|&&block| block.height); +/// Ok(highest_block.map(|&block| BlockHeight::from_u32(block.height as u32))) +/// } +/// +/// async fn read(&self, range: &ScanRange) -> Result, Self::Error> { +/// Ok(self +/// .cached_blocks +/// .lock() +/// .unwrap() +/// .iter() +/// .filter(|block| { +/// let block_height = BlockHeight::from_u32(block.height as u32); +/// range.block_range().contains(&block_height) +/// }) +/// .cloned() +/// .collect()) +/// } +/// +/// async fn insert(&self, mut compact_blocks: Vec) -> Result<(), Self::Error> { +/// self.cached_blocks +/// .lock() +/// .unwrap() +/// .append(&mut compact_blocks); +/// Ok(()) +/// } +/// +/// async fn delete(&self, range: &ScanRange) -> Result<(), Self::Error> { +/// self.cached_blocks +/// .lock() +/// .unwrap() +/// .retain(|block| !range.block_range().contains(&BlockHeight::from_u32(block.height as u32))); +/// Ok(()) +/// } +/// } +/// +/// // Example usage +/// let rt = tokio::runtime::Runtime::new().unwrap(); +/// let mut block_cache = ExampleBlockCache { +/// cached_blocks: Arc::new(Mutex::new(Vec::new())), +/// }; +/// let range = ScanRange::from_parts( +/// BlockHeight::from_u32(1)..BlockHeight::from_u32(3), +/// ScanPriority::Historic, +/// ); +/// # let extsk = sapling::zip32::ExtendedSpendingKey::master(&[]); +/// # let dfvk = extsk.to_diversifiable_full_viewing_key(); +/// # let compact_block1 = zcash_client_backend::scanning::testing::fake_compact_block( +/// # 1u32.into(), +/// # zcash_primitives::block::BlockHash([0; 32]), +/// # sapling::Nullifier([0; 32]), +/// # &dfvk, +/// # zcash_primitives::transaction::components::amount::NonNegativeAmount::const_from_u64(5), +/// # false, +/// # None, +/// # ); +/// # let compact_block2 = zcash_client_backend::scanning::testing::fake_compact_block( +/// # 2u32.into(), +/// # zcash_primitives::block::BlockHash([0; 32]), +/// # sapling::Nullifier([0; 32]), +/// # &dfvk, +/// # zcash_primitives::transaction::components::amount::NonNegativeAmount::const_from_u64(5), +/// # false, +/// # None, +/// # ); +/// let compact_blocks = vec![compact_block1, compact_block2]; +/// +/// // Insert blocks into the block cache +/// rt.block_on(async { +/// block_cache.insert(compact_blocks.clone()).await.unwrap(); +/// }); +/// assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 2); +/// +/// // Find highest block in the block cache +/// let get_tip_height = block_cache.get_tip_height(None).unwrap(); +/// assert_eq!(get_tip_height, Some(BlockHeight::from_u32(2))); +/// +/// // Read from the block cache +/// rt.block_on(async { +/// let blocks_from_cache = block_cache.read(&range).await.unwrap(); +/// assert_eq!(blocks_from_cache, compact_blocks); +/// }); +/// +/// // Truncate the block cache +/// rt.block_on(async { +/// block_cache.truncate(BlockHeight::from_u32(1)).await.unwrap(); +/// }); +/// assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 1); +/// assert_eq!( +/// block_cache.get_tip_height(None).unwrap(), +/// Some(BlockHeight::from_u32(1)) +/// ); +/// +/// // Delete blocks from the block cache +/// rt.block_on(async { +/// block_cache.delete(&range).await.unwrap(); +/// }); +/// assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 0); +/// assert_eq!(block_cache.get_tip_height(None).unwrap(), None); +/// ``` +#[async_trait] +pub trait BlockCache: BlockSource + Send + Sync +where + Self::Error: Send, +{ + /// Finds the height of the highest block known to the block cache within a specified range. + /// + /// If `range` is `None`, returns the tip of the entire cache. + /// If no blocks are found in the cache, returns Ok(`None`). + fn get_tip_height(&self, range: Option<&ScanRange>) + -> Result, Self::Error>; + + /// Retrieves contiguous compact blocks specified by the given `range` from the block cache. + /// + /// Short reads are allowed, meaning that this method may return fewer blocks than requested + /// provided that all returned blocks are contiguous and start from `range.block_range().start`. + /// + /// # Errors + /// + /// This method should return an error if contiguous blocks cannot be read from the cache, + /// indicating there are blocks missing. + async fn read(&self, range: &ScanRange) -> Result, Self::Error>; + + /// Inserts a vec of compact blocks into the block cache. + /// + /// This method permits insertion of non-contiguous compact blocks. + async fn insert(&self, compact_blocks: Vec) -> Result<(), Self::Error>; + + /// Removes all cached blocks above a specified block height. + async fn truncate(&self, block_height: BlockHeight) -> Result<(), Self::Error> { + if let Some(latest) = self.get_tip_height(None)? { + self.delete(&ScanRange::from_parts( + Range { + start: block_height.add(1), + end: latest.add(1), + }, + ScanPriority::Ignored, + )) + .await?; + } + Ok(()) + } + + /// Deletes a range of compact blocks from the block cache. + /// + /// # Errors + /// + /// In the case of an error, some blocks requested for deletion may remain in the block cache. + async fn delete(&self, range: &ScanRange) -> Result<(), Self::Error>; +} + /// Metadata about modifications to the wallet state made in the course of scanning a set of /// blocks. #[derive(Clone, Debug)] diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index 78b5852839..567528eb3c 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -1146,16 +1146,12 @@ fn find_received< (shielded_outputs, note_commitments) } -#[cfg(test)] -mod tests { - - use std::convert::Infallible; - +#[cfg(any(test, feature = "test-dependencies"))] +pub mod testing { use group::{ ff::{Field, PrimeField}, GroupEncoding, }; - use incrementalmerkletree::{Position, Retention}; use rand_core::{OsRng, RngCore}; use sapling::{ constants::SPENDING_KEY_GENERATOR, @@ -1165,26 +1161,18 @@ mod tests { zip32::DiversifiableFullViewingKey, Nullifier, }; - use zcash_keys::keys::UnifiedSpendingKey; use zcash_note_encryption::{Domain, COMPACT_NOTE_SIZE}; use zcash_primitives::{ block::BlockHash, consensus::{BlockHeight, Network}, memo::MemoBytes, transaction::components::{amount::NonNegativeAmount, sapling::zip212_enforcement}, - zip32::AccountId, }; - use crate::{ - data_api::BlockMetadata, - proto::compact_formats::{ - self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, - }, - scanning::{BatchRunners, ScanningKeys}, + use crate::proto::compact_formats::{ + self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }; - use super::{scan_block, scan_block_with_runners, Nullifiers}; - fn random_compact_tx(mut rng: impl RngCore) -> CompactTx { let fake_nf = { let mut nf = vec![0; 32]; @@ -1223,7 +1211,7 @@ mod tests { /// /// Set `initial_tree_sizes` to `None` to simulate a `CompactBlock` retrieved /// from a `lightwalletd` that is not currently tracking note commitment tree sizes. - fn fake_compact_block( + pub fn fake_compact_block( height: BlockHeight, prev_hash: BlockHash, nf: Nullifier, @@ -1302,6 +1290,29 @@ mod tests { cb } +} + +#[cfg(test)] +mod tests { + + use std::convert::Infallible; + + use incrementalmerkletree::{Position, Retention}; + use sapling::Nullifier; + use zcash_keys::keys::UnifiedSpendingKey; + use zcash_primitives::{ + block::BlockHash, + consensus::{BlockHeight, Network}, + transaction::components::amount::NonNegativeAmount, + zip32::AccountId, + }; + + use crate::{ + data_api::BlockMetadata, + scanning::{BatchRunners, ScanningKeys}, + }; + + use super::{scan_block, scan_block_with_runners, testing::fake_compact_block, Nullifiers}; #[test] fn scan_block_with_my_tx() {