From 927084b7ab4d7d12e59c26b94dbb5844e4fc354f Mon Sep 17 00:00:00 2001 From: Chengyu Lin Date: Wed, 29 Jan 2025 13:01:35 -0500 Subject: [PATCH] [VID Upgrade] add version number to VID interfaces (#4062) * add version information for VID * fix merge conflict * advz_scheme & upgrade_lock * fix * address comments --- crates/hotshot/src/tasks/mod.rs | 15 ++-- crates/task-impls/src/da.rs | 8 +- crates/task-impls/src/response.rs | 25 ++++-- crates/task-impls/src/vid.rs | 6 +- crates/testing/src/block_builder/mod.rs | 11 ++- crates/testing/src/block_builder/random.rs | 19 ++-- crates/testing/src/block_builder/simple.rs | 8 +- crates/testing/src/helpers.rs | 37 +++++--- crates/testing/src/predicates/event.rs | 10 ++- crates/testing/src/view_generator.rs | 29 ++++-- crates/testing/tests/tests_1/block_builder.rs | 7 +- crates/testing/tests/tests_1/da_task.rs | 10 ++- .../tests/tests_1/quorum_proposal_task.rs | 88 ++++++++++++++----- .../tests_1/upgrade_task_with_proposal.rs | 20 +++-- crates/testing/tests/tests_1/vid_task.rs | 13 ++- crates/types/src/consensus.rs | 21 +++-- crates/types/src/data.rs | 56 ++++++++---- crates/types/src/data/vid_disperse.rs | 10 +-- crates/types/src/traits/block_contents.rs | 14 ++- crates/types/src/vid.rs | 3 +- crates/types/src/vid/advz.rs | 0 crates/types/src/vid/avidm.rs | 0 22 files changed, 285 insertions(+), 125 deletions(-) create mode 100644 crates/types/src/vid/advz.rs create mode 100644 crates/types/src/vid/avidm.rs diff --git a/crates/hotshot/src/tasks/mod.rs b/crates/hotshot/src/tasks/mod.rs index 00ce5efb2c..767fe38ec2 100644 --- a/crates/hotshot/src/tasks/mod.rs +++ b/crates/hotshot/src/tasks/mod.rs @@ -80,18 +80,21 @@ pub async fn add_request_network_task< pub fn add_response_task, V: Versions>( handle: &mut SystemContextHandle, ) { - let state = NetworkResponseState::::new( + let state = NetworkResponseState::::new( handle.hotshot.consensus(), Arc::clone(&handle.memberships), handle.public_key().clone(), handle.private_key().clone(), handle.hotshot.id, + handle.hotshot.upgrade_lock.clone(), ); - handle.network_registry.register(run_response_task::( - state, - handle.internal_event_stream.1.activate_cloned(), - handle.internal_event_stream.0.clone(), - )); + handle + .network_registry + .register(run_response_task::( + state, + handle.internal_event_stream.1.activate_cloned(), + handle.internal_event_stream.0.clone(), + )); } /// Add a task which updates our queue length metric at a set interval diff --git a/crates/task-impls/src/da.rs b/crates/task-impls/src/da.rs index f26a201f15..8c759605a5 100644 --- a/crates/task-impls/src/da.rs +++ b/crates/task-impls/src/da.rs @@ -179,9 +179,11 @@ impl, V: Versions> DaTaskState(&txns, num_nodes, version)).await; let payload_commitment = payload_commitment.unwrap(); self.storage @@ -233,12 +235,14 @@ impl, V: Versions> DaTaskState( OuterConsensus::new(Arc::clone(&consensus.inner_consensus)), view_number, membership, &pk, + &upgrade_lock, ) .await; if let Some(Some(vid_share)) = consensus diff --git a/crates/task-impls/src/response.rs b/crates/task-impls/src/response.rs index 17116a1a2a..35be5523f4 100644 --- a/crates/task-impls/src/response.rs +++ b/crates/task-impls/src/response.rs @@ -12,9 +12,11 @@ use committable::Committable; use hotshot_types::{ consensus::{Consensus, LockedConsensusState, OuterConsensus}, data::VidDisperseShare, - message::Proposal, + message::{Proposal, UpgradeLock}, traits::{ - election::Membership, network::DataRequest, node_implementation::NodeType, + election::Membership, + network::DataRequest, + node_implementation::{NodeType, Versions}, signature_key::SignatureKey, }, }; @@ -29,7 +31,7 @@ const TXNS_TIMEOUT: Duration = Duration::from_millis(100); /// Task state for the Network Request Task. The task is responsible for handling /// requests sent to this node by the network. It will validate the sender, /// parse the request, and try to find the data request in the consensus stores. -pub struct NetworkResponseState { +pub struct NetworkResponseState { /// Locked consensus state consensus: LockedConsensusState, @@ -44,9 +46,12 @@ pub struct NetworkResponseState { /// The node's id id: u64, + + /// Lock for a decided upgrade + upgrade_lock: UpgradeLock, } -impl NetworkResponseState { +impl NetworkResponseState { /// Create the network request state with the info it needs pub fn new( consensus: LockedConsensusState, @@ -54,6 +59,7 @@ impl NetworkResponseState { pub_key: TYPES::SignatureKey, private_key: ::PrivateKey, id: u64, + upgrade_lock: UpgradeLock, ) -> Self { Self { consensus, @@ -61,6 +67,7 @@ impl NetworkResponseState { pub_key, private_key, id, + upgrade_lock, } } @@ -155,22 +162,24 @@ impl NetworkResponseState { drop(consensus_reader); - if Consensus::calculate_and_update_vid( + if Consensus::calculate_and_update_vid::( OuterConsensus::new(Arc::clone(&self.consensus)), view, Arc::clone(&self.membership), &self.private_key, + &self.upgrade_lock, ) .await .is_none() { // Sleep in hope we receive txns in the meantime sleep(TXNS_TIMEOUT).await; - Consensus::calculate_and_update_vid( + Consensus::calculate_and_update_vid::( OuterConsensus::new(Arc::clone(&self.consensus)), view, Arc::clone(&self.membership), &self.private_key, + &self.upgrade_lock, ) .await?; } @@ -208,8 +217,8 @@ fn valid_signature( /// Spawn the network response task to handle incoming request for data /// from other nodes. It will shutdown when it gets `HotshotEvent::Shutdown` /// on the `event_stream` arg. -pub fn run_response_task( - task_state: NetworkResponseState, +pub fn run_response_task( + task_state: NetworkResponseState, event_stream: Receiver>>, sender: Sender>>, ) -> JoinHandle<()> { diff --git a/crates/task-impls/src/vid.rs b/crates/task-impls/src/vid.rs index 41af76bae1..9968ea21dd 100644 --- a/crates/task-impls/src/vid.rs +++ b/crates/task-impls/src/vid.rs @@ -99,12 +99,13 @@ impl, V: Versions> VidTaskState( &payload, &Arc::clone(&self.membership), *view_number, epoch, epoch, + &self.upgrade_lock, ) .await .ok()?; @@ -203,12 +204,13 @@ impl, V: Versions> VidTaskState( payload.as_ref(), &Arc::clone(&self.membership), proposal_view_number, target_epoch, sender_epoch, + &self.upgrade_lock, ) .await .ok()?; diff --git a/crates/testing/src/block_builder/mod.rs b/crates/testing/src/block_builder/mod.rs index 891fbbef72..10e1cf6aed 100644 --- a/crates/testing/src/block_builder/mod.rs +++ b/crates/testing/src/block_builder/mod.rs @@ -22,13 +22,14 @@ use hotshot_builder_api::{ use hotshot_types::{ constants::{LEGACY_BUILDER_MODULE, MARKETPLACE_BUILDER_MODULE}, traits::{ - block_contents::EncodeBytes, node_implementation::NodeType, + block_contents::EncodeBytes, + node_implementation::{NodeType, Versions}, signature_key::BuilderSignatureKey, }, }; use tide_disco::{method::ReadState, App, Url}; use tokio::spawn; -use vbs::version::StaticVersionType; +use vbs::version::{StaticVersionType, Version}; use crate::test_builder::BuilderChange; @@ -166,11 +167,12 @@ pub fn run_builder_source_0_1( } /// Helper function to construct all builder data structures from a list of transactions -async fn build_block( +async fn build_block( transactions: Vec, num_storage_nodes: Arc>, pub_key: TYPES::BuilderSignatureKey, priv_key: ::BuilderPrivateKey, + version: Version, ) -> BlockEntry where ::InstanceState: Default, @@ -185,9 +187,10 @@ where let commitment = block_payload.builder_commitment(&metadata); - let vid_commitment = hotshot_types::traits::block_contents::vid_commitment( + let vid_commitment = hotshot_types::traits::block_contents::vid_commitment::( &block_payload.encode(), *num_storage_nodes.read_arc().await, + version, ); // Get block size from the encoded payload diff --git a/crates/testing/src/block_builder/random.rs b/crates/testing/src/block_builder/random.rs index b5c1fb93f7..12232a5f59 100644 --- a/crates/testing/src/block_builder/random.rs +++ b/crates/testing/src/block_builder/random.rs @@ -25,10 +25,13 @@ use hotshot_builder_api::v0_1::{ builder::BuildError, data_source::BuilderDataSource, }; -use hotshot_example_types::block_types::TestTransaction; +use hotshot_example_types::{block_types::TestTransaction, node_types::TestVersions}; use hotshot_types::{ network::RandomBuilderConfig, - traits::{node_implementation::NodeType, signature_key::BuilderSignatureKey}, + traits::{ + node_implementation::{NodeType, Versions}, + signature_key::BuilderSignatureKey, + }, utils::BuilderCommitment, vid::VidCommitment, }; @@ -36,6 +39,7 @@ use lru::LruCache; use rand::{rngs::SmallRng, Rng, RngCore, SeedableRng}; use tide_disco::{method::ReadState, Url}; use tokio::{spawn, time::sleep}; +use vbs::version::StaticVersionType; use super::{ build_block, run_builder_source_0_1, BlockEntry, BuilderTask, TestBuilderImplementation, @@ -110,7 +114,7 @@ pub struct RandomBuilderTask> { } impl> RandomBuilderTask { - async fn build_blocks( + async fn build_blocks( options: RandomBuilderConfig, num_nodes: Arc>, pub_key: ::BuilderSignatureKey, @@ -136,11 +140,14 @@ impl> RandomBuilderTask { }) .collect(); - let block = build_block( + // Let new VID scheme ship with Epochs upgrade. + let version = ::Epochs::VERSION; + let block = build_block::( transactions, num_nodes.clone(), pub_key.clone(), priv_key.clone(), + version, ) .await; @@ -171,7 +178,7 @@ where mut self: Box, mut stream: Box> + std::marker::Unpin + Send + 'static>, ) { - let mut task = Some(spawn(Self::build_blocks( + let mut task = Some(spawn(Self::build_blocks::( self.config.clone(), self.num_nodes.clone(), self.pub_key.clone(), @@ -191,7 +198,7 @@ where match change { BuilderChange::Up => { if task.is_none() { - task = Some(spawn(Self::build_blocks( + task = Some(spawn(Self::build_blocks::( self.config.clone(), self.num_nodes.clone(), self.pub_key.clone(), diff --git a/crates/testing/src/block_builder/simple.rs b/crates/testing/src/block_builder/simple.rs index e0af714f9a..6823b90236 100644 --- a/crates/testing/src/block_builder/simple.rs +++ b/crates/testing/src/block_builder/simple.rs @@ -31,12 +31,13 @@ use hotshot_builder_api::{ }, v0_99, }; +use hotshot_example_types::node_types::TestVersions; use hotshot_types::{ bundle::Bundle, constants::{LEGACY_BUILDER_MODULE, MARKETPLACE_BUILDER_MODULE}, traits::{ block_contents::{BlockHeader, BuilderFee}, - node_implementation::NodeType, + node_implementation::{NodeType, Versions}, signature_key::BuilderSignatureKey, }, utils::BuilderCommitment, @@ -246,11 +247,14 @@ where return Ok(vec![]); } - let block_entry = build_block( + // Let new VID scheme ships with Epochs upgrade + let version = ::Epochs::VERSION; + let block_entry = build_block::( transactions, self.num_nodes.clone(), self.pub_key.clone(), self.priv_key.clone(), + version, ) .await; diff --git a/crates/testing/src/helpers.rs b/crates/testing/src/helpers.rs index c476ccea44..1bbb099c93 100644 --- a/crates/testing/src/helpers.rs +++ b/crates/testing/src/helpers.rs @@ -36,13 +36,14 @@ use hotshot_types::{ node_implementation::{NodeType, Versions}, }, utils::{option_epoch_from_block_number, View, ViewInner}, - vid::{vid_scheme, VidCommitment, VidProposal, VidSchemeType}, + vid::{advz_scheme, VidCommitment, VidProposal, VidSchemeType}, vote::{Certificate, HasViewNumber, Vote}, ValidatorConfig, }; use jf_vid::VidScheme; use primitive_types::U256; use serde::Serialize; +use vbs::version::Version; use crate::{test_builder::TestDescription, test_launcher::TestLauncher}; @@ -270,67 +271,78 @@ pub fn key_pair_for_id( /// initialize VID /// # Panics /// if unable to create a [`VidSchemeType`] +/// TODO(Chengyu): use this version information #[must_use] -pub async fn vid_scheme_from_view_number( +pub async fn vid_scheme_from_view_number( membership: &Arc>, view_number: TYPES::View, epoch_number: Option, + _version: Version, ) -> VidSchemeType { let num_storage_nodes = membership .read() .await .committee_members(view_number, epoch_number) .len(); - vid_scheme(num_storage_nodes) + advz_scheme(num_storage_nodes) } -pub async fn vid_payload_commitment( +pub async fn vid_payload_commitment( membership: &Arc::Membership>>, view_number: TYPES::View, epoch_number: Option, transactions: Vec, + version: Version, ) -> VidCommitment { - let mut vid = vid_scheme_from_view_number::(membership, view_number, epoch_number).await; + let mut vid = + vid_scheme_from_view_number::(membership, view_number, epoch_number, version) + .await; let encoded_transactions = TestTransaction::encode(&transactions); let vid_disperse = vid.disperse(&encoded_transactions).unwrap(); vid_disperse.commit } -pub async fn da_payload_commitment( +pub async fn da_payload_commitment( membership: &Arc::Membership>>, transactions: Vec, epoch_number: Option, + version: Version, ) -> VidCommitment { let encoded_transactions = TestTransaction::encode(&transactions); - vid_commitment( + vid_commitment::( &encoded_transactions, membership.read().await.total_nodes(epoch_number), + version, ) } -pub async fn build_payload_commitment( +pub async fn build_payload_commitment( membership: &Arc::Membership>>, view: TYPES::View, epoch: Option, + version: Version, ) -> ::Commit { // Make some empty encoded transactions, we just care about having a commitment handy for the // later calls. We need the VID commitment to be able to propose later. - let mut vid = vid_scheme_from_view_number::(membership, view, epoch).await; + let mut vid = vid_scheme_from_view_number::(membership, view, epoch, version).await; let encoded_transactions = Vec::new(); vid.commit_only(&encoded_transactions).unwrap() } /// TODO: -pub async fn build_vid_proposal( +pub async fn build_vid_proposal( membership: &Arc::Membership>>, view_number: TYPES::View, epoch_number: Option, transactions: Vec, private_key: &::PrivateKey, + version: Version, ) -> VidProposal { - let mut vid = vid_scheme_from_view_number::(membership, view_number, epoch_number).await; + let mut vid = + vid_scheme_from_view_number::(membership, view_number, epoch_number, version) + .await; let encoded_transactions = TestTransaction::encode(&transactions); let vid_disperse = VidDisperse::from_membership( @@ -377,9 +389,10 @@ pub async fn build_da_certificate( ) -> DaCertificate2 { let encoded_transactions = TestTransaction::encode(&transactions); - let da_payload_commitment = vid_commitment( + let da_payload_commitment = vid_commitment::( &encoded_transactions, membership.read().await.total_nodes(epoch_number), + upgrade_lock.version_infallible(view_number).await, ); let da_data = DaData2 { diff --git a/crates/testing/src/predicates/event.rs b/crates/testing/src/predicates/event.rs index ebd33399fb..63727b8b21 100644 --- a/crates/testing/src/predicates/event.rs +++ b/crates/testing/src/predicates/event.rs @@ -11,7 +11,10 @@ use async_trait::async_trait; use hotshot_task_impls::events::{HotShotEvent, HotShotEvent::*}; use hotshot_types::{ data::null_block, - traits::{block_contents::BlockHeader, node_implementation::NodeType}, + traits::{ + block_contents::BlockHeader, + node_implementation::{NodeType, Versions}, + }, }; use crate::predicates::{Predicate, PredicateResult}; @@ -202,18 +205,19 @@ where Box::new(EventPredicate { check, info }) } -pub fn quorum_proposal_send_with_null_block( +pub fn quorum_proposal_send_with_null_block( num_storage_nodes: usize, ) -> Box> where TYPES: NodeType, + V: Versions, { let info = "QuorumProposalSend with null block payload".to_string(); let check: EventCallback = Arc::new(move |e: Arc>| match e.as_ref() { QuorumProposalSend(proposal, _) => { Some(proposal.data.block_header().payload_commitment()) - == null_block::commitment(num_storage_nodes) + == null_block::commitment::(num_storage_nodes) } _ => false, }); diff --git a/crates/testing/src/view_generator.rs b/crates/testing/src/view_generator.rs index c5d77232db..707e8593f2 100644 --- a/crates/testing/src/view_generator.rs +++ b/crates/testing/src/view_generator.rs @@ -100,16 +100,23 @@ impl TestView { let leader_public_key = public_key; - let payload_commitment = - da_payload_commitment::(membership, transactions.clone(), genesis_epoch) - .await; + let genesis_version = upgrade_lock.version_infallible(genesis_view).await; - let (vid_disperse, vid_proposal) = build_vid_proposal( + let payload_commitment = da_payload_commitment::( + membership, + transactions.clone(), + genesis_epoch, + genesis_version, + ) + .await; + + let (vid_disperse, vid_proposal) = build_vid_proposal::( membership, genesis_view, genesis_epoch, transactions.clone(), &private_key, + genesis_version, ) .await; @@ -248,16 +255,22 @@ impl TestView { &metadata, ); - let payload_commitment = - da_payload_commitment::(membership, transactions.clone(), self.epoch_number) - .await; + let version = self.upgrade_lock.version_infallible(next_view).await; + let payload_commitment = da_payload_commitment::( + membership, + transactions.clone(), + self.epoch_number, + version, + ) + .await; - let (vid_disperse, vid_proposal) = build_vid_proposal( + let (vid_disperse, vid_proposal) = build_vid_proposal::( membership, next_view, self.epoch_number, transactions.clone(), &private_key, + version, ) .await; diff --git a/crates/testing/tests/tests_1/block_builder.rs b/crates/testing/tests/tests_1/block_builder.rs index fc29b1c01d..7258c3ff00 100644 --- a/crates/testing/tests/tests_1/block_builder.rs +++ b/crates/testing/tests/tests_1/block_builder.rs @@ -33,6 +33,9 @@ use vbs::version::StaticVersion; #[tokio::test(flavor = "multi_thread")] #[ignore] async fn test_random_block_builder() { + use hotshot_example_types::node_types::TestVersions; + use vbs::version::Version; + let port = portpicker::pick_unused_port().expect("No free ports"); let api_url = Url::parse(&format!("http://localhost:{port}")).expect("Valid URL"); let task: Box> = RandomBuilderImplementation::start( @@ -58,11 +61,13 @@ async fn test_random_block_builder() { .expect("Failed to create dummy signature"); let dummy_view_number = 0u64; + let version = Version { major: 0, minor: 0 }; + let mut blocks = loop { // Test getting blocks let blocks = client .available_blocks( - vid_commitment(&[], 1), + vid_commitment::(&[], 1, version), dummy_view_number, pub_key, &signature, diff --git a/crates/testing/tests/tests_1/da_task.rs b/crates/testing/tests/tests_1/da_task.rs index 5422eb2a99..5006cab51d 100644 --- a/crates/testing/tests/tests_1/da_task.rs +++ b/crates/testing/tests/tests_1/da_task.rs @@ -29,7 +29,7 @@ use hotshot_types::{ node_implementation::{ConsensusTime, Versions}, }, }; -use vbs::version::StaticVersionType; +use vbs::version::{StaticVersionType, Version}; #[tokio::test(flavor = "multi_thread")] async fn test_da_task() { @@ -40,14 +40,16 @@ async fn test_da_task() { .0; let membership = Arc::clone(&handle.hotshot.memberships); + let default_version = Version { major: 0, minor: 0 }; // Make some empty encoded transactions, we just care about having a commitment handy for the // later calls. We need the VID commitment to be able to propose later. let transactions = vec![TestTransaction::new(vec![0])]; let encoded_transactions: Arc<[u8]> = Arc::from(TestTransaction::encode(&transactions)); - let payload_commit = hotshot_types::traits::block_contents::vid_commitment( + let payload_commit = hotshot_types::traits::block_contents::vid_commitment::( &encoded_transactions, handle.hotshot.memberships.read().await.total_nodes(None), + default_version, ); let mut generator = TestViewGenerator::::generate(membership.clone()); @@ -147,14 +149,16 @@ async fn test_da_task_storage_failure() { // Set the error flag here for the system handle. This causes it to emit an error on append. handle.storage().write().await.should_return_err = true; let membership = Arc::clone(&handle.hotshot.memberships); + let default_version = Version { major: 0, minor: 0 }; // Make some empty encoded transactions, we just care about having a commitment handy for the // later calls. We need the VID commitment to be able to propose later. let transactions = vec![TestTransaction::new(vec![0])]; let encoded_transactions: Arc<[u8]> = Arc::from(TestTransaction::encode(&transactions)); - let payload_commit = hotshot_types::traits::block_contents::vid_commitment( + let payload_commit = hotshot_types::traits::block_contents::vid_commitment::( &encoded_transactions, handle.hotshot.memberships.read().await.total_nodes(None), + default_version, ); let mut generator = TestViewGenerator::::generate(Arc::clone(&membership)); diff --git a/crates/testing/tests/tests_1/quorum_proposal_task.rs b/crates/testing/tests/tests_1/quorum_proposal_task.rs index 95c4010c03..9545f79cfc 100644 --- a/crates/testing/tests/tests_1/quorum_proposal_task.rs +++ b/crates/testing/tests/tests_1/quorum_proposal_task.rs @@ -52,11 +52,17 @@ async fn test_quorum_proposal_task_quorum_proposal_view_1() { .0; let membership = Arc::clone(&handle.hotshot.memberships); + let version = handle + .hotshot + .upgrade_lock + .version_infallible(ViewNumber::new(node_id)) + .await; - let payload_commitment = build_payload_commitment::( + let payload_commitment = build_payload_commitment::( &membership, ViewNumber::new(node_id), Some(EpochNumber::new(1)), + version, ) .await; @@ -195,14 +201,22 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { ) .unwrap(); + let upgrade_lock = &handle.hotshot.upgrade_lock; + let version_1 = upgrade_lock.version_infallible(ViewNumber::new(1)).await; + let version_2 = upgrade_lock.version_infallible(ViewNumber::new(2)).await; + let version_3 = upgrade_lock.version_infallible(ViewNumber::new(3)).await; + let version_4 = upgrade_lock.version_infallible(ViewNumber::new(4)).await; + let version_5 = upgrade_lock.version_infallible(ViewNumber::new(5)).await; + let inputs = vec![ random![ Qc2Formed(either::Left(genesis_cert.clone())), SendPayloadCommitmentAndMetadata( - build_payload_commitment::( + build_payload_commitment::( &membership, ViewNumber::new(1), - Some(EpochNumber::new(1)) + Some(EpochNumber::new(1)), + version_1, ) .await, builder_commitment.clone(), @@ -219,10 +233,11 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { QuorumProposalPreliminarilyValidated(proposals[0].clone()), Qc2Formed(either::Left(proposals[1].data.justify_qc().clone())), SendPayloadCommitmentAndMetadata( - build_payload_commitment::( + build_payload_commitment::( &membership, ViewNumber::new(2), - Some(EpochNumber::new(1)) + Some(EpochNumber::new(1)), + version_2, ) .await, builder_commitment.clone(), @@ -237,10 +252,11 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { QuorumProposalPreliminarilyValidated(proposals[1].clone()), Qc2Formed(either::Left(proposals[2].data.justify_qc().clone())), SendPayloadCommitmentAndMetadata( - build_payload_commitment::( + build_payload_commitment::( &membership, ViewNumber::new(3), - Some(EpochNumber::new(1)) + Some(EpochNumber::new(1)), + version_3, ) .await, builder_commitment.clone(), @@ -255,10 +271,11 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { QuorumProposalPreliminarilyValidated(proposals[2].clone()), Qc2Formed(either::Left(proposals[3].data.justify_qc().clone())), SendPayloadCommitmentAndMetadata( - build_payload_commitment::( + build_payload_commitment::( &membership, ViewNumber::new(4), - Some(EpochNumber::new(1)) + Some(EpochNumber::new(1)), + version_4, ) .await, builder_commitment.clone(), @@ -273,10 +290,11 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { QuorumProposalPreliminarilyValidated(proposals[3].clone()), Qc2Formed(either::Left(proposals[4].data.justify_qc().clone())), SendPayloadCommitmentAndMetadata( - build_payload_commitment::( + build_payload_commitment::( &membership, ViewNumber::new(5), - Some(EpochNumber::new(1)) + Some(EpochNumber::new(1)), + version_5, ) .await, builder_commitment, @@ -321,11 +339,17 @@ async fn test_quorum_proposal_task_qc_timeout() { .await .0; let membership = Arc::clone(&handle.hotshot.memberships); + let version = handle + .hotshot + .upgrade_lock + .version_infallible(ViewNumber::new(node_id)) + .await; - let payload_commitment = build_payload_commitment::( + let payload_commitment = build_payload_commitment::( &membership, ViewNumber::new(node_id), Some(EpochNumber::new(1)), + version, ) .await; let builder_commitment = BuilderCommitment::from_raw_digest(sha2::Sha256::new().finalize()); @@ -414,11 +438,17 @@ async fn test_quorum_proposal_task_view_sync() { .0; let membership = Arc::clone(&handle.hotshot.memberships); + let version = handle + .hotshot + .upgrade_lock + .version_infallible(ViewNumber::new(node_id)) + .await; - let payload_commitment = build_payload_commitment::( + let payload_commitment = build_payload_commitment::( &membership, ViewNumber::new(node_id), Some(EpochNumber::new(1)), + version, ) .await; let builder_commitment = BuilderCommitment::from_raw_digest(sha2::Sha256::new().finalize()); @@ -551,14 +581,22 @@ async fn test_quorum_proposal_task_liveness_check() { // updated properly. let genesis_cert = proposals[0].data.justify_qc().clone(); + let upgrade_lock = &handle.hotshot.upgrade_lock; + let version_1 = upgrade_lock.version_infallible(ViewNumber::new(1)).await; + let version_2 = upgrade_lock.version_infallible(ViewNumber::new(2)).await; + let version_3 = upgrade_lock.version_infallible(ViewNumber::new(3)).await; + let version_4 = upgrade_lock.version_infallible(ViewNumber::new(4)).await; + let version_5 = upgrade_lock.version_infallible(ViewNumber::new(5)).await; + let inputs = vec![ random![ Qc2Formed(either::Left(genesis_cert.clone())), SendPayloadCommitmentAndMetadata( - build_payload_commitment::( + build_payload_commitment::( &membership, ViewNumber::new(1), - Some(EpochNumber::new(1)) + Some(EpochNumber::new(1)), + version_1, ) .await, builder_commitment.clone(), @@ -575,10 +613,11 @@ async fn test_quorum_proposal_task_liveness_check() { QuorumProposalPreliminarilyValidated(proposals[0].clone()), Qc2Formed(either::Left(proposals[1].data.justify_qc().clone())), SendPayloadCommitmentAndMetadata( - build_payload_commitment::( + build_payload_commitment::( &membership, ViewNumber::new(2), - Some(EpochNumber::new(1)) + Some(EpochNumber::new(1)), + version_2, ) .await, builder_commitment.clone(), @@ -593,10 +632,11 @@ async fn test_quorum_proposal_task_liveness_check() { QuorumProposalPreliminarilyValidated(proposals[1].clone()), Qc2Formed(either::Left(proposals[2].data.justify_qc().clone())), SendPayloadCommitmentAndMetadata( - build_payload_commitment::( + build_payload_commitment::( &membership, ViewNumber::new(3), - Some(EpochNumber::new(1)) + Some(EpochNumber::new(1)), + version_3, ) .await, builder_commitment.clone(), @@ -611,10 +651,11 @@ async fn test_quorum_proposal_task_liveness_check() { QuorumProposalPreliminarilyValidated(proposals[2].clone()), Qc2Formed(either::Left(proposals[3].data.justify_qc().clone())), SendPayloadCommitmentAndMetadata( - build_payload_commitment::( + build_payload_commitment::( &membership, ViewNumber::new(4), - Some(EpochNumber::new(1)) + Some(EpochNumber::new(1)), + version_4, ) .await, builder_commitment.clone(), @@ -629,10 +670,11 @@ async fn test_quorum_proposal_task_liveness_check() { QuorumProposalPreliminarilyValidated(proposals[3].clone()), Qc2Formed(either::Left(proposals[4].data.justify_qc().clone())), SendPayloadCommitmentAndMetadata( - build_payload_commitment::( + build_payload_commitment::( &membership, ViewNumber::new(5), - Some(EpochNumber::new(1)) + Some(EpochNumber::new(1)), + version_5, ) .await, builder_commitment, diff --git a/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs b/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs index 6a9a9e1f11..820fb3a562 100644 --- a/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs +++ b/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs @@ -151,14 +151,20 @@ async fn test_upgrade_task_with_proposal() { let upgrade_vote_recvs: Vec<_> = upgrade_votes.into_iter().map(UpgradeVoteRecv).collect(); + let upgrade_lock = &upgrade_state.upgrade_lock; + let version_1 = upgrade_lock.version_infallible(ViewNumber::new(1)).await; + let version_2 = upgrade_lock.version_infallible(ViewNumber::new(2)).await; + let version_3 = upgrade_lock.version_infallible(ViewNumber::new(3)).await; + let inputs = vec![ random![ Qc2Formed(either::Left(genesis_cert.clone())), SendPayloadCommitmentAndMetadata( - build_payload_commitment::( + build_payload_commitment::( &membership, ViewNumber::new(1), - Some(EpochNumber::new(1)) + Some(EpochNumber::new(1)), + version_1, ) .await, builder_commitment.clone(), @@ -175,10 +181,11 @@ async fn test_upgrade_task_with_proposal() { QuorumProposalPreliminarilyValidated(proposals[0].clone()), Qc2Formed(either::Left(proposals[1].data.justify_qc().clone())), SendPayloadCommitmentAndMetadata( - build_payload_commitment::( + build_payload_commitment::( &membership, ViewNumber::new(2), - Some(EpochNumber::new(1)) + Some(EpochNumber::new(1)), + version_2, ) .await, builder_commitment.clone(), @@ -194,10 +201,11 @@ async fn test_upgrade_task_with_proposal() { QuorumProposalPreliminarilyValidated(proposals[1].clone()), Qc2Formed(either::Left(proposals[2].data.justify_qc().clone())), SendPayloadCommitmentAndMetadata( - build_payload_commitment::( + build_payload_commitment::( &membership, ViewNumber::new(3), - Some(EpochNumber::new(1)) + Some(EpochNumber::new(1)), + version_3, ) .await, builder_commitment.clone(), diff --git a/crates/testing/tests/tests_1/vid_task.rs b/crates/testing/tests/tests_1/vid_task.rs index c79acbcad4..0cb13a0f45 100644 --- a/crates/testing/tests/tests_1/vid_task.rs +++ b/crates/testing/tests/tests_1/vid_task.rs @@ -30,7 +30,7 @@ use hotshot_types::{ }, }; use jf_vid::VidScheme; -use vbs::version::StaticVersionType; +use vbs::version::{StaticVersionType, Version}; use vec1::vec1; #[tokio::test(flavor = "multi_thread")] @@ -47,8 +47,15 @@ async fn test_vid_task() { let membership = Arc::clone(&handle.hotshot.memberships); - let mut vid = - vid_scheme_from_view_number::(&membership, ViewNumber::new(0), None).await; + let default_version = Version { major: 0, minor: 0 }; + + let mut vid = vid_scheme_from_view_number::( + &membership, + ViewNumber::new(0), + None, + default_version, + ) + .await; let transactions = vec![TestTransaction::new(vec![0])]; let (payload, metadata) = >::from_transactions( diff --git a/crates/types/src/consensus.rs b/crates/types/src/consensus.rs index a2d92c6578..c6cc918444 100644 --- a/crates/types/src/consensus.rs +++ b/crates/types/src/consensus.rs @@ -25,12 +25,12 @@ use crate::{ drb::DrbSeedsAndResults, error::HotShotError, event::{HotShotAction, LeafInfo}, - message::Proposal, + message::{Proposal, UpgradeLock}, simple_certificate::{DaCertificate2, NextEpochQuorumCertificate2, QuorumCertificate2}, traits::{ block_contents::BuilderFee, metrics::{Counter, Gauge, Histogram, Metrics, NoMetrics}, - node_implementation::{ConsensusTime, NodeType}, + node_implementation::{ConsensusTime, NodeType, Versions}, signature_key::SignatureKey, BlockPayload, ValidatedState, }, @@ -952,11 +952,12 @@ impl Consensus { /// and updates `vid_shares` map with the signed `VidDisperseShare` proposals. /// Returned `Option` indicates whether the update has actually happened or not. #[instrument(skip_all, target = "Consensus", fields(view = *view))] - pub async fn calculate_and_update_vid( + pub async fn calculate_and_update_vid( consensus: OuterConsensus, view: ::View, membership: Arc>, private_key: &::PrivateKey, + upgrade_lock: &UpgradeLock, ) -> Option<()> { let payload = Arc::clone(consensus.read().await.saved_payloads().get(&view)?); let epoch = consensus @@ -967,10 +968,16 @@ impl Consensus { .view_inner .epoch()?; - let vid = - VidDisperse::calculate_vid_disperse(payload.as_ref(), &membership, view, epoch, epoch) - .await - .ok()?; + let vid = VidDisperse::calculate_vid_disperse::( + payload.as_ref(), + &membership, + view, + epoch, + epoch, + upgrade_lock, + ) + .await + .ok()?; let shares = VidDisperseShare::from_vid_disperse(vid); let mut consensus_writer = consensus.write().await; diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index 1bbf6df66c..c3148551cb 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -24,6 +24,7 @@ use rand::Rng; use serde::{Deserialize, Serialize}; use thiserror::Error; use utils::anytrace::*; +use vbs::version::Version; use vec1::Vec1; use vid_disperse::{ADVZDisperse, ADVZDisperseShare, VidDisperseShare2}; @@ -263,12 +264,13 @@ impl VidDisperse { /// # Errors /// Returns an error if the disperse or commitment calculation fails #[allow(clippy::panic)] - pub async fn calculate_vid_disperse( + pub async fn calculate_vid_disperse( payload: &TYPES::BlockPayload, membership: &Arc>, view: TYPES::View, target_epoch: Option, data_epoch: Option, + _upgrade_lock: &UpgradeLock, ) -> Result { ADVZDisperse::calculate_vid_disperse(payload, membership, view, target_epoch, data_epoch) .await @@ -893,7 +895,14 @@ impl Leaf2 { let builder_commitment = payload.builder_commitment(&metadata); let payload_bytes = payload.encode(); - let payload_commitment = vid_commitment(&payload_bytes, GENESIS_VID_NUM_STORAGE_NODES); + let genesis_view = TYPES::View::genesis(); + let upgrade_lock = UpgradeLock::::new(); + let genesis_version = upgrade_lock.version_infallible(genesis_view).await; + let payload_commitment = vid_commitment::( + &payload_bytes, + GENESIS_VID_NUM_STORAGE_NODES, + genesis_version, + ); let block_header = TYPES::BlockHeader::genesis( instance_state, @@ -910,13 +919,13 @@ impl Leaf2 { let justify_qc = QuorumCertificate2::new( null_quorum_data.clone(), null_quorum_data.commit(), - ::genesis(), + genesis_view, None, PhantomData, ); Self { - view_number: TYPES::View::genesis(), + view_number: genesis_view, justify_qc, next_epoch_justify_qc: None, parent_commitment: null_quorum_data.leaf_commit, @@ -973,13 +982,14 @@ impl Leaf2 { /// /// Fails if the payload commitment doesn't match `self.block_header.payload_commitment()` /// or if the transactions are of invalid length - pub fn fill_block_payload( + pub fn fill_block_payload( &mut self, block_payload: TYPES::BlockPayload, num_storage_nodes: usize, + version: Version, ) -> std::result::Result<(), BlockError> { let encoded_txns = block_payload.encode(); - let commitment = vid_commitment(&encoded_txns, num_storage_nodes); + let commitment = vid_commitment::(&encoded_txns, num_storage_nodes, version); if commitment != self.block_header.payload_commitment() { return Err(BlockError::InconsistentPayloadCommitment); } @@ -1168,7 +1178,7 @@ impl QuorumCertificate { let genesis_view = ::genesis(); let data = QuorumData { - leaf_commit: Leaf::genesis(validated_state, instance_state) + leaf_commit: Leaf::genesis::(validated_state, instance_state) .await .commit(&upgrade_lock) .await, @@ -1233,7 +1243,7 @@ impl Leaf { /// Panics if the genesis payload (`TYPES::BlockPayload::genesis()`) is malformed (unable to be /// interpreted as bytes). #[must_use] - pub async fn genesis( + pub async fn genesis( validated_state: &TYPES::ValidatedState, instance_state: &TYPES::InstanceState, ) -> Self { @@ -1244,7 +1254,14 @@ impl Leaf { let builder_commitment = payload.builder_commitment(&metadata); let payload_bytes = payload.encode(); - let payload_commitment = vid_commitment(&payload_bytes, GENESIS_VID_NUM_STORAGE_NODES); + let genesis_view = TYPES::View::genesis(); + let upgrade_lock = UpgradeLock::::new(); + let genesis_version = upgrade_lock.version_infallible(genesis_view).await; + let payload_commitment = vid_commitment::( + &payload_bytes, + GENESIS_VID_NUM_STORAGE_NODES, + genesis_version, + ); let block_header = TYPES::BlockHeader::genesis( instance_state, @@ -1260,13 +1277,13 @@ impl Leaf { let justify_qc = QuorumCertificate::new( null_quorum_data.clone(), null_quorum_data.commit(), - ::genesis(), + genesis_view, None, PhantomData, ); Self { - view_number: TYPES::View::genesis(), + view_number: genesis_view, justify_qc, parent_commitment: null_quorum_data.leaf_commit, upgrade_certificate: None, @@ -1312,13 +1329,14 @@ impl Leaf { /// /// Fails if the payload commitment doesn't match `self.block_header.payload_commitment()` /// or if the transactions are of invalid length - pub fn fill_block_payload( + pub fn fill_block_payload( &mut self, block_payload: TYPES::BlockPayload, num_storage_nodes: usize, + version: Version, ) -> std::result::Result<(), BlockError> { let encoded_txns = block_payload.encode(); - let commitment = vid_commitment(&encoded_txns, num_storage_nodes); + let commitment = vid_commitment::(&encoded_txns, num_storage_nodes, version); if commitment != self.block_header.payload_commitment() { return Err(BlockError::InconsistentPayloadCommitment); } @@ -1534,7 +1552,6 @@ pub mod null_block { #![allow(missing_docs)] use jf_vid::VidScheme; - use memoize::memoize; use vbs::version::StaticVersionType; use crate::{ @@ -1544,7 +1561,7 @@ pub mod null_block { signature_key::BuilderSignatureKey, BlockPayload, }, - vid::{vid_scheme, VidCommitment}, + vid::{advz_scheme, VidCommitment}, }; /// The commitment for a null block payload. @@ -1553,10 +1570,11 @@ pub mod null_block { /// and may change (albeit rarely) during execution. /// /// We memoize the result to avoid having to recalculate it. - #[memoize(SharedCache, Capacity: 10)] + // TODO(Chengyu): fix it. Empty commitment must be computed at every upgrade. + // #[memoize(SharedCache, Capacity: 10)] #[must_use] - pub fn commitment(num_storage_nodes: usize) -> Option { - let vid_result = vid_scheme(num_storage_nodes).commit_only(Vec::new()); + pub fn commitment(num_storage_nodes: usize) -> Option { + let vid_result = advz_scheme(num_storage_nodes).commit_only(Vec::new()); match vid_result { Ok(r) => Some(r), @@ -1600,7 +1618,7 @@ pub mod null_block { &priv_key, FEE_AMOUNT, &null_block_metadata, - &commitment(num_storage_nodes)?, + &commitment::(num_storage_nodes)?, ) { Ok(sig) => Some(BuilderFee { fee_amount: FEE_AMOUNT, diff --git a/crates/types/src/data/vid_disperse.rs b/crates/types/src/data/vid_disperse.rs index d416dd8632..0534805cbc 100644 --- a/crates/types/src/data/vid_disperse.rs +++ b/crates/types/src/data/vid_disperse.rs @@ -14,7 +14,7 @@ use crate::{ block_contents::EncodeBytes, election::Membership, node_implementation::NodeType, signature_key::SignatureKey, }, - vid::{vid_scheme, VidCommitment, VidCommon, VidSchemeType, VidShare}, + vid::{advz_scheme, VidCommitment, VidCommon, VidSchemeType, VidShare}, vote::HasViewNumber, }; use async_lock::RwLock; @@ -101,7 +101,7 @@ impl ADVZDisperse { let txns_clone = Arc::clone(&txns); let num_txns = txns.len(); - let vid_disperse = spawn_blocking(move || vid_scheme(num_nodes).disperse(&txns_clone)) + let vid_disperse = spawn_blocking(move || advz_scheme(num_nodes).disperse(&txns_clone)) .await .wrap() .context(error!("Join error"))? @@ -114,7 +114,7 @@ impl ADVZDisperse { let num_nodes = membership.read().await.total_nodes(data_epoch); Some( - spawn_blocking(move || vid_scheme(num_nodes).commit_only(&txns)) + spawn_blocking(move || advz_scheme(num_nodes).commit_only(&txns)) .await .wrap() .context(error!("Join error"))? @@ -247,7 +247,7 @@ impl ADVZDisperseShare { /// Verification fail #[allow(clippy::result_unit_err)] pub fn verify_share(&self, total_nodes: usize) -> std::result::Result<(), ()> { - vid_scheme(total_nodes) + advz_scheme(total_nodes) .verify_share(&self.share, &self.common, &self.payload_commitment) .unwrap_or(Err(())) } @@ -376,7 +376,7 @@ impl VidDisperseShare2 { /// # Errors #[allow(clippy::result_unit_err)] pub fn verify_share(&self, total_nodes: usize) -> std::result::Result<(), ()> { - vid_scheme(total_nodes) + advz_scheme(total_nodes) .verify_share(&self.share, &self.common, &self.payload_commitment) .unwrap_or(Err(())) } diff --git a/crates/types/src/traits/block_contents.rs b/crates/types/src/traits/block_contents.rs index 2c312831b7..f9f188ce25 100644 --- a/crates/types/src/traits/block_contents.rs +++ b/crates/types/src/traits/block_contents.rs @@ -26,9 +26,13 @@ use vbs::version::Version; use super::signature_key::BuilderSignatureKey; use crate::{ data::Leaf2, - traits::{node_implementation::NodeType, states::InstanceState, ValidatedState}, + traits::{ + node_implementation::{NodeType, Versions}, + states::InstanceState, + ValidatedState, + }, utils::BuilderCommitment, - vid::{vid_scheme, VidCommitment, VidCommon, VidSchemeType}, + vid::{advz_scheme, VidCommitment, VidCommon, VidSchemeType}, }; /// Trait for structures that need to be unambiguously encoded as bytes. @@ -141,16 +145,18 @@ pub trait TestableBlock: BlockPayload + Debug { /// Compute the VID payload commitment. /// TODO(Gus) delete this function? +/// TODO(Chengyu): use the version information /// # Panics /// If the VID computation fails. #[must_use] #[allow(clippy::panic)] -pub fn vid_commitment( +pub fn vid_commitment( encoded_transactions: &[u8], num_storage_nodes: usize, + _version: Version, ) -> ::Commit { let encoded_tx_len = encoded_transactions.len(); - vid_scheme(num_storage_nodes).commit_only(encoded_transactions).unwrap_or_else(|err| panic!("VidScheme::commit_only failure:(num_storage_nodes,payload_byte_len)=({num_storage_nodes},{encoded_tx_len}) error: {err}")) + advz_scheme(num_storage_nodes).commit_only(encoded_transactions).unwrap_or_else(|err| panic!("VidScheme::commit_only failure:(num_storage_nodes,payload_byte_len)=({num_storage_nodes},{encoded_tx_len}) error: {err}")) } /// The number of storage nodes to use when computing the genesis VID commitment. diff --git a/crates/types/src/vid.rs b/crates/types/src/vid.rs index c1a185ab62..4d5dd550c7 100644 --- a/crates/types/src/vid.rs +++ b/crates/types/src/vid.rs @@ -58,9 +58,10 @@ use crate::{ /// /// # Panics /// When the construction fails for the underlying VID scheme. +// TODO(Chengyu): move all things below to advz submodule. #[must_use] #[memoize::memoize(SharedCache, Capacity: 10)] -pub fn vid_scheme(num_storage_nodes: usize) -> VidSchemeType { +pub fn advz_scheme(num_storage_nodes: usize) -> VidSchemeType { // recovery_threshold is currently num_storage_nodes rounded down to a power of two // TODO recovery_threshold should be a function of the desired erasure code rate // https://github.com/EspressoSystems/HotShot/issues/2152 diff --git a/crates/types/src/vid/advz.rs b/crates/types/src/vid/advz.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/types/src/vid/avidm.rs b/crates/types/src/vid/avidm.rs new file mode 100644 index 0000000000..e69de29bb2