Merge branch 'main' into bump/0.5.83

.config/nextest.toml

@@ -3,3 +3,9 @@ retries = 2
 slow-timeout = { period = "1m", terminate-after = 3 }
 final-status-level = "flaky"
 threads-required = "num-test-threads"
+
+[profile.local]
+retries = 0
+slow-timeout = { period = "1m", terminate-after = 3 }
+final-status-level = "slow"
+threads-required = "num-test-threads"

.github/workflows/build-and-test.yml

@@ -24,6 +24,7 @@ jobs:
           - test-ci-3
           - test-ci-4
           - test-ci-5
+          - test-ci-6
           - test-ci-rest
       fail-fast: false
     runs-on: ubuntu-latest

crates/example-types/src/node_types.rs

@@ -295,7 +295,23 @@ impl Versions for EpochsTestVersions {
         0, 0,
     ];
 
-    type Marketplace = StaticVersion<0, 3>;
+    type Marketplace = StaticVersion<0, 5>;
+
+    type Epochs = StaticVersion<0, 4>;
+}
+
+#[derive(Clone, Debug, Copy)]
+pub struct EpochUpgradeTestVersions {}
+
+impl Versions for EpochUpgradeTestVersions {
+    type Base = StaticVersion<0, 3>;
+    type Upgrade = StaticVersion<0, 4>;
+    const UPGRADE_HASH: [u8; 32] = [
+        1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,
+        0, 0,
+    ];
+
+    type Marketplace = StaticVersion<0, 5>;
 
     type Epochs = StaticVersion<0, 4>;
 }

crates/hotshot/src/lib.rs

@@ -61,7 +61,6 @@ use hotshot_types::{
         signature_key::SignatureKey,
         states::ValidatedState,
         storage::Storage,
-        EncodeBytes,
     },
     utils::epoch_from_block_number,
     HotShotConfig,
@@ -321,9 +320,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
             saved_leaves.insert(leaf.commit(), leaf.clone());
         }
         if let Some(payload) = anchored_leaf.block_payload() {
-            let encoded_txns = payload.encode();
-
-            saved_payloads.insert(anchored_leaf.view_number(), Arc::clone(&encoded_txns));
+            saved_payloads.insert(anchored_leaf.view_number(), Arc::new(payload));
         }
 
         let anchored_epoch = if config.epoch_height == 0 {

crates/task-impls/src/da.rs

@@ -24,6 +24,7 @@ use hotshot_types::{
         node_implementation::{NodeImplementation, NodeType, Versions},
         signature_key::SignatureKey,
         storage::Storage,
+        BlockPayload, EncodeBytes,
     },
     utils::EpochTransitionIndicator,
     vote::HasViewNumber,
@@ -108,7 +109,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> DaTaskState<TYP
                 );
 
                 if let Some(payload) = self.consensus.read().await.saved_payloads().get(&view) {
-                    ensure!(*payload == proposal.data.encoded_transactions, error!(
+                    ensure!(payload.encode() == proposal.data.encoded_transactions, error!(
                       "Received DA proposal for view {:?} but we already have a payload for that view and they are not identical.  Throwing it away",
                       view)
                     );
@@ -216,11 +217,12 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> DaTaskState<TYP
                     tracing::trace!("{e:?}");
                 }
 
+                let payload = Arc::new(TYPES::BlockPayload::from_bytes(
+                    proposal.data.encoded_transactions.as_ref(),
+                    &proposal.data.metadata,
+                ));
                 // Record the payload we have promised to make available.
-                if let Err(e) = consensus_writer.update_saved_payloads(
-                    view_number,
-                    Arc::clone(&proposal.data.encoded_transactions),
-                ) {
+                if let Err(e) = consensus_writer.update_saved_payloads(view_number, payload) {
                     tracing::trace!("{e:?}");
                 }
                 // Optimistically calculate and update VID if we know that the primary network is down.
@@ -352,12 +354,16 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> DaTaskState<TYP
                     &event_stream,
                 )
                 .await;
+                let payload = Arc::new(TYPES::BlockPayload::from_bytes(
+                    encoded_transactions.as_ref(),
+                    metadata,
+                ));
                 // Save the payload early because we might need it to calculate VID for the next epoch nodes.
                 if let Err(e) = self
                     .consensus
                     .write()
                     .await
-                    .update_saved_payloads(view_number, Arc::clone(encoded_transactions))
+                    .update_saved_payloads(view_number, payload)
                 {
                     tracing::trace!("{e:?}");
                 }

crates/task-impls/src/helpers.rs

@@ -313,14 +313,12 @@ pub async fn decide_from_proposal_2<TYPES: NodeType>(
         res.leaf_views.push(info.clone());
         // If the block payload is available for this leaf, include it in
         // the leaf chain that we send to the client.
-        if let Some(encoded_txns) = consensus_reader
+        if let Some(payload) = consensus_reader
             .saved_payloads()
             .get(&info.leaf.view_number())
         {
-            let payload =
-                BlockPayload::from_bytes(encoded_txns, info.leaf.block_header().metadata());
-
-            info.leaf.fill_block_payload_unchecked(payload);
+            info.leaf
+                .fill_block_payload_unchecked(payload.as_ref().clone());
         }
 
         if let Some(ref payload) = info.leaf.block_payload() {
@@ -451,13 +449,8 @@ pub async fn decide_from_proposal<TYPES: NodeType>(
                 }
                 // If the block payload is available for this leaf, include it in
                 // the leaf chain that we send to the client.
-                if let Some(encoded_txns) =
-                    consensus_reader.saved_payloads().get(&leaf.view_number())
-                {
-                    let payload =
-                        BlockPayload::from_bytes(encoded_txns, leaf.block_header().metadata());
-
-                    leaf.fill_block_payload_unchecked(payload);
+                if let Some(payload) = consensus_reader.saved_payloads().get(&leaf.view_number()) {
+                    leaf.fill_block_payload_unchecked(payload.as_ref().clone());
                 }
 
                 // Get the VID share at the leaf's view number, corresponding to our key

crates/task-impls/src/transactions.rs

@@ -725,13 +725,6 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> TransactionTask
             bail!("No available blocks");
         }
 
-        let version = match self.upgrade_lock.version(view_number).await {
-            Ok(v) => v,
-            Err(err) => {
-                bail!("Upgrade certificate requires unsupported version, refusing to request blocks: {}", err);
-            }
-        };
-
         for (block_info, builder_idx) in available_blocks {
             // Verify signature over chosen block.
             if !block_info.sender.validate_block_info_signature(
@@ -758,19 +751,9 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> TransactionTask
             let response = {
                 let client = &self.builder_clients[builder_idx];
 
-                // If epochs are supported, provide the latest `num_nodes` information to the
-                // builder for VID computation.
-                let (block, header_input) = if version >= V::Epochs::VERSION {
-                    let total_nodes = self.membership.read().await.total_nodes(self.cur_epoch);
-                    futures::join! {
-                        client.claim_block_with_num_nodes(block_info.block_hash.clone(), view_number.u64(), self.public_key.clone(), &request_signature, total_nodes),
-                        client.claim_block_header_input(block_info.block_hash.clone(), view_number.u64(), self.public_key.clone(), &request_signature)
-                    }
-                } else {
-                    futures::join! {
-                        client.claim_block(block_info.block_hash.clone(), view_number.u64(), self.public_key.clone(), &request_signature),
-                        client.claim_block_header_input(block_info.block_hash.clone(), view_number.u64(), self.public_key.clone(), &request_signature)
-                    }
+                let (block, header_input) = futures::join! {
+                    client.claim_block(block_info.block_hash.clone(), view_number.u64(), self.public_key.clone(), &request_signature),
+                    client.claim_block_header_input(block_info.block_hash.clone(), view_number.u64(), self.public_key.clone(), &request_signature)
                 };
 
                 let block_data = match block {
@@ -815,7 +798,11 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> TransactionTask
                     fee,
                     block_payload: block_data.block_payload,
                     metadata: block_data.metadata,
-                    precompute_data: Some(header_input.vid_precompute_data),
+                    // we discard the precompute data,
+                    // because we cannot trust that the builder is able to calculate this correctly.
+                    //
+                    // in particular, the builder needs to know `num_nodes` and there aren't any practical ways to verify the result it sent us.
+                    precompute_data: None,
                 }
             };
 

crates/task-impls/src/vid.rs

@@ -76,7 +76,6 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>> VidTaskState<TYPES, I> {
                     metadata,
                     view_number,
                     sequencing_fees,
-                    vid_precompute,
                     auction_result,
                     ..
                 } = packed_bundle;
@@ -98,14 +97,14 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>> VidTaskState<TYPES, I> {
                     return None;
                 }
                 let vid_disperse = VidDisperse::calculate_vid_disperse(
-                    Arc::clone(encoded_transactions),
+                    &payload,
                     &Arc::clone(&self.membership),
                     *view_number,
                     epoch,
                     epoch,
-                    vid_precompute.clone(),
                 )
-                .await;
+                .await
+                .ok()?;
                 let payload_commitment = vid_disperse.payload_commitment;
                 let shares = VidDisperseShare2::from_vid_disperse(vid_disperse.clone());
                 let mut consensus_writer = self.consensus.write().await;
@@ -192,26 +191,26 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>> VidTaskState<TYPES, I> {
                 );
 
                 let consensus_reader = self.consensus.read().await;
-                let Some(txns) = consensus_reader.saved_payloads().get(&proposal_view_number)
+                let Some(payload) = consensus_reader.saved_payloads().get(&proposal_view_number)
                 else {
                     tracing::warn!(
                         "We need to calculate VID for the nodes in the next epoch \
                          but we don't have the transactions"
                     );
                     return None;
                 };
-                let txns = Arc::clone(txns);
+                let payload = Arc::clone(payload);
                 drop(consensus_reader);
 
                 let next_epoch_vid_disperse = VidDisperse::calculate_vid_disperse(
-                    txns,
+                    payload.as_ref(),
                     &Arc::clone(&self.membership),
                     proposal_view_number,
                     target_epoch,
                     sender_epoch,
-                    None,
                 )
-                .await;
+                .await
+                .ok()?;
                 let Ok(next_epoch_signature) = TYPES::SignatureKey::sign(
                     &self.private_key,
                     next_epoch_vid_disperse.payload_commitment.as_ref(),

crates/task/src/task.rs

@@ -127,9 +127,9 @@ impl<EVENT: Send + Sync + Clone + TaskEvent> ConsensusTaskRegistry<EVENT> {
         let handles = &mut self.task_handles;
 
         while let Some(handle) = handles.pop() {
-            let mut task_state = handle.await.unwrap();
-
-            task_state.cancel_subtasks();
+            let _ = handle
+                .await
+                .map(|mut task_state| task_state.cancel_subtasks());
         }
     }
     /// Take a task, run it, and register it