feat: VID API tweaks, plus optimization for ADVZ disperse (#450)

* new LengthGetter trait and impl for advz

* add trait CommitChecker, delete fn check_common_commit_consistency

* use batch_commit in disperse for parallelism

* remove LengthGetter, CommitChecker; make them mandatory in CommitBounds, VidScheme

* use serde for CommonBounds instead of canonical

* add serde to VidScheme::[Commit | Common], need generic-array with serde (boo)

* add TODO for VidScheme::is_consistent

* move payload_byte_len into VidScheme, delete CommonBounds

primitives/Cargo.toml

@@ -35,6 +35,9 @@ derivative = { version = "2", features = ["use_core"] }
 digest = { version = "0.10.1", default-features = false, features = ["alloc"] }
 displaydoc = { version = "0.2.3", default-features = false }
 espresso-systems-common = { git = "https://github.com/espressosystems/espresso-systems-common", tag = "0.4.0" }
+generic-array = { version = "0", features = [
+        "serde",
+] } # not a direct dependency, but we need serde
 hashbrown = "0.13.1"
 itertools = { workspace = true, features = ["use_alloc"] }
 jf-relation = { path = "../relation", default-features = false }

primitives/src/vid.rs

@@ -6,28 +6,20 @@
 
 //! Trait and implementation for a Verifiable Information Retrieval (VID).
 /// See <https://arxiv.org/abs/2111.12323> section 1.3--1.4 for intro to VID semantics.
-use ark_serialize::{CanonicalDeserialize, CanonicalSerialize};
 use ark_std::{error::Error, fmt::Debug, hash::Hash, string::String, vec::Vec};
 use displaydoc::Display;
-use serde::{Deserialize, Serialize};
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
 
 /// VID: Verifiable Information Dispersal
 pub trait VidScheme {
     /// Payload commitment.
-    type Commit: Clone + Debug + Eq + PartialEq + Hash + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253
+    type Commit: Clone + Debug + DeserializeOwned + Eq + PartialEq + Hash + Serialize + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253
 
     /// Share-specific data sent to a storage node.
-    type Share: Clone + Debug + Eq + PartialEq + Hash + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253
+    type Share: Clone + Debug + DeserializeOwned + Eq + PartialEq + Hash + Serialize + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253
 
     /// Common data sent to all storage nodes.
-    type Common: CanonicalSerialize
-        + CanonicalDeserialize
-        + Clone
-        + Debug
-        + Eq
-        + PartialEq
-        + Hash
-        + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253
+    type Common: Clone + Debug + DeserializeOwned + Eq + PartialEq + Hash + Serialize + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253
 
     /// Compute a payload commitment
     fn commit_only<B>(&self, payload: B) -> VidResult<Self::Commit>
@@ -55,6 +47,16 @@ pub trait VidScheme {
     /// Recover payload from shares.
     /// Do not verify shares or check recovered payload against anything.
     fn recover_payload(&self, shares: &[Self::Share], common: &Self::Common) -> VidResult<Vec<u8>>;
+
+    /// Check that a [`VidScheme::Common`] is consistent with a
+    /// [`VidScheme::Commit`].
+    ///
+    /// TODO conform to nested result pattern like [`VidScheme::verify_share`].
+    /// Unfortunately, `VidResult<()>` is more user-friently.
+    fn is_consistent(commit: &Self::Commit, common: &Self::Common) -> VidResult<()>;
+
+    /// Extract the payload byte length data from a [`VidScheme::Common`].
+    fn get_payload_byte_len(common: &Self::Common) -> usize;
 }
 
 /// Convenience struct to aggregate disperse data.

primitives/src/vid/advz.rs

@@ -294,11 +294,7 @@ where
 
         let common_timer = start_timer!(|| format!("compute {} KZG commitments", polys.len()));
         let common = Common {
-            poly_commits: polys
-                .iter()
-                .map(|poly| UnivariateKzgPCS::commit(&self.ck, poly))
-                .collect::<Result<_, _>>()
-                .map_err(vid)?,
+            poly_commits: UnivariateKzgPCS::batch_commit(&self.ck, &polys).map_err(vid)?,
             all_evals_digest: all_evals_commit.commitment().digest(),
             bytes_len: payload_len,
         };
@@ -371,8 +367,7 @@ where
         if share.index >= self.num_storage_nodes {
             return Ok(Err(())); // not an arg error
         }
-
-        Self::check_common_commit_consistency(common, commit)?;
+        Self::is_consistent(commit, common)?;
 
         // verify eval proof
         if KzgEvalsMerkleTree::<E, H>::verify(
@@ -471,6 +466,19 @@ where
         payload.truncate(common.bytes_len);
         Ok(payload)
     }
+
+    fn is_consistent(commit: &Self::Commit, common: &Self::Common) -> VidResult<()> {
+        if *commit != Advz::<E, H>::derive_commit(&common.poly_commits, common.bytes_len)? {
+            return Err(VidError::Argument(
+                "common inconsistent with commit".to_string(),
+            ));
+        }
+        Ok(())
+    }
+
+    fn get_payload_byte_len(common: &Self::Common) -> usize {
+        common.bytes_len
+    }
 }
 
 impl<E, H> Advz<E, H>
@@ -544,18 +552,6 @@ where
         }
         Ok(hasher.finalize())
     }
-
-    fn check_common_commit_consistency(
-        common: &<Self as VidScheme>::Common,
-        commit: &<Self as VidScheme>::Commit,
-    ) -> VidResult<()> {
-        if *commit != Self::derive_commit(&common.poly_commits, common.bytes_len)? {
-            return Err(VidError::Argument(
-                "common inconsistent with commit".to_string(),
-            ));
-        }
-        Ok(())
-    }
 }
 
 /// Evaluate a generalized polynomial at a given point using Horner's method.

primitives/src/vid/advz/payload_prover.rs

@@ -24,7 +24,7 @@ use crate::{
     pcs::prelude::UnivariateKzgPCS,
     vid::{
         payload_prover::{PayloadProver, Statement},
-        vid, VidError,
+        vid, VidError, VidScheme,
     },
 };
 use anyhow::anyhow;
@@ -329,7 +329,7 @@ where
                 stmt.range.len()
             )));
         }
-        Self::check_common_commit_consistency(stmt.common, stmt.commit)
+        Self::is_consistent(stmt.commit, stmt.common)
     }
 }