Feature branch blob proof

api/docgen/exampledata/blobProof.json

@@ -1,31 +1,53 @@
-[
-  {
-    "end": 8,
-    "nodes": [
-      "/////////////////////////////////////////////////////////////////////////////wuxStDHcZ7+b5byNQMVLJbzBT3wmObsThoQ0sCTjTCP"
+{
+  "ShareToRowRootProof": [
+    {
+      "start": 3,
+      "end": 4,
+      "nodes": [
+        "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ72eTVOUxB9THxFjAEwtTePJQA1b0xcz2f6TJc400Uw",
+        "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBD0CYbGYoGN4q9VfSmeGZeg/h1NDBA/jtXjZrrKRHE6",
+        "/////////////////////////////////////////////////////////////////////////////8KDE4JDf0N2lZB7DW1Fpasdk/wz4jHOxuBPAk5Vf5ZI"
+      ]
+    },
+    {
+      "end": 1,
+      "nodes": [
+        "//////////////////////////////////////7//////////////////////////////////////plEqgR/c4IAVkNdYRWOYOAESD4whneKR54Dz5Dfe4p2",
+        "//////////////////////////////////////7//////////////////////////////////////lrD0qJ9dspxSO1Yl8NDioZfgOm8Yj63Y+BGDRHlKCRj",
+        "/////////////////////////////////////////////////////////////////////////////xQyI+g89aM6rhy9rl2eKr0Uc2NPauf3fkLY3Z+gBtuM"
+      ]
+    }
+  ],
+  "RowProof": {
+    "row_roots": [
+      "00000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000808080808080808BC517066A5A8C81E2A4353DB500EBB3410047A93D2EE8ADF0B6797B9A5519557",
+      "0000000000000000000000000000000000000000000808080808080808FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE015AB6AAC6FAF0ABF26F9453AF390FDA3B39EB384F0B71D0170D84CF69CBA2BC"
     ],
-    "is_max_namespace_ignored": true
-  },
-  {
-    "end": 8,
-    "nodes": [
-      "//////////////////////////////////////////////////////////////////////////////n1NeJxPU2bZUAccKZZ+LAu2Wj5ajbVYURV9ojhSKwp"
+    "proofs": [
+      {
+        "total": 16,
+        "index": 1,
+        "leaf_hash": "lJek/BHnKH6PyRB8jlk69F6EY9Tfx2LRanaF74JVciU=",
+        "aunts": [
+          "bLjvftajE6jVsgQQBkV4RUPESRc+v4bhP0Ljf36858Q=",
+          "QaF9mNskaURxk98S3BExB1PzRAjOqVydrDLvUu0B5/M=",
+          "K2xW8JJ3Ff4FvtbfZi5ZD/ygnswaNCNIKXsSzbO2Jrc=",
+          "uySRG/gINLAgGgywJCTiXMlFkfQivF1O1zLg5+RRUP8="
+        ]
+      },
+      {
+        "total": 16,
+        "index": 2,
+        "leaf_hash": "h+4ND52kT4qkc9nWW22dIMAK/4YjkC6fBoD01WF0+Uo=",
+        "aunts": [
+          "2x8OISRBMLYJRV8NfTNtVvZUg2F7MtCK5xCZuE9fQwQ=",
+          "Xvr5IalE2y3pxHjxh5kcHFSRaz4g5MxdOj4NIGwRXY0=",
+          "K2xW8JJ3Ff4FvtbfZi5ZD/ygnswaNCNIKXsSzbO2Jrc=",
+          "uySRG/gINLAgGgywJCTiXMlFkfQivF1O1zLg5+RRUP8="
+        ]
+      }
     ],
-    "is_max_namespace_ignored": true
-  },
-  {
-    "end": 8,
-    "nodes": [
-      "/////////////////////////////////////////////////////////////////////////////0xK8BKnzDmwK0HR4ZJvyB4kh3jPPXGxaGPFoga8vPxF"
-    ],
-    "is_max_namespace_ignored": true
-  },
-  {
-    "end": 7,
-    "nodes": [
-      "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAMJ/xGlNMdEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwn/EaU0x0UTO9HUGKjyjcv5U2gHeSjJ8S1rftqv6k8kxlVWW8e/7",
-      "/////////////////////////////////////////////////////////////////////////////wexh4khLQ9HQ2X6nh9wU5B+m6r+LWwPTEDTa5/CosDF"
-    ],
-    "is_max_namespace_ignored": true
+    "start_row": 1,
+    "end_row": 3
   }
-]
+}

blob/blob.go

@@ -6,54 +6,162 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/tendermint/tendermint/crypto/merkle"
+	coretypes "github.com/tendermint/tendermint/types"
+
 	"github.com/celestiaorg/celestia-app/v3/pkg/appconsts"
-	"github.com/celestiaorg/go-square/merkle"
 	"github.com/celestiaorg/go-square/v2/inclusion"
 	libshare "github.com/celestiaorg/go-square/v2/share"
 	"github.com/celestiaorg/nmt"
 )
 
-// appVersion is the current application version of celestia-app.
-const appVersion = appconsts.LatestVersion
-
 var errEmptyShares = errors.New("empty shares")
 
-var subtreeRootThreshold = appconsts.SubtreeRootThreshold(appVersion)
+// Proof constructs the proof of a blob to the data root.
+type Proof struct {
+	// SubtreeRoots are the subtree roots of the blob's data that are
+	// used to create the commitment.
+	SubtreeRoots [][]byte `json:"subtree_roots"`
+	// SubtreeRootProofs the proofs of the subtree roots to the row roots they belong to.
+	// If the blob spans across multiple rows, then this will contain multiple proofs.
+	SubtreeRootProofs []*nmt.Proof `json:"share_to_row_root_proofs"`
+	// RowToDataRootProof the proofs of the row roots containing the blob shares
+	// to the data root.
+	RowToDataRootProof coretypes.RowProof `json:"row_to_data_root_proof"`
+}
 
-// The Proof is a set of nmt proofs that can be verified only through
-// the included method (due to limitation of the nmt https://github.com/celestiaorg/nmt/issues/218).
-// Proof proves the WHOLE namespaced data to the row roots.
-// TODO (@vgonkivs): rework `Proof` in order to prove a particular blob.
-// https://github.com/celestiaorg/celestia-node/issues/2303
-type Proof []*nmt.Proof
+// namespaceToRowRootProof a proof of a set of namespace shares to the row
+// roots they belong to.
+type namespaceToRowRootProof []*nmt.Proof
 
-func (p Proof) Len() int { return len(p) }
+// Commitment is a Merkle Root of the subtree built from shares of the Blob.
+// It is computed by splitting the blob into shares and building the Merkle subtree to be included
+// after Submit.
+type Commitment []byte
 
-// equal is a temporary method that compares two proofs.
-// should be removed in BlobService V1.
-func (p Proof) equal(input Proof) error {
-	if p.Len() != input.Len() {
-		return ErrInvalidProof
+// Verify takes a data root and verifies if the
+// provided proof's subtree roots were committed to the given data root.
+func (p *Proof) Verify(dataRoot []byte) (bool, error) {
+	if len(dataRoot) == 0 {
+		return false, errors.New("root must be non-empty")
 	}
 
-	for i, proof := range p {
-		pNodes := proof.Nodes()
-		inputNodes := input[i].Nodes()
-		for i, node := range pNodes {
-			if !bytes.Equal(node, inputNodes[i]) {
-				return ErrInvalidProof
-			}
-		}
+	subtreeRootThreshold := appconsts.SubtreeRootThreshold(appconsts.LatestVersion)
+	if subtreeRootThreshold <= 0 {
+		return false, errors.New("subtreeRootThreshold must be > 0")
+	}
+
+	// this check is < instead of != because we can have two subtree roots
+	// at the same height, depending on the subtree root threshold,
+	// and they can be used to create the above inner node without needing a proof inner node.
+	if len(p.SubtreeRoots) < len(p.SubtreeRootProofs) {
+		return false, fmt.Errorf(
+			"the number of subtree roots %d should be bigger than the number of subtree root proofs %d",
+			len(p.SubtreeRoots),
+			len(p.SubtreeRootProofs),
+		)
+	}
+
+	// for each row, one or more subtree roots' inclusion is verified against
+	// their corresponding row root. then, these row roots' inclusion is verified
+	// against the data root. so their number should be the same.
+	if len(p.SubtreeRootProofs) != len(p.RowToDataRootProof.Proofs) {
+		return false, fmt.Errorf(
+			"the number of subtree root proofs %d should be equal to the number of row root proofs %d",
+			len(p.SubtreeRootProofs),
+			len(p.RowToDataRootProof.Proofs),
+		)
+	}
+
+	// the row root proofs' ranges are defined as [startRow, endRow].
+	if int(p.RowToDataRootProof.EndRow-p.RowToDataRootProof.StartRow+1) != len(p.RowToDataRootProof.RowRoots) {
+		return false, fmt.Errorf(
+			"the number of rows %d must equal the number of row roots %d",
+			int(p.RowToDataRootProof.EndRow-p.RowToDataRootProof.StartRow+1),
+			len(p.RowToDataRootProof.RowRoots),
+		)
+	}
+	if len(p.RowToDataRootProof.Proofs) != len(p.RowToDataRootProof.RowRoots) {
+		return false, fmt.Errorf(
+			"the number of proofs %d must equal the number of row roots %d",
+			len(p.RowToDataRootProof.Proofs),
+			len(p.RowToDataRootProof.RowRoots),
+		)
+	}
+
+	// verify the inclusion of the rows to the data root
+	if err := p.RowToDataRootProof.Validate(dataRoot); err != nil {
+		return false, err
+	}
+
+	// computes the total number of shares proven given that each subtree root
+	// references a specific set of leaves.
+	numberOfShares := 0
+	for _, proof := range p.SubtreeRootProofs {
+		numberOfShares += proof.End() - proof.Start()
+	}
 
-		if proof.Start() != input[i].Start() || proof.End() != input[i].End() {
-			return ErrInvalidProof
+	// use the computed total number of shares to calculate the subtree roots
+	// width.
+	// the subtree roots width is defined in ADR-013:
+	//
+	//https://github.com/celestiaorg/celestia-app/blob/main/docs/architecture/adr-013-non-interactive-default-rules-for-zero-padding.md
+	subtreeRootsWidth := inclusion.SubTreeWidth(numberOfShares, subtreeRootThreshold)
+
+	nmtHasher := nmt.NewNmtHasher(appconsts.NewBaseHashFunc(), libshare.NamespaceSize, true)
+	// verify the proof of the subtree roots
+	subtreeRootsCursor := 0
+	for i, subtreeRootProof := range p.SubtreeRootProofs {
+		// calculate the share range that each subtree root commits to.
+		ranges, err := nmt.ToLeafRanges(subtreeRootProof.Start(), subtreeRootProof.End(), subtreeRootsWidth)
+		if err != nil {
+			return false, err
 		}
 
-		if !bytes.Equal(proof.LeafHash(), input[i].LeafHash()) {
-			return ErrInvalidProof
+		if len(p.SubtreeRoots) < subtreeRootsCursor {
+			return false, fmt.Errorf("len(commitmentProof.SubtreeRoots)=%d < subtreeRootsCursor=%d",
+				len(p.SubtreeRoots), subtreeRootsCursor)
 		}
+		if len(p.SubtreeRoots) < subtreeRootsCursor+len(ranges) {
+			return false, fmt.Errorf("len(commitmentProof.SubtreeRoots)=%d < subtreeRootsCursor+len(ranges)=%d",
+				len(p.SubtreeRoots), subtreeRootsCursor+len(ranges))
+		}
+		valid, err := subtreeRootProof.VerifySubtreeRootInclusion(
+			nmtHasher,
+			p.SubtreeRoots[subtreeRootsCursor:subtreeRootsCursor+len(ranges)],
+			subtreeRootsWidth,
+			p.RowToDataRootProof.RowRoots[i],
+		)
+		if err != nil {
+			return false, err
+		}
+		if !valid {
+			return false,
+				fmt.Errorf(
+					"subtree root proof for range [%d, %d) is invalid",
+					subtreeRootProof.Start(),
+					subtreeRootProof.End(),
+				)
+		}
+		subtreeRootsCursor += len(ranges)
 	}
-	return nil
+
+	return true, nil
+}
+
+// GenerateCommitment generates the share commitment corresponding
+// to the proof's subtree roots
+func (p *Proof) GenerateCommitment() Commitment {
+	return merkle.HashFromByteSlices(p.SubtreeRoots)
+}
+
+func (com Commitment) String() string {
+	return string(com)
+}
+
+// Equal ensures that commitments are the same
+func (com Commitment) Equal(c Commitment) bool {
+	return bytes.Equal(com, c)
 }
 
 // Blob represents any application-specific binary data that anyone can submit to Celestia.
@@ -94,7 +202,7 @@ func NewBlob(shareVersion uint8, namespace libshare.Namespace, data, signer []by
 		return nil, err
 	}
 
-	com, err := inclusion.CreateCommitment(libBlob, merkle.HashFromByteSlices, subtreeRootThreshold)
+	com, err := inclusion.CreateCommitment(libBlob, merkle.HashFromByteSlices, appconsts.DefaultSubtreeRootThreshold)
 	if err != nil {
 		return nil, err
 	}
@@ -177,3 +285,14 @@ func (b *Blob) UnmarshalJSON(data []byte) error {
 	*b = *blob
 	return nil
 }
+
+func (b *Blob) ComputeSubtreeRoots() ([][]byte, error) {
+	return inclusion.GenerateSubtreeRoots(b.Blob, appconsts.DefaultSubtreeRootThreshold)
+}
+
+// proveRowRootsToDataRoot creates a set of binary merkle proofs for all the
+// roots defined by the range [start, end).
+func proveRowRootsToDataRoot(roots [][]byte, start, end int) []*merkle.Proof {
+	_, proofs := merkle.ProofsFromByteSlices(roots)
+	return proofs[start:end]
+}

blob/blob_fuzz_test.go

@@ -41,9 +41,9 @@ func FuzzProofEqual(f *testing.F) {
 }
 
 type verifyCorpus struct {
-	CP         *CommitmentProof `json:"commitment_proof"`
-	Root       []byte           `json:"root"`
-	SThreshold int              `json:"sub_threshold"`
+	Proof      *Proof `json:"proof"`
+	Root       []byte `json:"root"`
+	SThreshold int    `json:"sub_threshold"`
 }
 
 func FuzzCommitmentProofVerify(f *testing.F) {
@@ -83,10 +83,10 @@ func FuzzCommitmentProofVerify(f *testing.F) {
 		if err := json.Unmarshal(valueJSON, val); err != nil {
 			return
 		}
-		commitProof := val.CP
+		commitProof := val.Proof
 		if commitProof == nil {
 			return
 		}
-		_, _ = commitProof.Verify(val.Root, val.SThreshold)
+		_, _ = commitProof.Verify(val.Root)
 	})
 }

blob/blob_test.go

@@ -6,8 +6,9 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/crypto/merkle"
 
-	"github.com/celestiaorg/go-square/merkle"
+	"github.com/celestiaorg/celestia-app/v3/pkg/appconsts"
 	"github.com/celestiaorg/go-square/v2/inclusion"
 	libshare "github.com/celestiaorg/go-square/v2/share"
 )
@@ -28,7 +29,7 @@ func TestBlob(t *testing.T) {
 			expectedRes: func(t *testing.T) {
 				require.NotEmpty(t, blob)
 				require.NotEmpty(t, blob[0].Namespace())
-				require.NotEmpty(t, blob[0].Data)
+				require.NotEmpty(t, blob[0].Data())
 				require.NotEmpty(t, blob[0].Commitment)
 			},
 		},
@@ -38,7 +39,7 @@ func TestBlob(t *testing.T) {
 				comm, err := inclusion.CreateCommitment(
 					blob[0].Blob,
 					merkle.HashFromByteSlices,
-					subtreeRootThreshold,
+					appconsts.DefaultSubtreeRootThreshold,
 				)
 				require.NoError(t, err)
 				assert.Equal(t, blob[0].Commitment, Commitment(comm))