primitives: Add core merkle tree root calcs.

internal/staging/primitives/README.md

@@ -27,6 +27,8 @@ The provided functions fall into the following categories:
   - Calculating work values based on the target difficulty bits
   - Checking that a block hash satisfies a target difficulty and that the target
     difficulty is within a valid range
+- Merkle root calculation
+  - Calculation from individual leaf hashes
 
 ## Maintainer Note
 

internal/staging/primitives/merkle.go

@@ -0,0 +1,104 @@
+// Copyright (c) 2019-2021 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package primitives
+
+import (
+	"github.com/decred/dcrd/chaincfg/chainhash"
+)
+
+// CalcMerkleRootInPlace is an in-place version of CalcMerkleRoot that reuses
+// the backing array of the provided slice to perform the calculation thereby
+// preventing extra allocations.  It is the caller's responsibility to ensure it
+// is safe to mutate the entries in the provided slice.
+//
+// The function internally appends an additional entry in the case the number of
+// provided leaves is odd, so the caller may wish to pre-allocate space for one
+// additional element in the backing array in that case to ensure it doesn't
+// need to be reallocated to expand it.
+//
+// For example:
+//
+//   allocLen := len(leaves) + len(leaves)&1
+//   leaves := make([]chainhash.Hash, len(leaves), allocLen)
+//   // populate the leaves
+//
+// See CalcMerkleRoot for more details on how the merkle root is calculated.
+func CalcMerkleRootInPlace(leaves []chainhash.Hash) chainhash.Hash {
+	if len(leaves) == 0 {
+		// All zero.
+		return chainhash.Hash{}
+	}
+
+	// Create a buffer to reuse for hashing the branches and some long lived
+	// slices into it to avoid reslicing.
+	var buf [2 * chainhash.HashSize]byte
+	var left = buf[:chainhash.HashSize]
+	var right = buf[chainhash.HashSize:]
+	var both = buf[:]
+
+	// The following algorithm works by replacing the leftmost entries in the
+	// slice with the concatenations of each subsequent set of 2 hashes and
+	// shrinking the slice by half to account for the fact that each level of
+	// the tree is half the size of the previous one.  In the case a level is
+	// unbalanced (there is no final right child), the final node is duplicated
+	// so it ultimately is concatenated with itself.
+	//
+	// For example, the following illustrates calculating a tree with 5 leaves:
+	//
+	// [0 1 2 3 4]                              (5 entries)
+	// 1st iteration: [h(0||1) h(2||3) h(4||4)] (3 entries)
+	// 2nd iteration: [h(h01||h23) h(h44||h44)] (2 entries)
+	// 3rd iteration: [h(h0123||h4444)]         (1 entry)
+	for len(leaves) > 1 {
+		// When there is no right child, the parent is generated by hashing the
+		// concatenation of the left child with itself.
+		if len(leaves)&1 != 0 {
+			leaves = append(leaves, leaves[len(leaves)-1])
+		}
+
+		// Set the parent node to the hash of the concatenation of the left and
+		// right children.
+		for i := 0; i < len(leaves)/2; i++ {
+			copy(left, leaves[i*2][:])
+			copy(right, leaves[i*2+1][:])
+			leaves[i] = chainhash.HashH(both)
+		}
+		leaves = leaves[:len(leaves)/2]
+	}
+	return leaves[0]
+}
+
+// CalcMerkleRoot treats the provided slice of hashes as leaves of a merkle tree
+// and returns the resulting merkle root.
+//
+// A merkle tree is a tree in which every non-leaf node is the hash of its
+// children nodes.  A diagram depicting how this works for Decred transactions
+// where h(x) is a blake256 hash follows:
+//
+//	         root = h1234 = h(h12 + h34)
+//	        /                           \
+//	  h12 = h(h1 + h2)            h34 = h(h3 + h4)
+//	   /            \              /            \
+//	h1 = h(tx1)  h2 = h(tx2)    h3 = h(tx3)  h4 = h(tx4)
+//
+// The number of inputs is not always a power of two which results in a
+// balanced tree structure as above.  In that case, parent nodes with no
+// children are also zero and parent nodes with only a single left node
+// are calculated by concatenating the left node with itself before hashing.
+func CalcMerkleRoot(leaves []chainhash.Hash) chainhash.Hash {
+	if len(leaves) == 0 {
+		// All zero.
+		return chainhash.Hash{}
+	}
+
+	// Copy the leaves so they can be safely mutated by the in-place merkle root
+	// calculation.  Note that the backing array is provided with space for one
+	// additional item when the number of leaves is odd as an optimization for
+	// the in-place calculation to avoid the need to grow the backing array.
+	allocLen := len(leaves) + len(leaves)&1
+	dupLeaves := make([]chainhash.Hash, len(leaves), allocLen)
+	copy(dupLeaves, leaves)
+	return CalcMerkleRootInPlace(dupLeaves)
+}

internal/staging/primitives/merkle_bench_test.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2019-2021 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package primitives
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/decred/dcrd/chaincfg/chainhash"
+)
+
+// BenchmarkCalcMerkleRootInPlace benchmarks merkle root calculation for various
+// numbers of leaves using the mutable in-place algorithm.
+func BenchmarkCalcMerkleRootInPlace(b *testing.B) {
+	// Create several slices of leaves of various sizes to benchmark.
+	numLeavesToBench := []int{20, 1000, 2000, 4000, 8000, 16000, 32000}
+	origLeaves := make([][]chainhash.Hash, len(numLeavesToBench))
+	for i, numLeaves := range numLeavesToBench {
+		origLeaves[i] = make([]chainhash.Hash, numLeaves)
+	}
+
+	for benchIdx := range origLeaves {
+		testLeaves := origLeaves[benchIdx]
+		benchName := fmt.Sprintf("%d leaves", len(testLeaves))
+		b.Run(benchName, func(b *testing.B) {
+			b.ResetTimer()
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				_ = CalcMerkleRootInPlace(testLeaves)
+			}
+		})
+	}
+}
+
+// BenchmarkCalcMerkleRoot benchmarks merkle root calculation for various
+// numbers of leaves using the non-mutable version.
+func BenchmarkCalcMerkleRoot(b *testing.B) {
+	// Create several slices of leaves of various sizes to benchmark.
+	numLeavesToBench := []int{20, 1000, 2000, 4000, 8000, 16000, 32000}
+	origLeaves := make([][]chainhash.Hash, len(numLeavesToBench))
+	for i, numLeaves := range numLeavesToBench {
+		origLeaves[i] = make([]chainhash.Hash, numLeaves)
+	}
+
+	for benchIdx := range origLeaves {
+		testLeaves := origLeaves[benchIdx]
+		benchName := fmt.Sprintf("%d leaves", len(testLeaves))
+		b.Run(benchName, func(b *testing.B) {
+			b.ResetTimer()
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				_ = CalcMerkleRoot(testLeaves)
+			}
+		})
+	}
+}

internal/staging/primitives/merkle_test.go

@@ -0,0 +1,132 @@
+// Copyright (c) 2019-2021 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package primitives
+
+import (
+	"testing"
+
+	"github.com/decred/dcrd/chaincfg/chainhash"
+)
+
+// TestCalcMerkleRoot ensures the expected merkle root is produced for known
+// valid leaf values.
+func TestCalcMerkleRoot(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name   string   // test description
+		leaves []string // leaves to test
+		want   string   // expected result
+	}{{
+		name:   "no leaves",
+		leaves: nil,
+		want:   "0000000000000000000000000000000000000000000000000000000000000000",
+	}, {
+		name: "single leaf (mainnet block 1)",
+		leaves: []string{
+			"b4895fb9d0b54822550828f2ba07a68ddb1894796800917f8672e65067696347",
+		},
+		want: "b4895fb9d0b54822550828f2ba07a68ddb1894796800917f8672e65067696347",
+	}, {
+		name: "even number of leaves (mainnet block 257)",
+		leaves: []string{
+			"46670d055dae85e8f9eceb5d30b1433c7232d3b09068fbde4741db3714dafdb7",
+			"9518f53fccc008baf771a6610d4ac506a931286b7e67d98d49bde68e3dec10aa",
+			"c9bf74b6da5a82e5f720859f9b7730aab59e774fb1c22bef534e60206c1f87b4",
+			"c0657dd580e76866de1a008e691ffcafe790deb733ec79b7b4dea64ab4abd002",
+			"7ce1b2613e21f40d7076c1b2283f363134be992b5fd648a928f023e9cf42de5e",
+			"2f568d89cde2957d68a27f41854245b73c1469314e7f31783614bf1919761bcf",
+			"e146022bebf7a4273a61084ce20ee5c03f94afbe6744ed48e436169a147a1d1c",
+			"a714a3a6f16b18c5b82321b9425a4205b205afd4d83d3f392d6a36af4222c9dd",
+			"25f65b3814c55de20576d35fc68ecc202bf058352746c9e2347f7e59f5a2c677",
+			"81120d7af7f8d37287ecf558a2d47f1e631bec486e485cb4aab4996a1c2ee7ab",
+			"0e3e1ffd23240dbc3e148754eb63faa784e9d338f196cf77b5d821749282fb0c",
+			"91d53551633e8b7a894b4e7277616f65203e997c4346895d234a8a2dcea6c849",
+			"3caf3db1714a8f7c9b847be782ee2750f3f7073eadbc43a309c800a3d6b1c887",
+			"41161b6e5cc65bee31a26b1603e5d701151d9778de6cd0044fb5533dd0da7fe7",
+			"a1273c356109ff1d6145eca2ed14b1c5025f0024bf18ae249b8d185b4192cf6e",
+			"ceed5ebb8faa597795d04fe06c404e32e72d9d6db43d57b41affc842c402a5c8",
+			"7c756776f01aa0e2b115bbef0527a12fe03aadf598fdbf99576dc973fbc42cdc",
+			"472c27828b8ecd51f038a676aa9dc2e8d144cc292885e342a37852ec6d0d78a7",
+			"bbc48709276a223b6689d181aacfd8684fbb5a91bd7c890e487a3b73ab4b43d5",
+			"6c796c53a51ecf8fa0dd7feffbf3c1ca277b17533bb6fc87645527471c2d5499",
+			"bec32f1016fd40f2adac39dfbcedb3e45b6d7f9b37cb340d22bce14015759632",
+			"06024a8ddaafa5c4b448168bebd8f37d7fb15eef079933579cf29b45dd40edfb",
+		},
+		want: "4aa7bcd77d51f6f4db4983e731b5e08b3ea724c5cb99d3debd3d75fd67e7c72b",
+	}, {
+		name: "odd number of leaves > 1 (mainnet block 260)",
+		leaves: []string{
+			"5e574591d900f7f9abb8f8eb31cc9330247d27ba293ad79c348d602ece717b8b",
+			"b3b70fe08c2da744c9559d533e8db35b3bfefba1b0f1c7b31e7d9d523c00a426",
+			"dd3058a7fc691ff4dee0a8cd6030f404ffda7e7aee88aff3985f7b2bbe4792f7",
+		},
+		want: "a144c719391569aa20bf612bf5588bce71cd397574cb6c060e0bac100f6e5805",
+	}}
+
+	testFuncs := []string{"CalcMerkleRoot", "CalcMerkleRootInPlace"}
+	for _, funcName := range testFuncs {
+	nextTest:
+		for _, test := range tests {
+			// Parse the leaves and store a copy for ensuring they were not
+			// mutated.
+			leaves := make([]chainhash.Hash, 0, len(test.leaves))
+			for _, hashStr := range test.leaves {
+				hash, err := chainhash.NewHashFromStr(hashStr)
+				if err != nil {
+					t.Errorf("%q: unexpected err parsing leaf %q: %v",
+						test.name, hashStr, err)
+					continue nextTest
+				}
+				leaves = append(leaves, *hash)
+			}
+			origLeaves := make([]chainhash.Hash, len(leaves))
+			copy(origLeaves, leaves)
+
+			// Parse the expected merkle root.
+			want, err := chainhash.NewHashFromStr(test.want)
+			if err != nil {
+				t.Errorf("%q: unexpected err parsing want hex: %v", test.name,
+					err)
+				continue nextTest
+			}
+
+			// Choose the correct function to use to calculate the merkle root
+			// for this iteration.
+			var f func([]chainhash.Hash) chainhash.Hash
+			switch funcName {
+			case "CalcMerkleRoot":
+				f = CalcMerkleRoot
+			case "CalcMerkleRootInPlace":
+				f = CalcMerkleRootInPlace
+			default:
+				t.Fatalf("invalid function name: %v", funcName)
+			}
+			result := f(leaves)
+			if result != *want {
+				t.Errorf("%q: mismatched result -- got %v, want %v", test.name,
+					result, *want)
+				continue nextTest
+			}
+
+			// Ensure the leaves were not mutated for the immutable version.
+			if funcName == "CalcMerkleRoot" {
+				if len(leaves) != len(origLeaves) {
+					t.Errorf("%q: unexpected leaf mutation -- len %v, want %v",
+						test.name, len(leaves), len(origLeaves))
+					continue nextTest
+				}
+
+				for i := range leaves {
+					if leaves[i] != origLeaves[i] {
+						t.Errorf("%q: unexpected mutation -- got %v, want %v",
+							test.name, leaves[i], origLeaves[i])
+						continue nextTest
+					}
+				}
+			}
+		}
+	}
+}