diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 5d8d7b0d4f68..880533c906ac 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -28,13 +28,14 @@ import ( "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - cli "gopkg.in/urfave/cli.v1" "github.com/gballet/go-verkle" - "github.com/protolambda/go-kzg/bls" "github.com/protolambda/go-kzg" + "github.com/protolambda/go-kzg/bls" + cli "gopkg.in/urfave/cli.v1" ) var ( @@ -489,6 +490,40 @@ func computeCommitment(ctx *cli.Context) error { chain, chaindb := utils.MakeChain(ctx, stack, true) defer chaindb.Close() + verkledb, err := stack.OpenDatabase("verkle", 0, 0, "") + if err != nil { + log.Error("Failed to open db for verkle nodes", "error", err) + return err + } + defer verkledb.Close() + + nodesCh := make(chan verkle.FlushableNode) + verkleGenerate := func(db ethdb.KeyValueWriter, in chan snapshot.TrieKV, out chan common.Hash) { + t := verkle.New() + for leaf := range in { + t.InsertOrdered(common.CopyBytes(leaf.Key[:]), leaf.Value, ks, lg1, nodesCh) + } + // Flush remaining nodes to nodes channel + t.Flush(nodesCh) + comm := t.ComputeCommitment(ks, lg1) + root := common.BytesToHash(bls.ToCompressedG1(comm)) + out <- root + } + + nodesCount := 0 + go func() { + for fn := range nodesCh { + nodesCount++ + value, err := fn.Node.Serialize() + if err != nil { + log.Error("Failed to serialize verkle node", "error", err) + } + if err := verkledb.Put(fn.Hash[:], value); err != nil { + log.Error("Failed to write verkle node to db", "error", err) + } + } + }() + if ctx.NArg() > 1 { log.Error("Too many arguments given") return errors.New("too many arguments") @@ -511,56 +546,18 @@ func computeCommitment(ctx *cli.Context) error { root = head.Root() log.Info("Start traversing the state", "root", root, "number", head.NumberU64()) } + triedb := trie.NewDatabase(chaindb) - t, err := trie.NewSecure(root, triedb) + t, err := snapshot.New(chaindb, triedb, 256, chain.CurrentBlock().Root(), false, false, false) if err != nil { - log.Error("Failed to open trie", "root", root, "error", err) + log.Error("Failed to open snapshot tree", "error", err) return err } - var ( - accounts int - //slots int - //codes int - lastReport time.Time - start = time.Now() - ) - vRoot := verkle.New() - accIter := trie.NewIterator(t.NodeIterator(nil)) - for accIter.Next() { - accounts += 1 - vRoot.InsertOrdered(accIter.Key, accIter.Value, ks, lg1) - var acc state.Account - if err := rlp.DecodeBytes(accIter.Value, &acc); err != nil { - log.Error("Invalid account encountered during traversal", "error", err) - return err - } - if acc.Root != emptyRoot { - sRoot := verkle.New() - storageTrie, err := trie.NewSecure(acc.Root, triedb) - if err != nil { - log.Error("Failed to open storage trie", "root", acc.Root, "error", err) - return err - } - storageIter := trie.NewIterator(storageTrie.NodeIterator(nil)) - for storageIter.Next() { - sRoot.InsertOrdered(storageIter.Key, storageIter.Value, ks, lg1) - } - if storageIter.Err != nil { - log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIter.Err) - return storageIter.Err - } - } - if time.Since(lastReport) > time.Second*8 { - log.Info("Traversing state", "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) - lastReport = time.Now() - } - } - if accIter.Err != nil { - log.Error("Failed to compute commitment", "root", root, "error", accIter.Err) - return accIter.Err + if err := t.ComputeVerkleCommitment(root, verkleGenerate); err != nil { + log.Error("Failed to compute verkle commitment", "error", err) } - log.Info("Commitment computation complete", "compressed", bls.ToCompressedG1(vRoot.GetCommitment()), "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Number of nodes written to DB: %d\n", nodesCount) return nil } diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index bb87ecddf189..d7cbd78b8498 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -34,16 +34,16 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// trieKV represents a trie key-value pair -type trieKV struct { - key common.Hash - value []byte +// TrieKV represents a trie key-value pair +type TrieKV struct { + Key common.Hash + Value []byte } type ( // trieGeneratorFn is the interface of trie generation which can // be implemented by different trie algorithm. - trieGeneratorFn func(db ethdb.KeyValueWriter, in chan (trieKV), out chan (common.Hash)) + trieGeneratorFn func(db ethdb.KeyValueWriter, in chan (TrieKV), out chan (common.Hash)) // leafCallbackFn is the callback invoked at the leaves of the trie, // returns the subtrie root with the specified subtrie identifier. @@ -52,12 +52,12 @@ type ( // GenerateAccountTrieRoot takes an account iterator and reproduces the root hash. func GenerateAccountTrieRoot(it AccountIterator) (common.Hash, error) { - return generateTrieRoot(nil, it, common.Hash{}, stackTrieGenerate, nil, newGenerateStats(), true) + return generateTrieRoot(nil, it, common.Hash{}, stackTrieGenerate, nil, newGenerateStats(), true, true) } // GenerateStorageTrieRoot takes a storage iterator and reproduces the root hash. func GenerateStorageTrieRoot(account common.Hash, it StorageIterator) (common.Hash, error) { - return generateTrieRoot(nil, it, account, stackTrieGenerate, nil, newGenerateStats(), true) + return generateTrieRoot(nil, it, account, stackTrieGenerate, nil, newGenerateStats(), true, true) } // GenerateTrie takes the whole snapshot tree as the input, traverses all the @@ -87,12 +87,12 @@ func GenerateTrie(snaptree *Tree, root common.Hash, src ethdb.Database, dst ethd } defer storageIt.Release() - hash, err := generateTrieRoot(dst, storageIt, accountHash, stackTrieGenerate, nil, stat, false) + hash, err := generateTrieRoot(dst, storageIt, accountHash, stackTrieGenerate, nil, stat, false, true) if err != nil { return common.Hash{}, err } return hash, nil - }, newGenerateStats(), true) + }, newGenerateStats(), true, true) if err != nil { return err @@ -242,9 +242,9 @@ func runReport(stats *generateStats, stop chan bool) { // generateTrieRoot generates the trie hash based on the snapshot iterator. // It can be used for generating account trie, storage trie or even the // whole state which connects the accounts and the corresponding storages. -func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, generatorFn trieGeneratorFn, leafCallback leafCallbackFn, stats *generateStats, report bool) (common.Hash, error) { +func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, generatorFn trieGeneratorFn, leafCallback leafCallbackFn, stats *generateStats, report bool, checkSubRoot bool) (common.Hash, error) { var ( - in = make(chan trieKV) // chan to pass leaves + in = make(chan TrieKV) // chan to pass leaves out = make(chan common.Hash, 1) // chan to collect result stoplog = make(chan bool, 1) // 1-size buffer, works when logging is not enabled wg sync.WaitGroup @@ -289,7 +289,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, var ( logged = time.Now() processed = uint64(0) - leaf trieKV + leaf TrieKV ) // Start to feed leaves for it.Next() { @@ -321,7 +321,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, results <- err return } - if !bytes.Equal(account.Root, subroot.Bytes()) { + if checkSubRoot && !bytes.Equal(account.Root, subroot.Bytes()) { results <- fmt.Errorf("invalid subroot(%x), want %x, got %x", it.Hash(), account.Root, subroot) return } @@ -332,9 +332,9 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, return stop(err) } } - leaf = trieKV{it.Hash(), fullData} + leaf = TrieKV{it.Hash(), fullData} } else { - leaf = trieKV{it.Hash(), common.CopyBytes(it.(StorageIterator).Slot())} + leaf = TrieKV{it.Hash(), common.CopyBytes(it.(StorageIterator).Slot())} } in <- leaf @@ -360,10 +360,10 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, return stop(nil) } -func stackTrieGenerate(db ethdb.KeyValueWriter, in chan trieKV, out chan common.Hash) { +func stackTrieGenerate(db ethdb.KeyValueWriter, in chan TrieKV, out chan common.Hash) { t := trie.NewStackTrie(db) for leaf := range in { - t.TryUpdate(leaf.key[:], leaf.value) + t.TryUpdate(leaf.Key[:], leaf.Value) } var root common.Hash if db == nil { diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index aa5f5900b0cf..3d7f8f9d2669 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -718,12 +718,12 @@ func (t *Tree) Verify(root common.Hash) error { } defer storageIt.Release() - hash, err := generateTrieRoot(nil, storageIt, accountHash, stackTrieGenerate, nil, stat, false) + hash, err := generateTrieRoot(nil, storageIt, accountHash, stackTrieGenerate, nil, stat, false, true) if err != nil { return common.Hash{}, err } return hash, nil - }, newGenerateStats(), true) + }, newGenerateStats(), true, true) if err != nil { return err @@ -734,6 +734,35 @@ func (t *Tree) Verify(root common.Hash) error { return nil } +// Computes verkle commitment against snapshot +func (t *Tree) ComputeVerkleCommitment(root common.Hash, generatorFn trieGeneratorFn) error { + acctIt, err := t.AccountIterator(root, common.Hash{}) + if err != nil { + return err + } + defer acctIt.Release() + + got, err := generateTrieRoot(nil, acctIt, common.Hash{}, generatorFn, func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { + storageIt, err := t.StorageIterator(root, accountHash, common.Hash{}) + if err != nil { + return common.Hash{}, err + } + defer storageIt.Release() + + hash, err := generateTrieRoot(nil, storageIt, accountHash, generatorFn, nil, stat, false, false) + if err != nil { + return common.Hash{}, err + } + return hash, nil + }, newGenerateStats(), true, false) + + if err != nil { + return err + } + log.Info("Computed verkle commitment", "commitment", got) + return nil +} + // disklayer is an internal helper function to return the disk layer. // The lock of snapTree is assumed to be held already. func (t *Tree) disklayer() *diskLayer { diff --git a/go.mod b/go.mod index e72158b000eb..9f46e9953299 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/fatih/color v1.7.0 github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff - github.com/gballet/go-verkle v0.0.0-20210318184615-3228314e99c9 + github.com/gballet/go-verkle v0.0.0-20210331172732-b0e741f9a8ec github.com/go-stack/stack v1.8.0 github.com/golang/protobuf v1.4.3 github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 @@ -26,6 +26,7 @@ require ( github.com/gorilla/websocket v1.4.2 github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d + github.com/herumi/bls-eth-go-binary v0.0.0-20210325004959-a24496ac0743 // indirect github.com/holiman/bloomfilter/v2 v2.0.3 github.com/holiman/uint256 v1.1.1 github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031 @@ -49,8 +50,8 @@ require ( github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef - golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670 - golang.org/x/sys v0.0.0-20210319071255-635bc2c9138d + golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 + golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 golang.org/x/text v0.3.3 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce diff --git a/go.sum b/go.sum index 9bf821500e08..2e6ccdcef93b 100644 --- a/go.sum +++ b/go.sum @@ -155,6 +155,8 @@ github.com/gballet/go-verkle v0.0.0-20210305125751-d07bfb0dd271 h1:SQ8VtqI0zpGTA github.com/gballet/go-verkle v0.0.0-20210305125751-d07bfb0dd271/go.mod h1:qJML0ZSm9IvJ4SV9Iy/s5mNYld5tyLvVzoHTaC1Weq0= github.com/gballet/go-verkle v0.0.0-20210318184615-3228314e99c9 h1:rOR2V9fSuvpAO+4k6c34OzDyAAlA4eGLSJoYTlxZQns= github.com/gballet/go-verkle v0.0.0-20210318184615-3228314e99c9/go.mod h1:s2tVAxI5FnThDvOCzrUpNU8HxoGvFBelNLZpDEEsFuk= +github.com/gballet/go-verkle v0.0.0-20210331172732-b0e741f9a8ec h1:SFb34albxbnAcY+wLS7nKL+nG6RVcQpmREE2lpJLEIk= +github.com/gballet/go-verkle v0.0.0-20210331172732-b0e741f9a8ec/go.mod h1:s2tVAxI5FnThDvOCzrUpNU8HxoGvFBelNLZpDEEsFuk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= @@ -258,6 +260,8 @@ github.com/herumi/bls-eth-go-binary v0.0.0-20210128192016-d750c7e71022/go.mod h1 github.com/herumi/bls-eth-go-binary v0.0.0-20210302070600-dfaa902c7773/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= github.com/herumi/bls-eth-go-binary v0.0.0-20210311024029-2939a1bfdec2 h1:WYYASILSlSdBFLQRysRQFuRW7VHxyNDbmYaGo3T2PkA= github.com/herumi/bls-eth-go-binary v0.0.0-20210311024029-2939a1bfdec2/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= +github.com/herumi/bls-eth-go-binary v0.0.0-20210325004959-a24496ac0743 h1:r9ipCR8S2CsnXftdEBGBRNvd69Hmggm83eajtzsguw4= +github.com/herumi/bls-eth-go-binary v0.0.0-20210325004959-a24496ac0743/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw= @@ -496,6 +500,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnk golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670 h1:gzMM0EjIYiRmJI3+jBdFuoynZlpxa2JQZsolKu09BXo= golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -600,6 +606,8 @@ golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEq golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210319071255-635bc2c9138d h1:jbzgAvDZn8aEnytae+4ou0J0GwFZoHR0hOrTg4qH8GA= golang.org/x/sys v0.0.0-20210319071255-635bc2c9138d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 h1:rF3Ohx8DRyl8h2zw9qojyLHLhrJpEMgyPOImREEryf0= +golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=