diff --git a/blockchain/msgs_test.go b/blockchain/msgs_test.go index ccac62c65b..818c4cb2cb 100644 --- a/blockchain/msgs_test.go +++ b/blockchain/msgs_test.go @@ -97,7 +97,7 @@ func TestBlockchainMessageVectors(t *testing.T) { BlockRequest: &bcproto.BlockRequest{Height: math.MaxInt64}}}, "0a0a08ffffffffffffffff7f"}, {"BlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_BlockResponse{ - BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1ac0020abd020a5b0a02080b1803220b088092b8c398feffffff012a0212003a20ba28ef83fed712be8a128587469a4effda9f4f4895bd5638dab1f10775c9bed66a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512130a0b48656c6c6f20576f726c6412001a0022001ac8010a300000000000000001000000000000000119251d276ad8a1831db7b86ead3f42c4e03093d50ecf026da7ecc3b0da8ec87d0a30ffffffffffffffffffffffffffffffff12d55aea72367d0d6a7899103a437c913ee5a6f9e86f42e0fe8743b0d8d3a1e812300000000000000001000000000000000119251d276ad8a1831db7b86ead3f42c4e03093d50ecf026da7ecc3b0da8ec87d1230ffffffffffffffffffffffffffffffff12d55aea72367d0d6a7899103a437c913ee5a6f9e86f42e0fe8743b0d8d3a1e8"}, + BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1ac0020abd020a5b0a02080b1803220b088092b8c398feffffff012a0212003a2016106406aededa633a265efd5833af53775af5a27c803518313c9b77e68875926a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512130a0b48656c6c6f20576f726c6412001a0022001ac8010a3000000000000000010000000000000001266fcca850585e31b54074399b0dbb5b04b408b77d5f357a528ed3e04b63c3c10a30ffffffffffffffffffffffffffffffffe04e02811b28234428f1c1219d5627f61c02c415c175bdabd9e7934519baef07123000000000000000010000000000000001266fcca850585e31b54074399b0dbb5b04b408b77d5f357a528ed3e04b63c3c11230ffffffffffffffffffffffffffffffffe04e02811b28234428f1c1219d5627f61c02c415c175bdabd9e7934519baef07"}, {"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{ NoBlockResponse: &bcproto.NoBlockResponse{Height: 1}}}, "12020801"}, {"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{ diff --git a/config/ipfs_config.go b/config/ipfs_config.go index ecaae8cbd5..1475a4b1c4 100644 --- a/config/ipfs_config.go +++ b/config/ipfs_config.go @@ -22,7 +22,7 @@ type IPFSConfig struct { // locally for testing purposes. func DefaultIPFSConfig() *IPFSConfig { return &IPFSConfig{ - ConfigRootPath: "ipfs/", + ConfigRootPath: ".ipfs/", API: "/ip4/127.0.0.1/tcp/5002", Gateway: "/ip4/127.0.0.1/tcp/5002", Swarm: []string{"/ip4/0.0.0.0/tcp/4002", "/ip6/::/tcp/4002"}, diff --git a/consensus/state.go b/consensus/state.go index 2a2f75ba75..7110ea047d 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -2,6 +2,7 @@ package consensus import ( "bytes" + "context" "errors" "fmt" "io/ioutil" @@ -11,7 +12,7 @@ import ( "time" "github.com/gogo/protobuf/proto" - + ipfsapi "github.com/ipfs/interface-go-ipfs-core" cfg "github.com/lazyledger/lazyledger-core/config" cstypes "github.com/lazyledger/lazyledger-core/consensus/types" "github.com/lazyledger/lazyledger-core/crypto" @@ -92,6 +93,8 @@ type State struct { // store blocks and commits blockStore sm.BlockStore + IpfsAPI ipfsapi.CoreAPI + // create and execute blocks blockExec *sm.BlockExecutor @@ -1100,6 +1103,19 @@ func (cs *State) defaultDecideProposal(height int64, round int32) { } else if !cs.replayMode { cs.Logger.Error("enterPropose: Error signing proposal", "height", height, "round", round, "err", err) } + + // post data to ipfs + // TODO(evan): don't hard code context and timeout + if cs.IpfsAPI != nil { + // longer timeouts result in block proposers failing to propose blocks in time. + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*1500) + defer cancel() + // TODO: post data to IPFS in a goroutine + err := block.PutBlock(ctx, cs.IpfsAPI.Dag().Pinning()) + if err != nil { + cs.Logger.Error(fmt.Sprintf("failure to post block data to IPFS: %s", err.Error())) + } + } } // Returns true if the proposal block is complete && diff --git a/docs/lazy-adr/adr-002-ipld-da-sampling.md b/docs/lazy-adr/adr-002-ipld-da-sampling.md index cb21e75499..aa18ce5403 100644 --- a/docs/lazy-adr/adr-002-ipld-da-sampling.md +++ b/docs/lazy-adr/adr-002-ipld-da-sampling.md @@ -161,7 +161,7 @@ func RetrieveBlockData(ctx contex.Context, dah *DataAvailabilityHeader) (types.D // the row to the Merkle Dag, in our case a Namespaced Merkle Tree. // Note, that this method could also fill the DA header. // The data will be pinned by default. -func (b *Block) PutBlock(ctx contex.Context) error +func (b *Block) PutBlock(ctx contex.Context, nodeAdder ipld.NodeAdder) error ``` We now describe the lower-level library that will be used by above methods. @@ -185,15 +185,6 @@ func GetLeafData( leafIndex uint32, totalLeafs uint32, // this corresponds to the extended square width ) ([]byte, error) - -// PutLeaves takes the namespaced leaves, a row of the from the extended data square, -// and calls nodes.DataSquareRowOrColumnRawInputParser of the ipld plugin. -// The resulting ipld nodes are passed to a Batch calling AddMany: -// https://github.com/ipfs/go-ipld-format/blob/d2e09424ddee0d7e696d01143318d32d0fb1ae63/batch.go#L29 -// Note, that this method could also return the row and column roots. -// Tha caller is responsible for making sure that the leaves are sorted by namespace ID. -// The data will be pinned by default. -func PutLeaves(ctx contex.Context, namespacedLeaves [][]byte) error ``` `GetLeafData` can be used by above `ValidateAvailability` and `RetrieveBlock` and diff --git a/go.mod b/go.mod index 6e06bdbe38..7626e2b521 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/hdevalence/ed25519consensus v0.0.0-20201207055737-7fde80a9d5ff github.com/ipfs/go-ipfs v0.8.0 github.com/ipfs/go-ipfs-config v0.12.0 + github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/interface-go-ipfs-core v0.4.0 github.com/lazyledger/lazyledger-core/p2p/ipld/plugin v0.0.0-20210219190522-0eccfb24e2aa github.com/lazyledger/nmt v0.2.0 @@ -46,5 +47,7 @@ require ( replace ( github.com/ipfs/go-ipfs => github.com/lazyledger/go-ipfs v0.8.0-lazypatch + // adding an extra replace statement here enforces usage of our fork of go-cerifcid + github.com/ipfs/go-verifcid => github.com/lazyledger/go-verifcid v0.0.1-lazypatch github.com/lazyledger/lazyledger-core/p2p/ipld/plugin => ./p2p/ipld/plugin ) diff --git a/go.sum b/go.sum index 87f8ad4eb7..7b97c08413 100644 --- a/go.sum +++ b/go.sum @@ -623,6 +623,8 @@ github.com/lazyledger/go-ipfs v0.8.0-lazypatch h1:8Dkw7Or6d0BmpFYFcxwgqWZ047BPGC github.com/lazyledger/go-ipfs v0.8.0-lazypatch/go.mod h1:CE4cJkjUmwW5LwJP26KKEAZ11ZED0DxzSryfv5RMf6E= github.com/lazyledger/go-leopard v0.0.0-20200604113236-298f93361181 h1:mUeCGuCgjZVadW4CzA2dMBq7p2BqaoCfpnKjxMmSaSE= github.com/lazyledger/go-leopard v0.0.0-20200604113236-298f93361181/go.mod h1:v1o1CRihQ9i7hizx23KK4aR79lxA6VDUIzUCfDva0XQ= +github.com/lazyledger/go-verifcid v0.0.1-lazypatch h1:jAVwUw+DhuCzx5IcYpFh6d6HWxRRz8nhJ3rQo+vlFAc= +github.com/lazyledger/go-verifcid v0.0.1-lazypatch/go.mod h1:kXPYu0XqTNUKWA1h3M95UHjUqBzDwXVVt/RXZDjKJmQ= github.com/lazyledger/merkletree v0.0.0-20201214195110-6901c4c3c75f h1:jbyPAH6o6hGte4RtZBaqWs2n4Fl6hS7qJGXX3qnjiy4= github.com/lazyledger/merkletree v0.0.0-20201214195110-6901c4c3c75f/go.mod h1:10PA0NlnYtB8HrtwIDQAyTKWp8TEZ0zBZCGlYC/7+QE= github.com/lazyledger/nmt v0.2.0 h1:jKpC+XMSc4P1asLvutSwyMlgUDnAvv1HAVyzal21MbA= diff --git a/node/node.go b/node/node.go index d25716a573..80febbbea9 100644 --- a/node/node.go +++ b/node/node.go @@ -13,6 +13,7 @@ import ( "time" ipfscore "github.com/ipfs/go-ipfs/core" + coreapi "github.com/ipfs/go-ipfs/core/coreapi" "github.com/ipfs/go-ipfs/core/node/libp2p" "github.com/ipfs/go-ipfs/plugin/loader" "github.com/ipfs/go-ipfs/repo/fsrepo" @@ -960,6 +961,13 @@ func (n *Node) OnStart() error { if err != nil { return fmt.Errorf("failed to create IPFS node: %w", err) } + + ipfsAPI, err := coreapi.NewCoreAPI(n.ipfsNode) + if err != nil { + return fmt.Errorf("failed to create an instance of the IPFS core API: %w", err) + } + + n.consensusState.IpfsAPI = ipfsAPI } return nil diff --git a/p2p/ipld/plugin/go.sum b/p2p/ipld/plugin/go.sum index 34e6f011fd..5ca7a3f255 100644 --- a/p2p/ipld/plugin/go.sum +++ b/p2p/ipld/plugin/go.sum @@ -316,8 +316,6 @@ github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0 github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= github.com/ipfs/go-graphsync v0.5.1 h1:4fXBRvRKicTgTmCFMmEua/H5jvmAOLgU9Z7PCPWt2ec= github.com/ipfs/go-graphsync v0.5.1/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk= -github.com/ipfs/go-graphsync v0.5.2 h1:USD+daaSC+7pLHCxROThSaF6SF7WYXF03sjrta0rCfA= -github.com/ipfs/go-graphsync v0.5.2/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk= github.com/ipfs/go-ipfs-api v0.2.0 h1:BXRctUU8YOUOQT/jW1s56d9wLa85ntOqK6bptvCKb8c= github.com/ipfs/go-ipfs-api v0.2.0/go.mod h1:zCTyTl+BuyvUqoSmVb8vjezCJLVTW7G/HBZbCXpTgeM= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= @@ -329,9 +327,8 @@ github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtL github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= +github.com/ipfs/go-ipfs-cmds v0.5.0 h1:T1ZT6Qu3IUCp6FgU2IzVtvGLaexEWo9q13+S5ic+Q5Y= github.com/ipfs/go-ipfs-cmds v0.5.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= -github.com/ipfs/go-ipfs-cmds v0.6.0 h1:yAxdowQZzoFKjcLI08sXVNnqVj3jnABbf9smrPQmBsw= -github.com/ipfs/go-ipfs-cmds v0.6.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= github.com/ipfs/go-ipfs-config v0.11.0 h1:w4t2pz415Gtg6MTUKAq06C7ezC59/Us+k3+n1Tje+wg= github.com/ipfs/go-ipfs-config v0.11.0/go.mod h1:Ei/FLgHGTdPyqCPK0oPCwGTe8VSnsjJjx7HZqUb6Ry0= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -349,8 +346,6 @@ github.com/ipfs/go-ipfs-files v0.0.8 h1:8o0oFJkJ8UkO/ABl8T6ac6tKF3+NIpj67aAB6Zpu github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs= github.com/ipfs/go-ipfs-pinner v0.1.0 h1:rjSrbUDYd1YYHZ5dOgu+QEOuLcU0m/2a/brcxC/ReeU= github.com/ipfs/go-ipfs-pinner v0.1.0/go.mod h1:EzyyaWCWeZJ/he9cDBH6QrEkSuRqTRWMmCoyNkylTTg= -github.com/ipfs/go-ipfs-pinner v0.1.1 h1:iJd1gwILGQJSZhhI0jn6yFOLg34Ua7fdKcB6mXp6k/M= -github.com/ipfs/go-ipfs-pinner v0.1.1/go.mod h1:EzyyaWCWeZJ/he9cDBH6QrEkSuRqTRWMmCoyNkylTTg= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= @@ -405,8 +400,6 @@ github.com/ipfs/go-mfs v0.1.2/go.mod h1:T1QBiZPEpkPLzDqEJLNnbK55BVKVlNi2a+gVm4di github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno= github.com/ipfs/go-path v0.0.8 h1:R0k6t9x/pa+g8qzl5apQIPurJFozXhopks3iw3MX+jU= github.com/ipfs/go-path v0.0.8/go.mod h1:VpDkSBKQ9EFQOUgi54Tq/O/tGi8n1RfYNks13M3DEs8= -github.com/ipfs/go-path v0.0.9 h1:BIi831cNED8YnIlIKo9y1SI3u+E+FwQQD+rIIw8PwFA= -github.com/ipfs/go-path v0.0.9/go.mod h1:VpDkSBKQ9EFQOUgi54Tq/O/tGi8n1RfYNks13M3DEs8= github.com/ipfs/go-peertaskqueue v0.0.4/go.mod h1:03H8fhyeMfKNFWqzYEVyMbcPUeYrqP1MX6Kd+aN+rMQ= github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= @@ -474,8 +467,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lazyledger/go-ipfs v0.7.0-lazypatch h1:ejAoRPRg5GC1d2MlK2mp0mmNWQKwqZNADBovAaRcfV0= -github.com/lazyledger/go-ipfs v0.7.0-lazypatch/go.mod h1:WHRSOcXhNUz8Pp0G1CNnlI6oEl0ODIXz/0XcBLw+1Ys= +github.com/lazyledger/go-ipfs v0.8.0-lazypatch h1:8Dkw7Or6d0BmpFYFcxwgqWZ047BPGCsWtG7v9+H0ofk= +github.com/lazyledger/go-ipfs v0.8.0-lazypatch/go.mod h1:CE4cJkjUmwW5LwJP26KKEAZ11ZED0DxzSryfv5RMf6E= github.com/lazyledger/go-leopard v0.0.0-20200604113236-298f93361181 h1:mUeCGuCgjZVadW4CzA2dMBq7p2BqaoCfpnKjxMmSaSE= github.com/lazyledger/go-leopard v0.0.0-20200604113236-298f93361181/go.mod h1:v1o1CRihQ9i7hizx23KK4aR79lxA6VDUIzUCfDva0XQ= github.com/lazyledger/go-verifcid v0.0.1-lazypatch h1:jAVwUw+DhuCzx5IcYpFh6d6HWxRRz8nhJ3rQo+vlFAc= @@ -515,8 +508,6 @@ github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniV github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.12.0 h1:+xai9RQnQ9l5elFOKvp5wRyjyWisSwEx+6nU2+onpUA= github.com/libp2p/go-libp2p v0.12.0/go.mod h1:FpHZrfC1q7nA8jitvdjKBDF31hguaC676g/nT9PgQM0= -github.com/libp2p/go-libp2p v0.13.0 h1:tDdrXARSghmusdm0nf1U/4M8aj8Rr0V2IzQOXmbzQ3s= -github.com/libp2p/go-libp2p v0.13.0/go.mod h1:pM0beYdACRfHO1WcJlp65WXyG2A6NqYM+t2DTVAJxMo= github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052 h1:BM7aaOF7RpmNn9+9g6uTjGJ0cTzWr5j9i9IKeun2M8U= github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= github.com/libp2p/go-libp2p-autonat v0.0.6/go.mod h1:uZneLdOkZHro35xIhpbtTzLlgYturpu4J5+0cZK3MqE= @@ -563,8 +554,6 @@ github.com/libp2p/go-libp2p-core v0.6.1 h1:XS+Goh+QegCDojUZp00CaPMfiEADCrLjNZskW github.com/libp2p/go-libp2p-core v0.6.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.7.0 h1:4a0TMjrWNTZlNvcqxZmrMRDi/NQWrhwO2pkTuLSQ/IQ= github.com/libp2p/go-libp2p-core v0.7.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= -github.com/libp2p/go-libp2p-core v0.8.0 h1:5K3mT+64qDTKbV3yTdbMCzJ7O6wbNsavAEb8iqBvBcI= -github.com/libp2p/go-libp2p-core v0.8.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= github.com/libp2p/go-libp2p-crypto v0.0.2/go.mod h1:eETI5OUfBnvARGOHrJz2eWNyTUxEGZnBxMcbUjfIj4I= github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= @@ -598,9 +587,6 @@ github.com/libp2p/go-libp2p-mplex v0.2.2/go.mod h1:74S9eum0tVQdAfFiKxAyKzNdSuLqw github.com/libp2p/go-libp2p-mplex v0.2.3/go.mod h1:CK3p2+9qH9x+7ER/gWWDYJ3QW5ZxWDkm+dVvjfuG3ek= github.com/libp2p/go-libp2p-mplex v0.3.0 h1:CZyqqKP0BSGQyPLvpRQougbfXaaaJZdGgzhCpJNuNSk= github.com/libp2p/go-libp2p-mplex v0.3.0/go.mod h1:l9QWxRbbb5/hQMECEb908GbS9Sm2UAR2KFZKUJEynEs= -github.com/libp2p/go-libp2p-mplex v0.4.0/go.mod h1:yCyWJE2sc6TBTnFpjvLuEJgTSw/u+MamvzILKdX7asw= -github.com/libp2p/go-libp2p-mplex v0.4.1 h1:/pyhkP1nLwjG3OM+VuaNJkQT/Pqq73WzB3aDN3Fx1sc= -github.com/libp2p/go-libp2p-mplex v0.4.1/go.mod h1:cmy+3GfqfM1PceHTLL7zQzAAYaryDu6iPSC+CIb094g= github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCvVOiGzlcHmBbY= github.com/libp2p/go-libp2p-nat v0.0.5/go.mod h1:1qubaE5bTZMJE+E/uu2URroMbzdubFz1ChgiN79yKPE= github.com/libp2p/go-libp2p-nat v0.0.6 h1:wMWis3kYynCbHoyKLPBEMu4YRLltbm8Mk08HGSfvTkU= @@ -634,14 +620,10 @@ github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1 github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.4.0 h1:YNVRyXqBgv9i4RG88jzoTtkSOaSB45CqHkL29NNBZb4= github.com/libp2p/go-libp2p-pubsub v0.4.0/go.mod h1:izkeMLvz6Ht8yAISXjx60XUQZMq9ZMe5h2ih4dLIBIQ= -github.com/libp2p/go-libp2p-pubsub v0.4.1 h1:j4umIg5nyus+sqNfU+FWvb9aeYFQH/A+nDFhWj+8yy8= -github.com/libp2p/go-libp2p-pubsub v0.4.1/go.mod h1:izkeMLvz6Ht8yAISXjx60XUQZMq9ZMe5h2ih4dLIBIQ= github.com/libp2p/go-libp2p-pubsub-router v0.4.0 h1:KjzTLIOBCt0+/4wH6epTxD/Qu4Up/IyeKHlj9MhWRJI= github.com/libp2p/go-libp2p-pubsub-router v0.4.0/go.mod h1:hs0j0ugcBjMOMgJ6diOlZM2rZEId/w5Gg86E+ac4SmQ= github.com/libp2p/go-libp2p-quic-transport v0.9.3 h1:e6eLN55agvemeYzdJTzIylVbLD5PJr0RJANr9HM+FVc= github.com/libp2p/go-libp2p-quic-transport v0.9.3/go.mod h1:STiznVmrLMFPXCrkEbNna/VxgdxBatcK0SA+1FSdxnI= -github.com/libp2p/go-libp2p-quic-transport v0.10.0 h1:koDCbWD9CCHwcHZL3/WEvP2A+e/o5/W5L3QS/2SPMA0= -github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA= github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg= @@ -666,8 +648,6 @@ github.com/libp2p/go-libp2p-swarm v0.2.8/go.mod h1:JQKMGSth4SMqonruY0a8yjlPVIkb0 github.com/libp2p/go-libp2p-swarm v0.3.0/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk= github.com/libp2p/go-libp2p-swarm v0.3.1 h1:UTobu+oQHGdXTOGpZ4RefuVqYoJXcT0EBtSR74m2LkI= github.com/libp2p/go-libp2p-swarm v0.3.1/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk= -github.com/libp2p/go-libp2p-swarm v0.4.0 h1:hahq/ijRoeH6dgROOM8x7SeaKK5VgjjIr96vdrT+NUA= -github.com/libp2p/go-libp2p-swarm v0.4.0/go.mod h1:XVFcO52VoLoo0eitSxNQWYq4D6sydGOweTOAjJNraCw= github.com/libp2p/go-libp2p-testing v0.0.1/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= @@ -675,9 +655,8 @@ github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MB github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= github.com/libp2p/go-libp2p-testing v0.1.1/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= github.com/libp2p/go-libp2p-testing v0.1.2-0.20200422005655-8775583591d8/go.mod h1:Qy8sAncLKpwXtS2dSnDOP8ktexIAHKu+J+pnZOFZLTc= +github.com/libp2p/go-libp2p-testing v0.3.0 h1:ZiBYstPamsi7y6NJZebRudUzsYmVkt998hltyLqf8+g= github.com/libp2p/go-libp2p-testing v0.3.0/go.mod h1:efZkql4UZ7OVsEfaxNHZPzIehtsBXMrXnCfJIgDti5g= -github.com/libp2p/go-libp2p-testing v0.4.0 h1:PrwHRi0IGqOwVQWR3xzgigSlhlLfxgfXgkHxr77EghQ= -github.com/libp2p/go-libp2p-testing v0.4.0/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0= github.com/libp2p/go-libp2p-tls v0.1.3 h1:twKMhMu44jQO+HgQK9X8NHO5HkeJu2QbhLzLJpa8oNM= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= github.com/libp2p/go-libp2p-transport v0.0.1/go.mod h1:UzbUs9X+PHOSw7S3ZmeOxfnwaQY5vGDzZmKPod3N3tk= @@ -687,8 +666,6 @@ github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07q github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns= github.com/libp2p/go-libp2p-transport-upgrader v0.3.0 h1:q3ULhsknEQ34eVDhv4YwKS8iet69ffs9+Fir6a7weN4= github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o= -github.com/libp2p/go-libp2p-transport-upgrader v0.4.0 h1:xwj4h3hJdBrxqMOyMUjwscjoVst0AASTsKtZiTChoHI= -github.com/libp2p/go-libp2p-transport-upgrader v0.4.0/go.mod h1:J4ko0ObtZSmgn5BX5AmegP+dK3CSnU2lMCKsSq/EY0s= github.com/libp2p/go-libp2p-yamux v0.1.2/go.mod h1:xUoV/RmYkg6BW/qGxA9XJyg+HzXFYkeXbnhjmnYzKp8= github.com/libp2p/go-libp2p-yamux v0.1.3/go.mod h1:VGSQVrqkh6y4nm0189qqxMtvyBft44MOYYPpYKXiVt4= github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= @@ -700,9 +677,6 @@ github.com/libp2p/go-libp2p-yamux v0.2.8 h1:0s3ELSLu2O7hWKfX1YjzudBKCP0kZ+m9e2+0 github.com/libp2p/go-libp2p-yamux v0.2.8/go.mod h1:/t6tDqeuZf0INZMTgd0WxIRbtK2EzI2h7HbFm9eAKI4= github.com/libp2p/go-libp2p-yamux v0.4.0 h1:qunEZzWwwmfSBYTtSyd81PlD1TjB5uuWcGYHWVXLbUg= github.com/libp2p/go-libp2p-yamux v0.4.0/go.mod h1:+DWDjtFMzoAwYLVkNZftoucn7PelNoy5nm3tZ3/Zw30= -github.com/libp2p/go-libp2p-yamux v0.5.0/go.mod h1:AyR8k5EzyM2QN9Bbdg6X1SkVVuqLwTGf0L4DFq9g6po= -github.com/libp2p/go-libp2p-yamux v0.5.1 h1:sX4WQPHMhRxJE5UZTfjEuBvlQWXB5Bo3A2JK9ZJ9EM0= -github.com/libp2p/go-libp2p-yamux v0.5.1/go.mod h1:dowuvDu8CRWmr0iqySMiSxK+W0iL5cMVO9S94Y6gkv4= github.com/libp2p/go-maddr-filter v0.0.1/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M= @@ -715,8 +689,6 @@ github.com/libp2p/go-mplex v0.1.2 h1:qOg1s+WdGLlpkrczDqmhYzyk3vCfsQ8+RxRTQjOZWwI github.com/libp2p/go-mplex v0.1.2/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk= github.com/libp2p/go-mplex v0.2.0 h1:Ov/D+8oBlbRkjBs1R1Iua8hJ8cUfbdiW8EOdZuxcgaI= github.com/libp2p/go-mplex v0.2.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= -github.com/libp2p/go-mplex v0.3.0 h1:U1T+vmCYJaEoDJPV1aq31N56hS+lJgb397GsylNSgrU= -github.com/libp2p/go-mplex v0.3.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= @@ -765,8 +737,6 @@ github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzl github.com/libp2p/go-ws-transport v0.3.0/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk= github.com/libp2p/go-ws-transport v0.3.1 h1:ZX5rWB8nhRRJVaPO6tmkGI/Xx8XNboYX20PW5hXIscw= github.com/libp2p/go-ws-transport v0.3.1/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk= -github.com/libp2p/go-ws-transport v0.4.0 h1:9tvtQ9xbws6cA5LvqdE6Ne3vcmGB4f1z9SByggk4s0k= -github.com/libp2p/go-ws-transport v0.4.0/go.mod h1:EcIEKqf/7GDjth6ksuS/6p7R49V4CBY6/E7R/iyhYUA= github.com/libp2p/go-yamux v1.2.1/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= @@ -777,10 +747,6 @@ github.com/libp2p/go-yamux v1.3.7 h1:v40A1eSPJDIZwz2AvrV3cxpTZEGDP11QJbukmEhYyQI github.com/libp2p/go-yamux v1.3.7/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= github.com/libp2p/go-yamux v1.4.0 h1:7nqe0T95T2CWh40IdJ/tp8RMor4ubc9/wYZpB2a/Hx0= github.com/libp2p/go-yamux v1.4.0/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= -github.com/libp2p/go-yamux v1.4.1 h1:P1Fe9vF4th5JOxxgQvfbOHkrGqIZniTLf+ddhZp8YTI= -github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= -github.com/libp2p/go-yamux/v2 v2.0.0 h1:vSGhAy5u6iHBq11ZDcyHH4Blcf9xlBhT4WQDoOE90LU= -github.com/libp2p/go-yamux/v2 v2.0.0/go.mod h1:NVWira5+sVUIU6tu1JWvaRn1dRnG+cawOJiflsAM+7U= github.com/lucas-clemente/quic-go v0.19.3 h1:eCDQqvGBB+kCTkA0XrAFtNe81FMa0/fn4QSoeAbmiF4= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= diff --git a/p2p/ipld/plugin/nodes/nodes.go b/p2p/ipld/plugin/nodes/nodes.go index 9a14ee0c58..e465c2361b 100644 --- a/p2p/ipld/plugin/nodes/nodes.go +++ b/p2p/ipld/plugin/nodes/nodes.go @@ -3,6 +3,7 @@ package nodes import ( "bufio" "bytes" + "context" "crypto/sha256" "errors" "fmt" @@ -24,6 +25,10 @@ const ( // Nmt is the codec used for leaf and inner nodes of an Namespaced Merkle Tree. Nmt = 0x7700 + + // NmtCodecName is the name used during registry of the Nmt codec + NmtCodecName = "nmt-node" + // Sha256Namespace8Flagged is the multihash code used to hash blocks // that contain an NMT node (inner and leaf nodes). Sha256Namespace8Flagged = 0x7701 @@ -48,6 +53,11 @@ func init() { nmtHashSize, sumSha256Namespace8Flagged, ) + // this should already happen when the plugin is injected but it doesn't for some CI tests + format.DefaultBlockDecoder.Register(Nmt, NmtNodeParser) + // register the codecs in the global maps + cid.Codecs[NmtCodecName] = Nmt + cid.CodecToStr[Nmt] = NmtCodecName } func mustRegisterNamespacedCodec( @@ -82,6 +92,8 @@ func sumSha256Namespace8Flagged(data []byte, _length int) ([]byte, error) { return nmt.Sha256Namespace8FlaggedInner(data[1:]), nil } +var Plugins = []plugin.Plugin{&LazyLedgerPlugin{}} + var _ plugin.PluginIPLD = &LazyLedgerPlugin{} type LazyLedgerPlugin struct{} @@ -121,34 +133,15 @@ func (l LazyLedgerPlugin) Init(env *plugin.Environment) error { // the commandline, the ipld Nodes will rather be created together with the NMT // root instead of re-computing it here. func DataSquareRowOrColumnRawInputParser(r io.Reader, _mhType uint64, _mhLen int) ([]node.Node, error) { - // The extendedRowOrColumnSize is hardcode this here to avoid importing: - // https://github.com/lazyledger/lazyledger-core/blob/585566317e519bbb6d35d149b7e856c4c1e8657c/types/consts.go#L23 - const extendedRowOrColumnSize = 2 * 128 br := bufio.NewReader(r) - nodes := make([]node.Node, 0, extendedRowOrColumnSize) - nodeCollector := func(hash []byte, children ...[]byte) { - cid := mustCidFromNamespacedSha256(hash) - switch len(children) { - case 1: - prependNode(nmtLeafNode{ - cid: cid, - Data: children[0], - }, &nodes) - case 2: - prependNode(nmtNode{ - cid: cid, - l: children[0], - r: children[1], - }, &nodes) - default: - panic("expected a binary tree") - } - } + collector := newNodeCollector() + n := nmt.New( sha256.New(), nmt.NamespaceIDSize(namespaceSize), - nmt.NodeVisitor(nodeCollector), + nmt.NodeVisitor(collector.visit), ) + for { namespacedLeaf := make([]byte, shareSize+namespaceSize) if _, err := io.ReadFull(br, namespacedLeaf); err != nil { @@ -163,13 +156,91 @@ func DataSquareRowOrColumnRawInputParser(r io.Reader, _mhType uint64, _mhLen int } // to trigger the collection of nodes: _ = n.Root() - return nodes, nil + return collector.ipldNodes(), nil +} + +// nmtNodeCollector creates and collects ipld.Nodes if inserted into a nmt tree. +// It is mainly used for testing. +type nmtNodeCollector struct { + nodes []node.Node +} + +func newNodeCollector() *nmtNodeCollector { + // The extendedRowOrColumnSize is hardcode this here to avoid importing: + // https://github.com/lazyledger/lazyledger-core/blob/585566317e519bbb6d35d149b7e856c4c1e8657c/types/consts.go#L23 + const extendedRowOrColumnSize = 2 * 128 + return &nmtNodeCollector{nodes: make([]node.Node, 0, extendedRowOrColumnSize)} +} + +func (n nmtNodeCollector) ipldNodes() []node.Node { + return n.nodes +} + +func (n *nmtNodeCollector) visit(hash []byte, children ...[]byte) { + cid := mustCidFromNamespacedSha256(hash) + switch len(children) { + case 1: + n.nodes = prependNode(nmtLeafNode{ + cid: cid, + Data: children[0], + }, n.nodes) + case 2: + n.nodes = prependNode(nmtNode{ + cid: cid, + l: children[0], + r: children[1], + }, n.nodes) + default: + panic("expected a binary tree") + } +} + +func prependNode(newNode node.Node, nodes []node.Node) []node.Node { + nodes = append(nodes, node.Node(nil)) + copy(nodes[1:], nodes) + nodes[0] = newNode + return nodes +} + +// NmtNodeAdder adds ipld.Nodes to the underlying ipld.Batch if it is inserted +// into an nmt tree +type NmtNodeAdder struct { + batch *format.Batch + ctx context.Context +} + +// NewNmtNodeAdder returns a new NmtNodeAdder with the provided context and +// batch. Note that the context provided should have a timeout +func NewNmtNodeAdder(ctx context.Context, batch *format.Batch) *NmtNodeAdder { + return &NmtNodeAdder{ + batch: batch, + ctx: ctx, + } +} + +// Visit can be inserted into an nmt tree to create ipld.Nodes while computing the root +func (n *NmtNodeAdder) Visit(hash []byte, children ...[]byte) { + cid := mustCidFromNamespacedSha256(hash) + switch len(children) { + case 1: + n.batch.Add(n.ctx, nmtLeafNode{ + cid: cid, + Data: children[0], + }) + case 2: + n.batch.Add(n.ctx, nmtNode{ + cid: cid, + l: children[0], + r: children[1], + }) + default: + panic("expected a binary tree") + } } -func prependNode(newNode node.Node, nodes *[]node.Node) { - *nodes = append(*nodes, node.Node(nil)) - copy((*nodes)[1:], *nodes) - (*nodes)[0] = newNode +// Batch return the ipld.Batch originally provided to the NmtNodeAdder +func (n *NmtNodeAdder) Batch() *format.Batch { + return n.batch } func NmtNodeParser(block blocks.Block) (node.Node, error) { @@ -241,13 +312,13 @@ func (n nmtNode) Loggable() map[string]interface{} { func (n nmtNode) Resolve(path []string) (interface{}, []string, error) { switch path[0] { case "0": - left, err := cidFromNamespacedSha256(n.l) + left, err := CidFromNamespacedSha256(n.l) if err != nil { return nil, nil, err } return &node.Link{Cid: left}, path[1:], nil case "1": - right, err := cidFromNamespacedSha256(n.r) + right, err := CidFromNamespacedSha256(n.r) if err != nil { return nil, nil, err } @@ -372,7 +443,8 @@ func (l nmtLeafNode) Size() (uint64, error) { return 0, nil } -func cidFromNamespacedSha256(namespacedHash []byte) (cid.Cid, error) { +// CidFromNamespacedSha256 uses a hash from an nmt tree to create a cide +func CidFromNamespacedSha256(namespacedHash []byte) (cid.Cid, error) { if got, want := len(namespacedHash), nmtHashSize; got != want { return cid.Cid{}, fmt.Errorf("invalid namespaced hash length, got: %v, want: %v", got, want) } @@ -386,7 +458,7 @@ func cidFromNamespacedSha256(namespacedHash []byte) (cid.Cid, error) { // mustCidFromNamespacedSha256 is a wrapper around cidFromNamespacedSha256 that panics // in case of an error. Use with care and only in places where no error should occur. func mustCidFromNamespacedSha256(hash []byte) cid.Cid { - cid, err := cidFromNamespacedSha256(hash) + cid, err := CidFromNamespacedSha256(hash) if err != nil { panic( fmt.Sprintf("malformed hash: %s, codec: %v", diff --git a/p2p/ipld/plugin/nodes/nodes_test.go b/p2p/ipld/plugin/nodes/nodes_test.go index 6c7e302222..b045df3aae 100644 --- a/p2p/ipld/plugin/nodes/nodes_test.go +++ b/p2p/ipld/plugin/nodes/nodes_test.go @@ -40,9 +40,42 @@ func TestDataSquareRowOrColumnRawInputParserCidEqNmtRoot(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - - n := nmt.New(sha256.New()) buf := createByteBufFromRawData(t, tt.leafData) + + gotNodes, err := DataSquareRowOrColumnRawInputParser(buf, 0, 0) + if err != nil { + t.Errorf("DataSquareRowOrColumnRawInputParser() unexpected error = %v", err) + return + } + + multiHashOverhead := 4 + lastNodeCid := gotNodes[len(gotNodes)-1].Cid() + if gotHash, wantHash := lastNodeCid.Hash(), nmt.Sha256Namespace8FlaggedLeaf(tt.leafData[0]); !bytes.Equal(gotHash[multiHashOverhead:], wantHash) { + t.Errorf("first node's hash does not match the Cid\ngot: %v\nwant: %v", gotHash[multiHashOverhead:], wantHash) + } + nodePrefixOffset := 1 // leaf / inner node prefix is one byte + lastLeafNodeData := gotNodes[len(gotNodes)-1].RawData() + if gotData, wantData := lastLeafNodeData[nodePrefixOffset:], tt.leafData[0]; !bytes.Equal(gotData, wantData) { + t.Errorf("first node's data does not match the leaf's data\ngot: %v\nwant: %v", gotData, wantData) + } + }) + } +} + +func TestNodeCollector(t *testing.T) { + tests := []struct { + name string + leafData [][]byte + }{ + {"16 leaves", generateRandNamespacedRawData(16, namespaceSize, shareSize)}, + {"32 leaves", generateRandNamespacedRawData(32, namespaceSize, shareSize)}, + {"extended row", generateExtendedRow(t)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + collector := newNodeCollector() + n := nmt.New(sha256.New(), nmt.NamespaceIDSize(namespaceSize), nmt.NodeVisitor(collector.visit)) + for _, share := range tt.leafData { err := n.Push(share[:namespaceSize], share[namespaceSize:]) if err != nil { @@ -50,17 +83,22 @@ func TestDataSquareRowOrColumnRawInputParserCidEqNmtRoot(t *testing.T) { return } } - gotNodes, err := DataSquareRowOrColumnRawInputParser(buf, 0, 0) - if err != nil { - t.Errorf("DataSquareRowOrColumnRawInputParser() unexpected error = %v", err) - return - } + + rootDigest := n.Root() + + gotNodes := collector.ipldNodes() + rootNodeCid := gotNodes[0].Cid() multiHashOverhead := 4 lastNodeHash := rootNodeCid.Hash() - if got, want := lastNodeHash[multiHashOverhead:], n.Root().Bytes(); !bytes.Equal(got, want) { + if got, want := lastNodeHash[multiHashOverhead:], rootDigest.Bytes(); !bytes.Equal(got, want) { t.Errorf("hashes don't match\ngot: %v\nwant: %v", got, want) } + + if mustCidFromNamespacedSha256(rootDigest.Bytes()).String() != rootNodeCid.String() { + t.Error("root cid nod and hash not identical") + } + lastNodeCid := gotNodes[len(gotNodes)-1].Cid() if gotHash, wantHash := lastNodeCid.Hash(), nmt.Sha256Namespace8FlaggedLeaf(tt.leafData[0]); !bytes.Equal(gotHash[multiHashOverhead:], wantHash) { t.Errorf("first node's hash does not match the Cid\ngot: %v\nwant: %v", gotHash[multiHashOverhead:], wantHash) @@ -70,6 +108,20 @@ func TestDataSquareRowOrColumnRawInputParserCidEqNmtRoot(t *testing.T) { if gotData, wantData := lastLeafNodeData[nodePrefixOffset:], tt.leafData[0]; !bytes.Equal(gotData, wantData) { t.Errorf("first node's data does not match the leaf's data\ngot: %v\nwant: %v", gotData, wantData) } + + // ensure that every leaf was collected + hasMap := make(map[string]bool) + for _, node := range gotNodes { + hasMap[node.Cid().String()] = true + } + hasher := nmt.NewNmtHasher(sha256.New(), namespaceSize, true) + for _, leaf := range tt.leafData { + leafCid := mustCidFromNamespacedSha256(hasher.HashLeaf(leaf)) + _, has := hasMap[leafCid.String()] + if !has { + t.Errorf("leaf CID not found in collected nodes. missing: %s", leafCid.String()) + } + } }) } } @@ -100,7 +152,7 @@ func TestDagPutWithPlugin(t *testing.T) { } // convert Nmt tree root to CID and verify it matches the CID returned by DagPut treeRootBytes := n.Root().Bytes() - nmtCid, err := cidFromNamespacedSha256(treeRootBytes) + nmtCid, err := CidFromNamespacedSha256(treeRootBytes) if err != nil { t.Fatalf("cidFromNamespacedSha256() failed: %v", err) } diff --git a/test/e2e/runner/rpc.go b/test/e2e/runner/rpc.go index 8813929cff..52507b3a4b 100644 --- a/test/e2e/runner/rpc.go +++ b/test/e2e/runner/rpc.go @@ -95,7 +95,7 @@ func waitForAllNodes(testnet *e2e.Testnet, height int64, timeout time.Duration) if node.Mode == e2e.ModeSeed { continue } - status, err := waitForNode(node, height, 20*time.Second) + status, err := waitForNode(node, height, timeout) if err != nil { return 0, err } diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index 17c6282a4b..6e4599cc57 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -86,7 +86,8 @@ func Setup(testnet *e2e.Testnet) error { if err != nil { return err } - cfg.IPFS.ConfigRootPath = filepath.Join(nodeDir, "ipfs") + // todo(evan): the path should be a constant + cfg.IPFS.ConfigRootPath = filepath.Join(nodeDir, ".ipfs") config.WriteConfigFile(filepath.Join(nodeDir, "config", "config.toml"), cfg) // panics appCfg, err := MakeAppConfig(node) diff --git a/types/block.go b/types/block.go index c45a1b86b5..2ffefad8ec 100644 --- a/types/block.go +++ b/types/block.go @@ -2,6 +2,8 @@ package types import ( "bytes" + "context" + "crypto/sha256" "errors" "fmt" "math" @@ -10,6 +12,7 @@ import ( "github.com/gogo/protobuf/proto" gogotypes "github.com/gogo/protobuf/types" + format "github.com/ipfs/go-ipld-format" "github.com/lazyledger/nmt" "github.com/lazyledger/nmt/namespace" @@ -23,6 +26,7 @@ import ( tmmath "github.com/lazyledger/lazyledger-core/libs/math" "github.com/lazyledger/lazyledger-core/libs/protoio" tmsync "github.com/lazyledger/lazyledger-core/libs/sync" + "github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" tmversion "github.com/lazyledger/lazyledger-core/proto/tendermint/version" "github.com/lazyledger/lazyledger-core/version" @@ -203,49 +207,50 @@ func (b *Block) fillDataAvailabilityHeader() { b.DataHash = b.DataAvailabilityHeader.Hash() return } + // TODO(ismail): for better efficiency and a larger number shares // we should switch to the rsmt2d.LeopardFF16 codec: extendedDataSquare, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.RSGF8, rsmt2d.NewDefaultTree) if err != nil { panic(fmt.Sprintf("unexpected error: %v", err)) } - // compute roots: + + // record the widths squareWidth := extendedDataSquare.Width() - originalDataWidth := squareWidth / 2 + b.DataAvailabilityHeader = DataAvailabilityHeader{ RowsRoots: make([]namespace.IntervalDigest, squareWidth), ColumnRoots: make([]namespace.IntervalDigest, squareWidth), } - // compute row and column roots: - // TODO(ismail): refactor this to use rsmt2d lib directly instead - // depends on https://github.com/lazyledger/rsmt2d/issues/8 - for outerIdx := uint(0); outerIdx < squareWidth; outerIdx++ { - rowTree := nmt.New(newBaseHashFunc(), nmt.NamespaceIDSize(NamespaceSize)) - colTree := nmt.New(newBaseHashFunc(), nmt.NamespaceIDSize(NamespaceSize)) - for innerIdx := uint(0); innerIdx < squareWidth; innerIdx++ { - if outerIdx < originalDataWidth && innerIdx < originalDataWidth { - rowShare := namespacedShares[outerIdx*originalDataWidth+innerIdx] - colShare := namespacedShares[innerIdx*originalDataWidth+outerIdx] - mustPush(rowTree, rowShare.NamespaceID(), rowShare.Data()) - mustPush(colTree, colShare.NamespaceID(), colShare.Data()) - } else { - rowData := extendedDataSquare.Row(outerIdx) - colData := extendedDataSquare.Column(outerIdx) + // flatten the square and add namespaces + leaves := flattenNamespacedEDS(namespacedShares, extendedDataSquare) - parityCellFromRow := rowData[innerIdx] - parityCellFromCol := colData[innerIdx] - mustPush(rowTree, ParitySharesNamespaceID, parityCellFromRow) - mustPush(colTree, ParitySharesNamespaceID, parityCellFromCol) - } + // compute the roots for each col/row + for i, leafSet := range leaves { + commitment := nmtCommitment(leafSet) + + // add the commitment to the header + if uint(i) < squareWidth { + b.DataAvailabilityHeader.ColumnRoots[i] = commitment + } else { + b.DataAvailabilityHeader.RowsRoots[uint(i)-squareWidth] = commitment } - b.DataAvailabilityHeader.RowsRoots[outerIdx] = rowTree.Root() - b.DataAvailabilityHeader.ColumnRoots[outerIdx] = colTree.Root() } + // return the root hash of DA Header b.DataHash = b.DataAvailabilityHeader.Hash() } +// nmtcommitment generates the nmt root of some namespaced data +func nmtCommitment(namespacedData [][]byte) namespace.IntervalDigest { + tree := nmt.New(newBaseHashFunc(), nmt.NamespaceIDSize(NamespaceSize)) + for _, leaf := range namespacedData { + mustPush(tree, leaf[:NamespaceSize], leaf[NamespaceSize:]) + } + return tree.Root() +} + func mustPush(rowTree *nmt.NamespacedMerkleTree, id namespace.ID, data []byte) { if err := rowTree.Push(id, data); err != nil { panic( @@ -259,6 +264,107 @@ func mustPush(rowTree *nmt.NamespacedMerkleTree, id namespace.ID, data []byte) { } } +// PutBlock posts and pins erasured block data to IPFS using the provided +// ipld.NodeAdder. Note: the erasured data is currently recomputed +func (b *Block) PutBlock(ctx context.Context, nodeAdder format.NodeAdder) error { + if nodeAdder == nil { + return errors.New("no ipfs node adder provided") + } + + // recompute the shares + namespacedShares := b.Data.computeShares() + shares := namespacedShares.RawShares() + + // don't do anything if there is no data to put on IPFS + if len(shares) == 0 { + return nil + } + + // recompute the eds + eds, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.RSGF8, rsmt2d.NewDefaultTree) + if err != nil { + return fmt.Errorf("failure to recompute the extended data square: %w", err) + } + + // add namespaces to erasured shares and flatten the eds + leaves := flattenNamespacedEDS(namespacedShares, eds) + + // iterate through each set of col and row leaves + for _, leafSet := range leaves { + // create a batch per each leafSet + batchAdder := nodes.NewNmtNodeAdder(ctx, format.NewBatch(ctx, nodeAdder)) + tree := nmt.New(sha256.New(), nmt.NodeVisitor(batchAdder.Visit)) + for _, share := range leafSet { + err = tree.Push(share[:NamespaceSize], share[NamespaceSize:]) + if err != nil { + return err + } + } + + // compute the root in order to collect the ipld.Nodes + tree.Root() + + // commit the batch to ipfs + err = batchAdder.Batch().Commit() + if err != nil { + return err + } + } + + return nil +} + +// flattenNamespacedEDS returns a flattend extendedDataSquare with namespaces +// added to each share. NOTE: output is columns first then rows +func flattenNamespacedEDS(nss NamespacedShares, eds *rsmt2d.ExtendedDataSquare) [][][]byte { + squareWidth := eds.Width() + originalDataWidth := squareWidth / 2 + + if uint(len(nss)) != originalDataWidth*originalDataWidth { + panic( + fmt.Sprintf( + "unexpected numbers of namespaces: actual %d expected %d", + len(nss), + squareWidth/2, + ), + ) + } + + leaves := make([][][]byte, 2*squareWidth) + // this is adding the namespace back to Q1 shares and the parity ns to the + // rest of the quadrants and flattens the square + for i := uint(0); i < squareWidth; i++ { + rowLeaves := make([][]byte, squareWidth) + colLeaves := make([][]byte, squareWidth) + + for j := uint(0); j < squareWidth; j++ { + if i < originalDataWidth && j < originalDataWidth { + rowShare := nss[i*originalDataWidth+j] + colShare := nss[j*originalDataWidth+i] + rowLeaves[j] = append(rowShare.NamespaceID(), rowShare.Data()...) + colLeaves[j] = append(colShare.NamespaceID(), colShare.Data()...) + } else { + rowData := eds.Row(i) + colData := eds.Column(i) + parityCellFromRow := rowData[j] + parityCellFromCol := colData[j] + rowLeaves[j] = append(copyOfParityNamespaceID(), parityCellFromRow...) + colLeaves[j] = append(copyOfParityNamespaceID(), parityCellFromCol...) + } + } + leaves[i] = colLeaves + leaves[i+squareWidth] = rowLeaves + } + + return leaves +} + +func copyOfParityNamespaceID() []byte { + out := make([]byte, len(ParitySharesNamespaceID)) + copy(out, ParitySharesNamespaceID) + return out +} + // Hash computes and returns the block hash. // If the block is incomplete, block hash is nil for safety. func (b *Block) Hash() tmbytes.HexBytes { diff --git a/types/block_test.go b/types/block_test.go index 9f42cf4a62..48d9f3e612 100644 --- a/types/block_test.go +++ b/types/block_test.go @@ -3,15 +3,22 @@ package types import ( // it is ok to use math/rand here: we do not need a cryptographically secure random // number generator here and we can run the tests a bit faster + stdbytes "bytes" + "context" "crypto/rand" "encoding/hex" "math" "os" "reflect" + "sort" "testing" "time" gogotypes "github.com/gogo/protobuf/types" + coreapi "github.com/ipfs/go-ipfs/core/coreapi" + coremock "github.com/ipfs/go-ipfs/core/mock" + "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1306,3 +1313,111 @@ func TestCommit_ValidateBasic(t *testing.T) { }) } } + +func TestPutBlock(t *testing.T) { + ipfsNode, err := coremock.NewMockNode() + if err != nil { + t.Error(err) + } + + ipfsAPI, err := coreapi.NewCoreAPI(ipfsNode) + if err != nil { + t.Error(err) + } + + testCases := []struct { + name string + blockData Data + expectErr bool + errString string + }{ + {"no leaves", generateRandomData(0), false, ""}, + {"single leaf", generateRandomData(1), false, ""}, + {"16 leaves", generateRandomData(16), false, ""}, + {"max square size", generateRandomData(MaxSquareSize), false, ""}, + } + ctx := context.Background() + for _, tc := range testCases { + tc := tc + + block := &Block{Data: tc.blockData} + + t.Run(tc.name, func(t *testing.T) { + err = block.PutBlock(ctx, ipfsAPI.Dag().Pinning()) + if tc.expectErr { + require.Error(t, err) + require.Contains(t, err.Error(), tc.errString) + return + } + + require.NoError(t, err) + + timeoutCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + block.fillDataAvailabilityHeader() + tc.blockData.computeShares() + for _, rowRoot := range block.DataAvailabilityHeader.RowsRoots.Bytes() { + // recreate the cids using only the computed roots + cid, err := nodes.CidFromNamespacedSha256(rowRoot) + if err != nil { + t.Error(err) + } + + // check if cid was successfully pinned to IPFS + _, pinned, err := ipfsAPI.Pin().IsPinned(ctx, path.IpldPath(cid)) + if err != nil { + t.Error(err) + } + if !pinned { + t.Errorf("failure to pin cid %s to IPFS", cid.String()) + } + + // retrieve the data from IPFS + _, err = ipfsAPI.Dag().Get(timeoutCtx, cid) + if err != nil { + t.Errorf("Root not found: %s", cid.String()) + } + } + }) + } +} + +func generateRandomData(msgCount int) Data { + out := make([]Message, msgCount) + for i, msg := range generateRandNamespacedRawData(msgCount, NamespaceSize, ShareSize) { + out[i] = Message{NamespaceID: msg[:NamespaceSize], Data: msg[:NamespaceSize]} + } + return Data{ + Messages: Messages{MessagesList: out}, + } +} + +// this code is copy pasted from the plugin, and should likely be exported in the plugin instead +func generateRandNamespacedRawData(total int, nidSize int, leafSize int) [][]byte { + data := make([][]byte, total) + for i := 0; i < total; i++ { + nid := make([]byte, nidSize) + _, err := rand.Read(nid) + if err != nil { + panic(err) + } + data[i] = nid + } + + sortByteArrays(data) + for i := 0; i < total; i++ { + d := make([]byte, leafSize) + _, err := rand.Read(d) + if err != nil { + panic(err) + } + data[i] = append(data[i], d...) + } + + return data +} + +func sortByteArrays(src [][]byte) { + sort.Slice(src, func(i, j int) bool { return stdbytes.Compare(src[i], src[j]) < 0 }) +} diff --git a/types/consts.go b/types/consts.go index f5041e72b3..78872f9a1a 100644 --- a/types/consts.go +++ b/types/consts.go @@ -1,8 +1,9 @@ package types import ( + "crypto/sha256" + "github.com/lazyledger/nmt/namespace" - "golang.org/x/crypto/sha3" ) // This contains all constants of: @@ -32,5 +33,5 @@ var ( ParitySharesNamespaceID = namespace.ID{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF} // change accordingly if another hash.Hash should be used as a base hasher in the NMT: - newBaseHashFunc = sha3.New256 + newBaseHashFunc = sha256.New )