From 5f40b1c58653120c151d329b7a3fc91818e7d6ec Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 13:46:44 -0300 Subject: [PATCH 01/60] Creates wallet contract through Staker's DataPoster --- cmd/nitro/nitro.go | 2 +- staker/validatorwallet/contract.go | 90 ++++++++++++++++++++++-------- system_tests/fast_confirm_test.go | 8 +-- system_tests/staker_test.go | 4 +- 4 files changed, 74 insertions(+), 30 deletions(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 39f204980d..8c26b95a92 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -372,7 +372,7 @@ func mainImpl() int { log.Crit("error getting rollup addresses config", "err", err) } // #nosec G115 - addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true) + addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, nil) if err != nil { log.Crit("error creating validator wallet contract", "error", err, "address", l1TransactionOptsValidator.From.Hex()) } diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 77b403b669..89d4fef11d 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -26,8 +26,11 @@ import ( "github.com/offchainlabs/nitro/util/headerreader" ) -var validatorABI abi.ABI -var walletCreatedID common.Hash +var ( + validatorABI abi.ABI + validatorWalletCreatorABI abi.ABI + walletCreatedID common.Hash +) func init() { parsedValidator, err := abi.JSON(strings.NewReader(rollupgen.ValidatorWalletABI)) @@ -40,6 +43,7 @@ func init() { if err != nil { panic(err) } + validatorWalletCreatorABI = parsedValidatorWalletCreator walletCreatedID = parsedValidatorWalletCreator.Events["WalletCreated"].ID } @@ -179,6 +183,37 @@ func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), *v.Address(), data, gas, auth.Value) } +// Exposed for tests +func (v *Contract) CreateWalletContract( + ctx context.Context, + validatorWalletFactoryAddr common.Address, +) (*types.Transaction, error) { + var initialExecutorAllowedDests []common.Address + txData, err := validatorWalletCreatorABI.Pack("createWallet", initialExecutorAllowedDests) + if err != nil { + return nil, err + } + + auth, err := v.getAuth(ctx, nil) + if err != nil { + return nil, err + } + + gas, err := gasForTxData( + ctx, + v.l1Reader, + auth, + &validatorWalletFactoryAddr, + txData, + v.getExtraGas, + ) + if err != nil { + return nil, fmt.Errorf("getting gas for tx data: %w", err) + } + + return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) +} + func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) error { if v.con != nil { return nil @@ -194,7 +229,7 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err if err != nil { return err } - addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing) + addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing, v.CreateWalletContract) if err != nil { return err } @@ -295,25 +330,29 @@ func (v *Contract) ExecuteTransactions(ctx context.Context, builder *txbuilder.B return arbTx, nil } -func (v *Contract) estimateGas(ctx context.Context, value *big.Int, data []byte) (uint64, error) { - h, err := v.l1Reader.LastHeader(ctx) +func gasForTxData(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, to *common.Address, data []byte, getExtraGas func() uint64) (uint64, error) { + if auth.GasLimit != 0 { + return auth.GasLimit, nil + } + + h, err := l1Reader.LastHeader(ctx) if err != nil { return 0, fmt.Errorf("getting the last header: %w", err) } gasFeeCap := new(big.Int).Mul(h.BaseFee, big.NewInt(2)) gasFeeCap = arbmath.BigMax(gasFeeCap, arbmath.FloatToBig(params.GWei)) - gasTipCap, err := v.l1Reader.Client().SuggestGasTipCap(ctx) + gasTipCap, err := l1Reader.Client().SuggestGasTipCap(ctx) if err != nil { return 0, fmt.Errorf("getting suggested gas tip cap: %w", err) } gasFeeCap.Add(gasFeeCap, gasTipCap) - g, err := v.l1Reader.Client().EstimateGas( + g, err := l1Reader.Client().EstimateGas( ctx, ethereum.CallMsg{ - From: v.auth.From, - To: v.Address(), - Value: value, + From: auth.From, + To: to, + Value: auth.Value, Data: data, GasFeeCap: gasFeeCap, GasTipCap: gasTipCap, @@ -322,7 +361,11 @@ func (v *Contract) estimateGas(ctx context.Context, value *big.Int, data []byte) if err != nil { return 0, fmt.Errorf("estimating gas: %w", err) } - return g + v.getExtraGas(), nil + return g + getExtraGas(), nil +} + +func (v *Contract) gasForTxData(ctx context.Context, auth *bind.TransactOpts, data []byte) (uint64, error) { + return gasForTxData(ctx, v.l1Reader, auth, v.Address(), data, v.getExtraGas) } func (v *Contract) TimeoutChallenges(ctx context.Context, challenges []uint64) (*types.Transaction, error) { @@ -341,14 +384,6 @@ func (v *Contract) TimeoutChallenges(ctx context.Context, challenges []uint64) ( return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), *v.Address(), data, gas, auth.Value) } -// gasForTxData returns auth.GasLimit if it's nonzero, otherwise returns estimate. -func (v *Contract) gasForTxData(ctx context.Context, auth *bind.TransactOpts, data []byte) (uint64, error) { - if auth.GasLimit != 0 { - return auth.GasLimit, nil - } - return v.estimateGas(ctx, auth.Value, data) -} - func (v *Contract) L1Client() arbutil.L1Interface { return v.l1Reader.Client() } @@ -407,6 +442,7 @@ func GetValidatorWalletContract( transactAuth *bind.TransactOpts, l1Reader *headerreader.HeaderReader, createIfMissing bool, + createWalletFunc func(context.Context, common.Address) (*types.Transaction, error), ) (*common.Address, error) { client := l1Reader.Client() @@ -443,10 +479,18 @@ func GetValidatorWalletContract( return nil, nil } - var initialExecutorAllowedDests []common.Address - tx, err := walletCreator.CreateWallet(transactAuth, initialExecutorAllowedDests) - if err != nil { - return nil, err + var tx *types.Transaction + if createWalletFunc == nil { + var initialExecutorAllowedDests []common.Address + tx, err = walletCreator.CreateWallet(transactAuth, initialExecutorAllowedDests) + if err != nil { + return nil, err + } + } else { + tx, err = createWalletFunc(ctx, validatorWalletFactoryAddr) + if err != nil { + return nil, err + } } receipt, err := l1Reader.WaitForTxApproval(ctx, tx) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index 4a679e5077..79b77524a2 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -79,10 +79,10 @@ func TestFastConfirmation(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "Validator", balance, builder.L1Info) l1auth := builder.L1Info.GetDefaultTransactOpts("Validator", ctx) - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true) + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) Require(t, err) valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) Require(t, err) if valWalletAddr == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) @@ -278,10 +278,10 @@ func TestFastConfirmationWithSafe(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index 03c9fd3628..13530adfd5 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -132,10 +132,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) From 26bd0bd59e0ed1d0948c197937fd601809bf5342 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 15:23:46 -0300 Subject: [PATCH 02/60] Staker tests creates wallet contract through Staker's DataPoster --- system_tests/staker_test.go | 42 +++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index 13530adfd5..ea34eb8a96 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -37,6 +37,7 @@ import ( "github.com/offchainlabs/nitro/util" "github.com/offchainlabs/nitro/util/arbmath" "github.com/offchainlabs/nitro/util/colors" + "github.com/offchainlabs/nitro/util/testhelpers" "github.com/offchainlabs/nitro/validator/valnode" ) @@ -57,7 +58,8 @@ func makeBackgroundTxs(ctx context.Context, builder *NodeBuilder) error { } func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) { - t.Parallel() + logHandler := testhelpers.InitTestLog(t, log.LvlTrace) + ctx, cancelCtx := context.WithCancel(context.Background()) defer cancelCtx() srv := externalsignertest.NewServer(t) @@ -132,15 +134,6 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - if valWalletAddrA == *valWalletAddrCheck { - Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) - } - rollup, err := rollupgen.NewRollupAdminLogic(l2nodeA.DeployInfo.Rollup, builder.L1.Client) Require(t, err) @@ -149,16 +142,9 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) rollupABI, err := abi.JSON(strings.NewReader(rollupgen.RollupAdminLogicABI)) Require(t, err, "unable to parse rollup ABI") - setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address}, []bool{true, true, true}) - Require(t, err, "unable to generate setValidator calldata") - tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) - Require(t, err, "unable to set validators") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - setMinAssertPeriodCalldata, err := rollupABI.Pack("setMinimumAssertionPeriod", big.NewInt(1)) Require(t, err, "unable to generate setMinimumAssertionPeriod calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) + tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) Require(t, err, "unable to set minimum assertion period") _, err = builder.L1.EnsureTxSucceeded(tx) Require(t, err) @@ -190,6 +176,22 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) valConfigA.Strategy = "MakeNodes" } + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + Require(t, err) + valWalletAddrA := *valWalletAddrAPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + Require(t, err) + if valWalletAddrA == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) + } + + setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address}, []bool{true, true, true}) + Require(t, err, "unable to generate setValidator calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) + Require(t, err, "unable to set validators") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + _, valStack := createTestValidationNode(t, ctx, &valnode.TestValidationConfig) blockValidatorConfig := staker.TestBlockValidatorConfig @@ -464,6 +466,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) if !stakerBWasStaked { Fatal(t, "staker B was never staked") } + + if logHandler.WasLogged("data poster expected next transaction to have nonce \\d+ but was requested to post transaction with nonce \\d+") { + Fatal(t, "Staker's DataPoster inferred nonce incorrectly") + } } func TestStakersCooperative(t *testing.T) { From c3ee555bc6c3679aed7c1347fc8c771d3d02caca Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 15:24:02 -0300 Subject: [PATCH 03/60] TestGetValidatorWalletContractWithoutDataPoster --- system_tests/staker_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index ea34eb8a96..3fdf82ca74 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -475,3 +475,29 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) func TestStakersCooperative(t *testing.T) { stakerTestImpl(t, false, false) } + +func TestGetValidatorWalletContractWithoutDataPoster(t *testing.T) { + t.Parallel() + + ctx, cancelCtx := context.WithCancel(context.Background()) + defer cancelCtx() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + cleanup := builder.Build(t) + defer cleanup() + + balance := big.NewInt(params.Ether) + balance.Mul(balance, big.NewInt(100)) + builder.L1Info.GenerateAccount("ValidatorA") + builder.L1.TransferBalance(t, "Faucet", "ValidatorA", balance, builder.L1Info) + l1auth := builder.L1Info.GetDefaultTransactOpts("ValidatorA", ctx) + + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + Require(t, err) + valWalletAddrA := *valWalletAddrAPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + Require(t, err) + if valWalletAddrA == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) + } +} From 2fde94e651c991df1e4a3efe858d758bd784e4e4 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 15:52:10 -0300 Subject: [PATCH 04/60] Adds comment related to v.CreateWalletContract --- staker/validatorwallet/contract.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 89d4fef11d..495e796cd2 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -229,6 +229,10 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err if err != nil { return err } + + // By passing v.CreateWalletContract as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. + // DataPoster keeps in its internal state information related to the transactions sent through it, which is used to infer the expected nonce in a transaction for example. + // If a transaction is sent using the Staker's DataPoster key, but not through the Staker's DataPoster object, DataPoster's internal state will be outdated, which can compromise the expected nonce inference. addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing, v.CreateWalletContract) if err != nil { return err From 04e3f12507ceff95812d0b72901d84ed643954a1 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 16:16:12 -0300 Subject: [PATCH 05/60] Fast confirmation test creates wallet contract through Staker's DataPoster --- system_tests/fast_confirm_test.go | 48 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index 79b77524a2..ae624be1e9 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -79,15 +79,6 @@ func TestFastConfirmation(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "Validator", balance, builder.L1Info) l1auth := builder.L1Info.GetDefaultTransactOpts("Validator", ctx) - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) - Require(t, err) - valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) - Require(t, err) - if valWalletAddr == *valWalletAddrCheck { - Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) - } - rollup, err := rollupgen.NewRollupAdminLogic(l2node.DeployInfo.Rollup, builder.L1.Client) Require(t, err) @@ -96,27 +87,13 @@ func TestFastConfirmation(t *testing.T) { rollupABI, err := abi.JSON(strings.NewReader(rollupgen.RollupAdminLogicABI)) Require(t, err, "unable to parse rollup ABI") - setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddr, srv.Address}, []bool{true, true}) - Require(t, err, "unable to generate setValidator calldata") - tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setValidatorCalldata) - Require(t, err, "unable to set validators") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - setMinAssertPeriodCalldata, err := rollupABI.Pack("setMinimumAssertionPeriod", big.NewInt(1)) Require(t, err, "unable to generate setMinimumAssertionPeriod calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setMinAssertPeriodCalldata) + tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setMinAssertPeriodCalldata) Require(t, err, "unable to set minimum assertion period") _, err = builder.L1.EnsureTxSucceeded(tx) Require(t, err) - setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", valWalletAddr) - Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) - Require(t, err, "unable to set anytrust fast confirmer") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - valConfig := staker.TestL1ValidatorConfig valConfig.EnableFastConfirmation = true parentChainID, err := builder.L1.Client.ChainID(ctx) @@ -138,6 +115,29 @@ func TestFastConfirmation(t *testing.T) { Require(t, err) valConfig.Strategy = "MakeNodes" + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + Require(t, err) + valWalletAddr := *valWalletAddrPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + Require(t, err) + if valWalletAddr == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) + } + + setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddr, srv.Address}, []bool{true, true}) + Require(t, err, "unable to generate setValidator calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setValidatorCalldata) + Require(t, err, "unable to set validators") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + + setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", valWalletAddr) + Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) + Require(t, err, "unable to set anytrust fast confirmer") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + _, valStack := createTestValidationNode(t, ctx, &valnode.TestValidationConfig) blockValidatorConfig := staker.TestBlockValidatorConfig From b933b74725e2dd34df1f472ab22fbdfcff277698 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 22 Aug 2024 22:19:16 +0200 Subject: [PATCH 06/60] pass targets to rawdb.WrapDatabaseWithWasm, change rawdb.Target to ethdb.WasmTarget --- arbnode/inbox_test.go | 4 +- arbos/programs/native.go | 11 ++--- arbos/programs/wasmstorehelper.go | 3 +- cmd/nitro/init.go | 6 +-- cmd/nitro/init_test.go | 4 ++ cmd/nitro/nitro.go | 2 +- execution/gethexec/executionengine.go | 35 ++++++++++------ execution/gethexec/node.go | 50 +++++++++++++++++++++-- go-ethereum | 2 +- staker/block_validator.go | 2 +- staker/challenge_manager.go | 3 +- staker/stateless_block_validator.go | 5 +-- system_tests/common_test.go | 47 +++++++++++---------- system_tests/forwarder_test.go | 4 +- system_tests/recreatestate_rpc_test.go | 18 ++++---- system_tests/retryable_test.go | 2 +- system_tests/validation_mock_test.go | 6 +-- validator/client/redis/producer.go | 9 ++-- validator/client/validation_client.go | 13 +++--- validator/interface.go | 4 +- validator/server_api/json.go | 8 ++-- validator/server_arb/validator_spawner.go | 5 ++- validator/server_jit/spawner.go | 5 ++- validator/validation_entry.go | 4 +- validator/valnode/validation_api.go | 4 +- 25 files changed, 161 insertions(+), 95 deletions(-) diff --git a/arbnode/inbox_test.go b/arbnode/inbox_test.go index 1c46c593b9..d579b7c278 100644 --- a/arbnode/inbox_test.go +++ b/arbnode/inbox_test.go @@ -72,7 +72,9 @@ func NewTransactionStreamerForTest(t *testing.T, ownerAddress common.Address) (* if err != nil { Fail(t, err) } - if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, &gethexec.DefaultStylusTargetConfig); err != nil { + stylusTargetConfig := &gethexec.DefaultStylusTargetConfig + Require(t, stylusTargetConfig.Validate()) // pre-processes config (i.a. parses wasmTargets) + if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, stylusTargetConfig); err != nil { Fail(t, err) } execSeq := &execClientWrapper{execEngine, t} diff --git a/arbos/programs/native.go b/arbos/programs/native.go index fd3dec25a0..36e200af4d 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/arbos/burn" "github.com/offchainlabs/nitro/arbos/util" @@ -71,7 +72,7 @@ func activateProgramInternal( version uint16, debug bool, gasLeft *uint64, -) (*activationInfo, map[rawdb.Target][]byte, error) { +) (*activationInfo, map[ethdb.WasmTarget][]byte, error) { output := &rustBytes{} moduleHash := &bytes32{} stylusData := &C.StylusData{} @@ -111,7 +112,7 @@ func activateProgramInternal( if status_asm != 0 { return nil, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm)) } - asmMap := map[rawdb.Target][]byte{ + asmMap := map[ethdb.WasmTarget][]byte{ rawdb.TargetWavm: module, target: asm, } @@ -171,7 +172,7 @@ func getLocalAsm(statedb vm.StateDB, moduleHash common.Hash, addressForLogging c } asm, exists := asmMap[localTarget] if !exists { - var availableTargets []rawdb.Target + var availableTargets []ethdb.WasmTarget for target := range asmMap { availableTargets = append(availableTargets, target) } @@ -203,7 +204,7 @@ func callProgram( } if db, ok := db.(*state.StateDB); ok { - targets := []rawdb.Target{ + targets := []ethdb.WasmTarget{ rawdb.TargetWavm, rawdb.LocalTarget(), } @@ -291,7 +292,7 @@ func ResizeWasmLruCache(size uint32) { const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon" const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2" -func SetTarget(name rawdb.Target, description string, native bool) error { +func SetTarget(name ethdb.WasmTarget, description string, native bool) error { output := &rustBytes{} status := userStatus(C.stylus_target_set( goSlice([]byte(name)), diff --git a/arbos/programs/wasmstorehelper.go b/arbos/programs/wasmstorehelper.go index 4f82d80282..95daaf544a 100644 --- a/arbos/programs/wasmstorehelper.go +++ b/arbos/programs/wasmstorehelper.go @@ -12,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -44,7 +45,7 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash } // If already in wasm store then return early - _, err = statedb.TryGetActivatedAsmMap([]rawdb.Target{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash) + _, err = statedb.TryGetActivatedAsmMap([]ethdb.WasmTarget{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash) if err == nil { return nil } diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index d9ae0df3b0..14d9a52203 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -475,7 +475,7 @@ func validateOrUpgradeWasmStoreSchemaVersion(db ethdb.Database) error { return nil } -func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { +func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, targetConfig *gethexec.StylusTargetConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { if !config.Init.Force { if readOnlyDb, err := stack.OpenDatabaseWithFreezerWithExtraOptions("l2chaindata", 0, 0, config.Persistent.Ancient, "l2chaindata/", true, persistentConfig.Pebble.ExtraOptions("l2chaindata")); err == nil { if chainConfig := gethexec.TryReadStoredChainConfig(readOnlyDb); chainConfig != nil { @@ -500,7 +500,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err := dbutil.UnfinishedConversionCheck(wasmDb); err != nil { return nil, nil, fmt.Errorf("wasm unfinished database conversion check error: %w", err) } - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets()) _, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb) if err != nil { return nil, nil, err @@ -617,7 +617,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err := validateOrUpgradeWasmStoreSchemaVersion(wasmDb); err != nil { return nil, nil, err } - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets()) _, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb) if err != nil { return nil, nil, err diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index b2773ed861..af2242af78 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -377,6 +377,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -393,6 +394,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -410,6 +412,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -545,6 +548,7 @@ func TestOpenInitializeChainDbEmptyInit(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index a052c146d1..052755d8a3 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -481,7 +481,7 @@ func mainImpl() int { } } - chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Persistent, l1Client, rollupAddrs) + chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Execution.StylusTarget, &nodeConfig.Persistent, l1Client, rollupAddrs) if l2BlockChain != nil { deferFuncs = append(deferFuncs, func() { l2BlockChain.Stop() }) } diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 806355b2c6..4e0217f527 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -154,20 +154,31 @@ func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusT if rustCacheSize != 0 { programs.ResizeWasmLruCache(rustCacheSize) } - var effectiveStylusTarget string - target := rawdb.LocalTarget() - switch target { - case rawdb.TargetArm64: - effectiveStylusTarget = targetConfig.Arm64 - case rawdb.TargetAmd64: - effectiveStylusTarget = targetConfig.Amd64 - case rawdb.TargetHost: - effectiveStylusTarget = targetConfig.Host + + localTarget := rawdb.LocalTarget() + targets := targetConfig.WasmTargets() + var nativeSet bool + for _, target := range targets { + var effectiveStylusTarget string + switch target { + case rawdb.TargetArm64: + effectiveStylusTarget = targetConfig.Arm64 + case rawdb.TargetAmd64: + effectiveStylusTarget = targetConfig.Amd64 + case rawdb.TargetHost: + effectiveStylusTarget = targetConfig.Host + } + isNative := target == localTarget + err := programs.SetTarget(target, effectiveStylusTarget, isNative) + if err != nil { + return fmt.Errorf("Failed to set stylus target: %w", err) + } + nativeSet = nativeSet || isNative } - err := programs.SetTarget(target, effectiveStylusTarget, true) - if err != nil { - return fmt.Errorf("Failed to set stylus target: %w", err) + if !nativeSet { + return fmt.Errorf("local target %v missing in list of archs %v", localTarget, targets) } + return nil } diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index b864332e83..f519cce6ed 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/arbitrum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/filters" @@ -29,21 +30,61 @@ import ( ) type StylusTargetConfig struct { - Arm64 string `koanf:"arm64"` - Amd64 string `koanf:"amd64"` - Host string `koanf:"host"` + Arm64 string `koanf:"arm64"` + Amd64 string `koanf:"amd64"` + Host string `koanf:"host"` + Archs []string `koanf:"archs"` + + wasmTargets []ethdb.WasmTarget +} + +func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { + return c.wasmTargets +} + +func (c *StylusTargetConfig) Validate() error { + localTarget := rawdb.LocalTarget() + var nativeFound bool + targets := make([]ethdb.WasmTarget, 0, len(c.Archs)) + for _, arch := range c.Archs { + target := ethdb.WasmTarget(arch) + if !rawdb.IsSupportedWasmTarget(target) { + return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) + } + var alreadyInList bool + for _, t := range targets { + if target == t { + alreadyInList = true + break + } + } + if alreadyInList { + continue + } + if target == localTarget { + nativeFound = true + } + targets = append(targets, target) + } + if !nativeFound { + return fmt.Errorf("native target not found in archs list, native target: %v, archs: %v", localTarget, c.Archs) + } + c.wasmTargets = targets + return nil } var DefaultStylusTargetConfig = StylusTargetConfig{ Arm64: programs.DefaultTargetDescriptionArm, Amd64: programs.DefaultTargetDescriptionX86, Host: "", + Archs: []string{string(rawdb.TargetWavm), string(rawdb.LocalTarget())}, } func StylusTargetConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".arm64", DefaultStylusTargetConfig.Arm64, "stylus programs compilation target for arm64 linux") f.String(prefix+".amd64", DefaultStylusTargetConfig.Amd64, "stylus programs compilation target for amd64 linux") f.String(prefix+".host", DefaultStylusTargetConfig.Host, "stylus programs compilation target for system other than 64-bit ARM or 64-bit x86") + f.StringSlice(prefix+".archs", DefaultStylusTargetConfig.Archs, fmt.Sprintf("Comma separated architectures list to compile stylus program to and cache in wasm store (available targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)) } type Config struct { @@ -82,6 +123,9 @@ func (c *Config) Validate() error { if c.forwardingTarget != "" && c.Sequencer.Enable { return errors.New("ForwardingTarget set and sequencer enabled") } + if err := c.StylusTarget.Validate(); err != nil { + return err + } return nil } diff --git a/go-ethereum b/go-ethereum index 575062fad7..cb03c12f7f 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 575062fad7ff4db9d7c235f49472f658be29e2fe +Subproject commit cb03c12f7f02d47344d2d089eef748e92b99c95e diff --git a/staker/block_validator.go b/staker/block_validator.go index 8f5724beac..5cfce84c99 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -499,7 +499,7 @@ func (v *BlockValidator) sendRecord(s *validationStatus) error { //nolint:gosec func (v *BlockValidator) writeToFile(validationEntry *validationEntry, moduleRoot common.Hash) error { - input, err := validationEntry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + input, err := validationEntry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return err } diff --git a/staker/challenge_manager.go b/staker/challenge_manager.go index b1421d7e41..978a876fd0 100644 --- a/staker/challenge_manager.go +++ b/staker/challenge_manager.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbutil" @@ -468,7 +469,7 @@ func (m *ChallengeManager) createExecutionBackend(ctx context.Context, step uint if err != nil { return fmt.Errorf("error creating validation entry for challenge %v msg %v for execution challenge: %w", m.challengeIndex, initialCount, err) } - input, err := entry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + input, err := entry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return fmt.Errorf("error getting validation entry input of challenge %v msg %v: %w", m.challengeIndex, initialCount, err) } diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index d5eeb8eb69..f54ec8b58c 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -12,7 +12,6 @@ import ( "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -134,7 +133,7 @@ type validationEntry struct { DelayedMsg []byte } -func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.ValidationInput, error) { +func (e *validationEntry) ToInput(stylusArchs []ethdb.WasmTarget) (*validator.ValidationInput, error) { if e.Stage != Ready { return nil, errors.New("cannot create input from non-ready entry") } @@ -143,7 +142,7 @@ func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.Valida HasDelayedMsg: e.HasDelayedMsg, DelayedMsgNr: e.DelayedMsgNr, Preimages: e.Preimages, - UserWasms: make(map[rawdb.Target]map[common.Hash][]byte, len(e.UserWasms)), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash][]byte, len(e.UserWasms)), BatchInfo: e.BatchInfo, DelayedMsg: e.DelayedMsg, StartState: e.Start, diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 6e7375a921..392f6de805 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -197,7 +197,7 @@ var TestSequencerConfig = gethexec.SequencerConfig{ EnableProfiling: false, } -func ExecConfigDefaultNonSequencerTest() *gethexec.Config { +func ExecConfigDefaultNonSequencerTest(t *testing.T) *gethexec.Config { config := gethexec.ConfigDefault config.Caching = TestCachingConfig config.ParentChainReader = headerreader.TestConfig @@ -206,12 +206,12 @@ func ExecConfigDefaultNonSequencerTest() *gethexec.Config { config.ForwardingTarget = "null" config.TxPreChecker.Strictness = gethexec.TxPreCheckerStrictnessNone - _ = config.Validate() + Require(t, config.Validate()) return &config } -func ExecConfigDefaultTest() *gethexec.Config { +func ExecConfigDefaultTest(t *testing.T) *gethexec.Config { config := gethexec.ConfigDefault config.Caching = TestCachingConfig config.Sequencer = TestSequencerConfig @@ -219,7 +219,7 @@ func ExecConfigDefaultTest() *gethexec.Config { config.ForwardingTarget = "null" config.TxPreChecker.Strictness = gethexec.TxPreCheckerStrictnessNone - _ = config.Validate() + Require(t, config.Validate()) return &config } @@ -272,7 +272,7 @@ func (b *NodeBuilder) DefaultConfig(t *testing.T, withL1 bool) *NodeBuilder { b.l2StackConfig = testhelpers.CreateStackConfigForTest(b.dataDir) cp := valnode.TestValidationConfig b.valnodeConfig = &cp - b.execConfig = ExecConfigDefaultTest() + b.execConfig = ExecConfigDefaultTest(t) return b } @@ -310,7 +310,7 @@ func (b *NodeBuilder) CheckConfig(t *testing.T) { b.nodeConfig = arbnode.ConfigDefaultL1Test() } if b.execConfig == nil { - b.execConfig = ExecConfigDefaultTest() + b.execConfig = ExecConfigDefaultTest(t) } if b.L1Info == nil { b.L1Info = NewL1TestInfo(t) @@ -346,7 +346,7 @@ func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() { var l2arbDb ethdb.Database var l2blockchain *core.BlockChain _, b.L2.Stack, l2chainDb, l2arbDb, l2blockchain = createL2BlockChainWithStackConfig( - t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, &b.execConfig.Caching) + t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) var sequencerTxOptsPtr *bind.TransactOpts var dataSigner signature.DataSignerFunc @@ -409,7 +409,7 @@ func (b *NodeBuilder) BuildL2(t *testing.T) func() { var arbDb ethdb.Database var blockchain *core.BlockChain b.L2Info, b.L2.Stack, chainDb, arbDb, blockchain = createL2BlockChain( - t, b.L2Info, b.dataDir, b.chainConfig, &b.execConfig.Caching) + t, b.L2Info, b.dataDir, b.chainConfig, b.execConfig) Require(t, b.execConfig.Validate()) execConfig := b.execConfig @@ -460,7 +460,7 @@ func (b *NodeBuilder) RestartL2Node(t *testing.T) { } b.L2.cleanup() - l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, &b.execConfig.Caching) + l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) execConfigFetcher := func() *gethexec.Config { return b.execConfig } execNode, err := gethexec.CreateExecutionNode(b.ctx, stack, chainDb, blockchain, nil, execConfigFetcher) @@ -1047,30 +1047,32 @@ func DeployOnTestL1( } func createL2BlockChain( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, cacheConfig *gethexec.CachingConfig, + t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { - return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, cacheConfig) + return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, execConfig) } func createL2BlockChainWithStackConfig( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, cacheConfig *gethexec.CachingConfig, + t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { if l2info == nil { l2info = NewArbTestInfo(t, chainConfig.ChainID) } - var stack *node.Node - var err error if stackConfig == nil { stackConfig = testhelpers.CreateStackConfigForTest(dataDir) } - stack, err = node.New(stackConfig) + if execConfig == nil { + execConfig = ExecConfigDefaultTest(t) + } + + stack, err := node.New(stackConfig) Require(t, err) chainData, err := stack.OpenDatabaseWithExtraOptions("l2chaindata", 0, 0, "l2chaindata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("l2chaindata")) Require(t, err) wasmData, err := stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) arbDb, err := stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) @@ -1085,11 +1087,8 @@ func createL2BlockChainWithStackConfig( SerializedChainConfig: serializedChainConfig, } } - var coreCacheConfig *core.CacheConfig - if cacheConfig != nil { - coreCacheConfig = gethexec.DefaultCacheConfigFor(stack, cacheConfig) - } - blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest().TxLookupLimit, 0) + coreCacheConfig := gethexec.DefaultCacheConfigFor(stack, &execConfig.Caching) + blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) return l2info, stack, chainDb, arbDb, blockchain @@ -1149,7 +1148,7 @@ func Create2ndNodeWithConfig( nodeConfig = arbnode.ConfigDefaultL1NonSequencerTest() } if execConfig == nil { - execConfig = ExecConfigDefaultNonSequencerTest() + execConfig = ExecConfigDefaultNonSequencerTest(t) } feedErrChan := make(chan error, 10) l1rpcClient := l1stack.Attach() @@ -1165,7 +1164,7 @@ func Create2ndNodeWithConfig( Require(t, err) wasmData, err := l2stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0) + l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) l2arbDb, err := l2stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) @@ -1179,7 +1178,7 @@ func Create2ndNodeWithConfig( chainConfig := firstExec.ArbInterface.BlockChain().Config() coreCacheConfig := gethexec.DefaultCacheConfigFor(l2stack, &execConfig.Caching) - l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest().TxLookupLimit, 0) + l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) AddValNodeIfNeeded(t, ctx, nodeConfig, true, "", valnodeConfig.Wasm.RootPath) diff --git a/system_tests/forwarder_test.go b/system_tests/forwarder_test.go index 9fe419593e..1b083d2089 100644 --- a/system_tests/forwarder_test.go +++ b/system_tests/forwarder_test.go @@ -38,7 +38,7 @@ func TestStaticForwarder(t *testing.T) { clientA := builder.L2.Client nodeConfigB := arbnode.ConfigDefaultL1Test() - execConfigB := ExecConfigDefaultTest() + execConfigB := ExecConfigDefaultTest(t) execConfigB.Sequencer.Enable = false nodeConfigB.Sequencer = false nodeConfigB.DelayedSequencer.Enable = false @@ -109,7 +109,7 @@ func createForwardingNode(t *testing.T, builder *NodeBuilder, ipcPath string, re nodeConfig.Sequencer = false nodeConfig.DelayedSequencer.Enable = false nodeConfig.BatchPoster.Enable = false - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Sequencer.Enable = false execConfig.Forwarder.RedisUrl = redisUrl execConfig.ForwardingTarget = fallbackPath diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index 09d53669ee..d5a6311cb4 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -95,7 +95,7 @@ func removeStatesFromDb(t *testing.T, bc *core.BlockChain, db ethdb.Database, fr func TestRecreateStateForRPCNoDepthLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -133,7 +133,7 @@ func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() depthGasLimit := int64(256 * util.NormalizeL2GasForL1GasInitial(800_000, params.GWei)) - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = depthGasLimit execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -170,7 +170,7 @@ func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { func TestRecreateStateForRPCDepthLimitExceeded(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = int64(200) execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -207,7 +207,7 @@ func TestRecreateStateForRPCMissingBlockParent(t *testing.T) { var headerCacheLimit uint64 = 512 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -255,7 +255,7 @@ func TestRecreateStateForRPCBeyondGenesis(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -293,7 +293,7 @@ func TestRecreateStateForRPCBlockNotFoundWhileRecreating(t *testing.T) { var blockCacheLimit uint64 = 256 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -341,7 +341,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = maxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -480,7 +480,7 @@ func TestSkippingSavingStateAndRecreatingAfterRestart(t *testing.T) { func TestGettingStateForRPCFullNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Caching.SnapshotCache = 0 // disable snapshots execConfig.Caching.BlockAge = 0 // use only Caching.BlockCount to keep only last N blocks in dirties cache, no matter how new they are execConfig.Sequencer.MaxBlockSpeed = 0 @@ -522,7 +522,7 @@ func TestGettingStateForRPCFullNode(t *testing.T) { func TestGettingStateForRPCHybridArchiveNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Caching.Archive = true // For now Archive node should use HashScheme execConfig.Caching.StateScheme = rawdb.HashScheme diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index 106dfc6d46..aa9fbfd72e 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -1042,7 +1042,7 @@ func elevateL2Basefee(t *testing.T, ctx context.Context, builder *NodeBuilder) { _, err = precompilesgen.NewArbosTest(common.HexToAddress("0x69"), builder.L2.Client) Require(t, err, "failed to deploy ArbosTest") - burnAmount := ExecConfigDefaultTest().RPC.RPCGasCap + burnAmount := ExecConfigDefaultTest(t).RPC.RPCGasCap burnTarget := uint64(5 * l2pricing.InitialSpeedLimitPerSecondV6 * l2pricing.InitialBacklogTolerance) for i := uint64(0); i < (burnTarget+burnAmount)/burnAmount; i++ { burnArbGas := arbosTestAbi.Methods["burnArbGas"] diff --git a/system_tests/validation_mock_test.go b/system_tests/validation_mock_test.go index 88421e4c4b..2739c7545e 100644 --- a/system_tests/validation_mock_test.go +++ b/system_tests/validation_mock_test.go @@ -8,9 +8,9 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbnode" @@ -61,8 +61,8 @@ func (s *mockSpawner) WasmModuleRoots() ([]common.Hash, error) { return mockWasmModuleRoots, nil } -func (s *mockSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{"mock"} +func (s *mockSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{"mock"} } func (s *mockSpawner) Launch(entry *validator.ValidationInput, moduleRoot common.Hash) validator.ValidationRun { diff --git a/validator/client/redis/producer.go b/validator/client/redis/producer.go index f98c246d0e..adc2f34af5 100644 --- a/validator/client/redis/producer.go +++ b/validator/client/redis/producer.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/go-redis/redis/v8" "github.com/offchainlabs/nitro/pubsub" @@ -35,7 +36,7 @@ func (c ValidationClientConfig) Enabled() bool { func (c ValidationClientConfig) Validate() error { for _, arch := range c.StylusArchs { - if !rawdb.Target(arch).IsValid() { + if !rawdb.IsSupportedWasmTarget(ethdb.WasmTarget(arch)) { return fmt.Errorf("Invalid stylus arch: %v", arch) } } @@ -162,10 +163,10 @@ func (c *ValidationClient) Name() string { return c.config.Name } -func (c *ValidationClient) StylusArchs() []rawdb.Target { - stylusArchs := make([]rawdb.Target, 0, len(c.config.StylusArchs)) +func (c *ValidationClient) StylusArchs() []ethdb.WasmTarget { + stylusArchs := make([]ethdb.WasmTarget, 0, len(c.config.StylusArchs)) for _, arch := range c.config.StylusArchs { - stylusArchs = append(stylusArchs, rawdb.Target(arch)) + stylusArchs = append(stylusArchs, ethdb.WasmTarget(arch)) } return stylusArchs } diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 80cff66675..9dd7780fbc 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" @@ -31,7 +32,7 @@ type ValidationClient struct { stopwaiter.StopWaiter client *rpcclient.RpcClient name string - stylusArchs []rawdb.Target + stylusArchs []ethdb.WasmTarget room atomic.Int32 wasmModuleRoots []common.Hash } @@ -40,7 +41,7 @@ func NewValidationClient(config rpcclient.ClientConfigFetcher, stack *node.Node) return &ValidationClient{ client: rpcclient.NewRpcClient(config, stack), name: "not started", - stylusArchs: []rawdb.Target{"not started"}, + stylusArchs: []ethdb.WasmTarget{"not started"}, } } @@ -67,14 +68,14 @@ func (c *ValidationClient) Start(ctx context.Context) error { if len(name) == 0 { return errors.New("couldn't read name from server") } - var stylusArchs []rawdb.Target + var stylusArchs []ethdb.WasmTarget if err := c.client.CallContext(ctx, &stylusArchs, server_api.Namespace+"_stylusArchs"); err != nil { var rpcError rpc.Error ok := errors.As(err, &rpcError) if !ok || rpcError.ErrorCode() != -32601 { return fmt.Errorf("could not read stylus arch from server: %w", err) } - stylusArchs = []rawdb.Target{rawdb.Target("pre-stylus")} // invalid, will fail if trying to validate block with stylus + stylusArchs = []ethdb.WasmTarget{ethdb.WasmTarget("pre-stylus")} // invalid, will fail if trying to validate block with stylus } else { if len(stylusArchs) == 0 { return fmt.Errorf("could not read stylus archs from validation server") @@ -117,11 +118,11 @@ func (c *ValidationClient) WasmModuleRoots() ([]common.Hash, error) { return nil, errors.New("not started") } -func (c *ValidationClient) StylusArchs() []rawdb.Target { +func (c *ValidationClient) StylusArchs() []ethdb.WasmTarget { if c.Started() { return c.stylusArchs } - return []rawdb.Target{"not started"} + return []ethdb.WasmTarget{"not started"} } func (c *ValidationClient) Stop() { diff --git a/validator/interface.go b/validator/interface.go index 81b40ae5cf..af08629137 100644 --- a/validator/interface.go +++ b/validator/interface.go @@ -4,7 +4,7 @@ import ( "context" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/containers" ) @@ -14,7 +14,7 @@ type ValidationSpawner interface { Start(context.Context) error Stop() Name() string - StylusArchs() []rawdb.Target + StylusArchs() []ethdb.WasmTarget Room() int } diff --git a/validator/server_api/json.go b/validator/server_api/json.go index dbe2bb1fee..6fe936e17d 100644 --- a/validator/server_api/json.go +++ b/validator/server_api/json.go @@ -11,7 +11,7 @@ import ( "os" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbcompress" "github.com/offchainlabs/nitro/arbutil" @@ -64,7 +64,7 @@ type InputJSON struct { BatchInfo []BatchInfoJson DelayedMsgB64 string StartState validator.GoGlobalState - UserWasms map[rawdb.Target]map[common.Hash]string + UserWasms map[ethdb.WasmTarget]map[common.Hash]string DebugChain bool } @@ -96,7 +96,7 @@ func ValidationInputToJson(entry *validator.ValidationInput) *InputJSON { DelayedMsgB64: base64.StdEncoding.EncodeToString(entry.DelayedMsg), StartState: entry.StartState, PreimagesB64: jsonPreimagesMap, - UserWasms: make(map[rawdb.Target]map[common.Hash]string), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash]string), DebugChain: entry.DebugChain, } for _, binfo := range entry.BatchInfo { @@ -128,7 +128,7 @@ func ValidationInputFromJson(entry *InputJSON) (*validator.ValidationInput, erro DelayedMsgNr: entry.DelayedMsgNr, StartState: entry.StartState, Preimages: preimages, - UserWasms: make(map[rawdb.Target]map[common.Hash][]byte), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash][]byte), DebugChain: entry.DebugChain, } delayed, err := base64.StdEncoding.DecodeString(entry.DelayedMsgB64) diff --git a/validator/server_arb/validator_spawner.go b/validator/server_arb/validator_spawner.go index 844a988d28..96bc5d619d 100644 --- a/validator/server_arb/validator_spawner.go +++ b/validator/server_arb/validator_spawner.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/pflag" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/stopwaiter" @@ -89,8 +90,8 @@ func (s *ArbitratorSpawner) WasmModuleRoots() ([]common.Hash, error) { return s.locator.ModuleRoots(), nil } -func (s *ArbitratorSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{rawdb.TargetWavm} +func (s *ArbitratorSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{rawdb.TargetWavm} } func (s *ArbitratorSpawner) Name() string { diff --git a/validator/server_jit/spawner.go b/validator/server_jit/spawner.go index 92b50b17cb..d77317d218 100644 --- a/validator/server_jit/spawner.go +++ b/validator/server_jit/spawner.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" @@ -72,8 +73,8 @@ func (v *JitSpawner) WasmModuleRoots() ([]common.Hash, error) { return v.locator.ModuleRoots(), nil } -func (v *JitSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{rawdb.LocalTarget()} +func (v *JitSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{rawdb.LocalTarget()} } func (v *JitSpawner) execute( diff --git a/validator/validation_entry.go b/validator/validation_entry.go index 2c357659ad..d340993fa2 100644 --- a/validator/validation_entry.go +++ b/validator/validation_entry.go @@ -2,7 +2,7 @@ package validator import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbutil" ) @@ -17,7 +17,7 @@ type ValidationInput struct { HasDelayedMsg bool DelayedMsgNr uint64 Preimages map[arbutil.PreimageType]map[common.Hash][]byte - UserWasms map[rawdb.Target]map[common.Hash][]byte + UserWasms map[ethdb.WasmTarget]map[common.Hash][]byte BatchInfo []BatchInfo DelayedMsg []byte StartState GoGlobalState diff --git a/validator/valnode/validation_api.go b/validator/valnode/validation_api.go index a79ac7fa55..a10d931dfc 100644 --- a/validator/valnode/validation_api.go +++ b/validator/valnode/validation_api.go @@ -12,7 +12,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" @@ -45,7 +45,7 @@ func (a *ValidationServerAPI) WasmModuleRoots() ([]common.Hash, error) { return a.spawner.WasmModuleRoots() } -func (a *ValidationServerAPI) StylusArchs() ([]rawdb.Target, error) { +func (a *ValidationServerAPI) StylusArchs() ([]ethdb.WasmTarget, error) { return a.spawner.StylusArchs(), nil } From eefd7294777775f69689c316ca97c4b9d1338172 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 00:52:30 +0200 Subject: [PATCH 07/60] compile to targets in WasmTargets list --- arbos/programs/native.go | 61 +++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 36e200af4d..98abf9443c 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -100,25 +100,50 @@ func activateProgramInternal( } return nil, nil, err } - target := rawdb.LocalTarget() - status_asm := C.stylus_compile( - goSlice(wasm), - u16(version), - cbool(debug), - goSlice([]byte(target)), - output, - ) - asm := output.intoBytes() - if status_asm != 0 { - return nil, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm)) + targets := db.Database().WasmTargets() + type result struct { + target ethdb.WasmTarget + asm []byte + err error } - asmMap := map[ethdb.WasmTarget][]byte{ - rawdb.TargetWavm: module, - target: asm, + results := make(chan result, len(targets)) + for _, target := range targets { + if target == rawdb.TargetWavm { + results <- result{target, module, nil} + } else { + target := target + go func() { + output := &rustBytes{} + status_asm := C.stylus_compile( + goSlice(wasm), + u16(version), + cbool(debug), + goSlice([]byte(target)), + output, + ) + asm := output.intoBytes() + if status_asm != 0 { + results <- result{target, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm))} + return + } + results <- result{target, asm, nil} + }() + } + } + asmMap := make(map[ethdb.WasmTarget][]byte, len(targets)) + for range targets { + res := <-results + if res.err != nil { + err = errors.Join(res.err, err) + } else { + asmMap[res.target] = res.asm + } + } + if err != nil { + return nil, nil, fmt.Errorf("compilation failed for one or more targets: %w", err) } hash := moduleHash.toHash() - info := &activationInfo{ moduleHash: hash, initGas: uint16(stylusData.init_cost), @@ -204,11 +229,7 @@ func callProgram( } if db, ok := db.(*state.StateDB); ok { - targets := []ethdb.WasmTarget{ - rawdb.TargetWavm, - rawdb.LocalTarget(), - } - db.RecordProgram(targets, moduleHash) + db.RecordProgram(moduleHash) } evmApi := newApi(interpreter, tracingInfo, scope, memoryModel) From 2832703599b4749c32e9af9760b4d3ffdacbba9c Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 01:21:02 +0200 Subject: [PATCH 08/60] use wasm targets list in SaveActiveProgramToWasmStore --- arbos/programs/wasmstorehelper.go | 4 ++-- go-ethereum | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arbos/programs/wasmstorehelper.go b/arbos/programs/wasmstorehelper.go index 95daaf544a..434820dd9c 100644 --- a/arbos/programs/wasmstorehelper.go +++ b/arbos/programs/wasmstorehelper.go @@ -12,7 +12,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -44,8 +43,9 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash return err } + targets := statedb.Database().WasmTargets() // If already in wasm store then return early - _, err = statedb.TryGetActivatedAsmMap([]ethdb.WasmTarget{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash) + _, err = statedb.TryGetActivatedAsmMap(targets, moduleHash) if err == nil { return nil } diff --git a/go-ethereum b/go-ethereum index cb03c12f7f..c7d753f2c4 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit cb03c12f7f02d47344d2d089eef748e92b99c95e +Subproject commit c7d753f2c4d5eb41c94e2e2499219a72fbf241a0 From 10f9b159550b59d97304cd935db25b076adf6443 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 01:48:47 +0200 Subject: [PATCH 09/60] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index c7d753f2c4..7fc4c64607 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit c7d753f2c4d5eb41c94e2e2499219a72fbf241a0 +Subproject commit 7fc4c646075eb9b6e26374886ea6fe9463b106aa From 17319ad2d1bb00d0c63a22693844652f6b739b1e Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 16:18:01 +0200 Subject: [PATCH 10/60] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 7fc4c64607..0a376604c9 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 7fc4c646075eb9b6e26374886ea6fe9463b106aa +Subproject commit 0a376604c9ce0c588de0d943184380acd7e25db4 From cdb43d77e5ef976c6ad3e09ac63d07034c85bfcf Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 17:57:18 +0200 Subject: [PATCH 11/60] update validation client stylus archs sanity check --- validator/client/validation_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 36316a8328..3b18ad1851 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -81,7 +81,7 @@ func (c *ValidationClient) Start(ctx context.Context) error { return fmt.Errorf("could not read stylus archs from validation server") } for _, stylusArch := range stylusArchs { - if stylusArch != rawdb.TargetWavm && stylusArch != rawdb.LocalTarget() && stylusArch != "mock" { + if !rawdb.IsSupportedWasmTarget(ethdb.WasmTarget(stylusArch)) && stylusArch != "mock" { return fmt.Errorf("unsupported stylus architecture: %v", stylusArch) } } From 71ab928cb42b66247ac9fdc3acd82280255de712 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 19:27:15 +0200 Subject: [PATCH 12/60] change archs list option to extra-archs that must include wavm and is appended with local target if not in list --- execution/gethexec/node.go | 47 ++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index f519cce6ed..bc29e9b7f2 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -30,10 +30,10 @@ import ( ) type StylusTargetConfig struct { - Arm64 string `koanf:"arm64"` - Amd64 string `koanf:"amd64"` - Host string `koanf:"host"` - Archs []string `koanf:"archs"` + Arm64 string `koanf:"arm64"` + Amd64 string `koanf:"amd64"` + Host string `koanf:"host"` + ExtraArchs []string `koanf:"extra-archs"` wasmTargets []ethdb.WasmTarget } @@ -44,47 +44,40 @@ func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { func (c *StylusTargetConfig) Validate() error { localTarget := rawdb.LocalTarget() - var nativeFound bool - targets := make([]ethdb.WasmTarget, 0, len(c.Archs)) - for _, arch := range c.Archs { + targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)) + targetsSet := make(map[ethdb.WasmTarget]struct{}, len(c.ExtraArchs)+1) + for _, arch := range c.ExtraArchs { target := ethdb.WasmTarget(arch) if !rawdb.IsSupportedWasmTarget(target) { return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) } - var alreadyInList bool - for _, t := range targets { - if target == t { - alreadyInList = true - break - } - } - if alreadyInList { - continue + if _, duplicate := targetsSet[target]; !duplicate { + targets = append(targets, target) + targetsSet[target] = struct{}{} } - if target == localTarget { - nativeFound = true - } - targets = append(targets, target) } - if !nativeFound { - return fmt.Errorf("native target not found in archs list, native target: %v, archs: %v", localTarget, c.Archs) + if _, has := targetsSet[rawdb.TargetWavm]; !has { + return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) + } + if _, has := targetsSet[localTarget]; !has { + targets = append(targets, localTarget) } c.wasmTargets = targets return nil } var DefaultStylusTargetConfig = StylusTargetConfig{ - Arm64: programs.DefaultTargetDescriptionArm, - Amd64: programs.DefaultTargetDescriptionX86, - Host: "", - Archs: []string{string(rawdb.TargetWavm), string(rawdb.LocalTarget())}, + Arm64: programs.DefaultTargetDescriptionArm, + Amd64: programs.DefaultTargetDescriptionX86, + Host: "", + ExtraArchs: []string{string(rawdb.TargetWavm)}, } func StylusTargetConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".arm64", DefaultStylusTargetConfig.Arm64, "stylus programs compilation target for arm64 linux") f.String(prefix+".amd64", DefaultStylusTargetConfig.Amd64, "stylus programs compilation target for amd64 linux") f.String(prefix+".host", DefaultStylusTargetConfig.Host, "stylus programs compilation target for system other than 64-bit ARM or 64-bit x86") - f.StringSlice(prefix+".archs", DefaultStylusTargetConfig.Archs, fmt.Sprintf("Comma separated architectures list to compile stylus program to and cache in wasm store (available targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)) + f.StringSlice(prefix+".extra-archs", DefaultStylusTargetConfig.ExtraArchs, fmt.Sprintf("Comma separated list of extra architectures to cross-compile stylus program to and cache in wasm store (additionally to local target). Currently must include at least %s. (supported targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)) } type Config struct { From 29df43a287833aa9541fa12db853bc6e3b07fd5f Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 23 Aug 2024 12:06:22 -0300 Subject: [PATCH 13/60] Remove wasm database after downloading snapshot By default, remove the wasm DB because it contains native executables. Hence, it is a security concern when downloading a snapshot from a untrusted source. --- cmd/conf/init.go | 3 +++ cmd/nitro/init.go | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index d88bcdd241..242f43dc82 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -22,6 +22,7 @@ type InitConfig struct { DevInitAddress string `koanf:"dev-init-address"` DevInitBlockNum uint64 `koanf:"dev-init-blocknum"` Empty bool `koanf:"empty"` + ExtractWasm bool `koanf:"extract-wasm"` AccountsPerSync uint `koanf:"accounts-per-sync"` ImportFile string `koanf:"import-file"` ThenQuit bool `koanf:"then-quit"` @@ -48,6 +49,7 @@ var InitConfigDefault = InitConfig{ DevInitAddress: "", DevInitBlockNum: 0, Empty: false, + ExtractWasm: false, ImportFile: "", AccountsPerSync: 100000, ThenQuit: false, @@ -74,6 +76,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".dev-init-address", InitConfigDefault.DevInitAddress, "Address of dev-account. Leave empty to use the dev-wallet.") f.Uint64(prefix+".dev-init-blocknum", InitConfigDefault.DevInitBlockNum, "Number of preinit blocks. Must exist in ancient database.") f.Bool(prefix+".empty", InitConfigDefault.Empty, "init with empty state") + f.Bool(prefix+".extract-wasm", InitConfigDefault.ExtractWasm, "if set, extract the wasm directory when downloading a database (this can be security concern because it contains native executables)") f.Bool(prefix+".then-quit", InitConfigDefault.ThenQuit, "quit after init is done") f.String(prefix+".import-file", InitConfigDefault.ImportFile, "path for json data to import") f.Uint(prefix+".accounts-per-sync", InitConfigDefault.AccountsPerSync, "during init - sync database every X accounts. Lower value for low-memory systems. 0 disables.") diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index d9ae0df3b0..b5cab75cf6 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -602,6 +602,17 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err != nil { return nil, nil, fmt.Errorf("couln't extract init archive '%v' err:%w", initFile, err) } + wasmDb := path.Join(stack.InstanceDir(), "wasm") + if dirExists(wasmDb) && !config.Init.ExtractWasm { + // By default, remove the wasm DB because it contains native executables. + // Hence, it is a security concern when downloading a snapshot from a + // untrusted source. + err := os.RemoveAll(wasmDb) + if err != nil { + return nil, nil, fmt.Errorf("failed to remove extracted wasm database: %w", err) + } + log.Debug("init: removed wasm database") + } } var initDataReader statetransfer.InitDataReader = nil From a84a88803db272922160d620d1d4df9bdd550064 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 23 Aug 2024 14:53:37 -0300 Subject: [PATCH 14/60] Don't recurse submodules in submodule-pin-check github action --- .github/workflows/submodule-pin-check.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index e459bad34d..b8b97a27c5 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -14,8 +14,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - submodules: recursive + submodules: true - name: Check all submodules are ancestors of origin/HEAD or configured branch run: ${{ github.workspace }}/.github/workflows/submodule-pin-check.sh - From f4da3d42d2f7166f4acddcb5d79e8cc7d4c0f05c Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 23 Aug 2024 15:07:58 -0300 Subject: [PATCH 15/60] Workflow "Merge Checks" creates "Submodule Pin Check" through github API --- .github/workflows/submodule-pin-check.yml | 32 +++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index b8b97a27c5..12a9656e84 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,10 +1,13 @@ -name: Submodule Pin Check +name: Merge Checks on: pull_request: branches: [ master ] types: [synchronize, opened, reopened] +permissions: + statuses: write + jobs: submodule-pin-check: name: Submodule Pin Check @@ -17,4 +20,29 @@ jobs: submodules: true - name: Check all submodules are ancestors of origin/HEAD or configured branch - run: ${{ github.workspace }}/.github/workflows/submodule-pin-check.sh + run: | + status_state="pending" + if ${{ github.workspace }}/.github/workflows/submodule-pin-check.sh; then + status_state="success" + else + resp="$(curl -sSL --fail-with-body \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/commits/${{ github.event.pull_request.head.sha }}/statuses")" + + if ! jq -e '.[] | select(.context == "Submodule Pin Check")' > /dev/null <<< "$resp"; then + # Submodule pin check is failling and no status exists + # Keep it without a status to keep the green checkmark appearing + # Otherwise, the commit and PR's CI will appear to be indefinitely pending + # Merging will still be blocked until the required status appears + exit 0 + fi + fi + curl -sSL --fail-with-body \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/${{ github.event.pull_request.head.sha }}" \ + -d '{"context":"Submodule Pin Check","state":"'"$status_state"'"}' From 966ec11ac96d892c56d7f4dcffb6b4dadd2e7abb Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 26 Aug 2024 11:13:45 -0300 Subject: [PATCH 16/60] Rename conf init.extract-wasm to init.import-wasm --- cmd/conf/init.go | 6 +++--- cmd/nitro/init.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index d0ca592e42..dddf3c0c28 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -22,7 +22,7 @@ type InitConfig struct { DevInitAddress string `koanf:"dev-init-address"` DevInitBlockNum uint64 `koanf:"dev-init-blocknum"` Empty bool `koanf:"empty"` - ExtractWasm bool `koanf:"extract-wasm"` + ImportWasm bool `koanf:"import-wasm"` AccountsPerSync uint `koanf:"accounts-per-sync"` ImportFile string `koanf:"import-file"` ThenQuit bool `koanf:"then-quit"` @@ -49,7 +49,7 @@ var InitConfigDefault = InitConfig{ DevInitAddress: "", DevInitBlockNum: 0, Empty: false, - ExtractWasm: false, + ImportWasm: false, ImportFile: "", AccountsPerSync: 100000, ThenQuit: false, @@ -76,7 +76,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".dev-init-address", InitConfigDefault.DevInitAddress, "Address of dev-account. Leave empty to use the dev-wallet.") f.Uint64(prefix+".dev-init-blocknum", InitConfigDefault.DevInitBlockNum, "Number of preinit blocks. Must exist in ancient database.") f.Bool(prefix+".empty", InitConfigDefault.Empty, "init with empty state") - f.Bool(prefix+".extract-wasm", InitConfigDefault.ExtractWasm, "if set, extract the wasm directory when downloading a database (this can be security concern because it contains native executables)") + f.Bool(prefix+".import-wasm", InitConfigDefault.ImportWasm, "if set, extract the wasm directory when downloading a database (this can be security concern because it contains native executables)") f.Bool(prefix+".then-quit", InitConfigDefault.ThenQuit, "quit after init is done") f.String(prefix+".import-file", InitConfigDefault.ImportFile, "path for json data to import") f.Uint(prefix+".accounts-per-sync", InitConfigDefault.AccountsPerSync, "during init - sync database every X accounts. Lower value for low-memory systems. 0 disables.") diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index aaef5798f5..55325f1b93 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -611,7 +611,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return nil, nil, fmt.Errorf("couln't extract init archive '%v' err:%w", initFile, err) } wasmDb := path.Join(stack.InstanceDir(), "wasm") - if dirExists(wasmDb) && !config.Init.ExtractWasm { + if dirExists(wasmDb) && !config.Init.ImportWasm { // By default, remove the wasm DB because it contains native executables. // Hence, it is a security concern when downloading a snapshot from a // untrusted source. From a3bf49a54e5dddba0e36f80162f7e2d9115e4e44 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 26 Aug 2024 11:14:52 -0300 Subject: [PATCH 17/60] Improve init.import-wasm description --- cmd/conf/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index dddf3c0c28..f01d99f8b7 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -76,7 +76,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".dev-init-address", InitConfigDefault.DevInitAddress, "Address of dev-account. Leave empty to use the dev-wallet.") f.Uint64(prefix+".dev-init-blocknum", InitConfigDefault.DevInitBlockNum, "Number of preinit blocks. Must exist in ancient database.") f.Bool(prefix+".empty", InitConfigDefault.Empty, "init with empty state") - f.Bool(prefix+".import-wasm", InitConfigDefault.ImportWasm, "if set, extract the wasm directory when downloading a database (this can be security concern because it contains native executables)") + f.Bool(prefix+".import-wasm", InitConfigDefault.ImportWasm, "if set, import the wasm directory when downloading a database (contains executable code - only use with highly trusted source)") f.Bool(prefix+".then-quit", InitConfigDefault.ThenQuit, "quit after init is done") f.String(prefix+".import-file", InitConfigDefault.ImportFile, "path for json data to import") f.Uint(prefix+".accounts-per-sync", InitConfigDefault.AccountsPerSync, "during init - sync database every X accounts. Lower value for low-memory systems. 0 disables.") From 57249c83fc7ad215a874d56c5e7200b0ae77cefb Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 26 Aug 2024 11:27:35 -0300 Subject: [PATCH 18/60] Remove WASM database even if the extraction fails --- cmd/nitro/init.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 55325f1b93..318c134eb1 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -608,18 +608,22 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo log.Info("extracting downloaded init archive", "size", fmt.Sprintf("%dMB", stat.Size()/1024/1024)) err = extract.Archive(context.Background(), reader, stack.InstanceDir(), nil) if err != nil { - return nil, nil, fmt.Errorf("couln't extract init archive '%v' err:%w", initFile, err) + err = fmt.Errorf("couln't extract init archive '%v' err:%w", initFile, err) } wasmDb := path.Join(stack.InstanceDir(), "wasm") if dirExists(wasmDb) && !config.Init.ImportWasm { // By default, remove the wasm DB because it contains native executables. // Hence, it is a security concern when downloading a snapshot from a // untrusted source. - err := os.RemoveAll(wasmDb) - if err != nil { - return nil, nil, fmt.Errorf("failed to remove extracted wasm database: %w", err) + removeErr := os.RemoveAll(wasmDb) + if removeErr != nil { + err = errors.Join(err, fmt.Errorf("failed to remove extracted wasm database: %w", removeErr)) + } else { + log.Debug("init: removed wasm database") } - log.Debug("init: removed wasm database") + } + if err != nil { + return nil, nil, err } } From 304f2c1127a4def9d292b617b3b52a437e9891b2 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 26 Aug 2024 16:53:34 +0200 Subject: [PATCH 19/60] add cross compilation to progam test --- system_tests/common_test.go | 5 ++ system_tests/program_test.go | 92 +++++++++++++++++++++++++++++++++--- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 392f6de805..457dae0910 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -293,6 +293,11 @@ func (b *NodeBuilder) WithWasmRootDir(wasmRootDir string) *NodeBuilder { return b } +func (b *NodeBuilder) WithExtraArchs(targets []string) *NodeBuilder { + b.execConfig.StylusTarget.ExtraArchs = targets + return b +} + func (b *NodeBuilder) Build(t *testing.T) func() { b.CheckConfig(t) if b.withL1 { diff --git a/system_tests/program_test.go b/system_tests/program_test.go index ed640809db..8c4ca2e438 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -45,18 +45,31 @@ import ( var oneEth = arbmath.UintToBig(1e18) +var allWasmTargets = []string{string(rawdb.TargetWavm), string(rawdb.TargetArm64), string(rawdb.TargetAmd64), string(rawdb.TargetHost)} + func TestProgramKeccak(t *testing.T) { t.Parallel() - keccakTest(t, true) + t.Run("WithDefaultWasmTargets", func(t *testing.T) { + keccakTest(t, true) + }) + + t.Run("WithAllWasmTargets", func(t *testing.T) { + keccakTest(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs(allWasmTargets) + }) + }) } -func keccakTest(t *testing.T, jit bool) { - builder, auth, cleanup := setupProgramTest(t, jit) +func keccakTest(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { + builder, auth, cleanup := setupProgramTest(t, jit, builderOpts...) ctx := builder.ctx l2client := builder.L2.Client defer cleanup() programAddress := deployWasm(t, ctx, auth, l2client, rustFile("keccak")) + wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + wasm, _ := readWasmFile(t, rustFile("keccak")) otherAddressSameCode := deployContract(t, ctx, auth, l2client, wasm) arbWasm, err := pgen.NewArbWasm(types.ArbWasmAddress, l2client) @@ -68,6 +81,7 @@ func keccakTest(t *testing.T, jit bool) { Fatal(t, "activate should have failed with ProgramUpToDate", err) } }) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) if programAddress == otherAddressSameCode { Fatal(t, "expected to deploy at two separate program addresses") @@ -141,11 +155,18 @@ func keccakTest(t *testing.T, jit bool) { func TestProgramActivateTwice(t *testing.T) { t.Parallel() - testActivateTwice(t, true) + t.Run("WithDefaultWasmTargets", func(t *testing.T) { + testActivateTwice(t, true) + }) + t.Run("WithAllWasmTargets", func(t *testing.T) { + testActivateTwice(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs(allWasmTargets) + }) + }) } -func testActivateTwice(t *testing.T, jit bool) { - builder, auth, cleanup := setupProgramTest(t, jit) +func testActivateTwice(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { + builder, auth, cleanup := setupProgramTest(t, jit, builderOpts...) ctx := builder.ctx l2info := builder.L2Info l2client := builder.L2.Client @@ -171,6 +192,10 @@ func testActivateTwice(t *testing.T, jit bool) { colors.PrintBlue("keccak program B deployed to ", keccakB) multiAddr := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) + + wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + preimage := []byte("it's time to du-du-du-du d-d-d-d-d-d-d de-duplicate") keccakArgs := []byte{0x01} // keccak the preimage once @@ -194,6 +219,7 @@ func testActivateTwice(t *testing.T, jit bool) { // Calling the contract pre-activation should fail. checkReverts() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // mechanisms for creating calldata activateProgram, _ := util.NewCallParser(pgen.ArbWasmABI, "activateProgram") @@ -216,6 +242,7 @@ func testActivateTwice(t *testing.T, jit bool) { // Ensure the revert also reverted keccak's activation checkReverts() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // Activate keccak program A, then call into B, which should succeed due to being the same codehash args = argsForMulticall(vm.CALL, types.ArbWasmAddress, oneEth, pack(activateProgram(keccakA))) @@ -223,6 +250,7 @@ func testActivateTwice(t *testing.T, jit bool) { tx = l2info.PrepareTxTo("Owner", &multiAddr, 1e9, oneEth, args) ensure(tx, l2client.SendTransaction(ctx, tx)) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 2) validateBlocks(t, 7, jit, builder) } @@ -1834,7 +1862,9 @@ func createMapFromDb(db ethdb.KeyValueStore) (map[string][]byte, error) { } func TestWasmStoreRebuilding(t *testing.T) { - builder, auth, cleanup := setupProgramTest(t, true) + builder, auth, cleanup := setupProgramTest(t, true, func(b *NodeBuilder) { + b.WithExtraArchs(allWasmTargets) + }) ctx := builder.ctx l2info := builder.L2Info l2client := builder.L2.Client @@ -1871,6 +1901,7 @@ func TestWasmStoreRebuilding(t *testing.T) { storeMap, err := createMapFromDb(wasmDb) Require(t, err) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // close nodeB cleanupB() @@ -1926,5 +1957,52 @@ func TestWasmStoreRebuilding(t *testing.T) { } } + checkWasmStoreContent(t, wasmDbAfterRebuild, builder.execConfig.StylusTarget.ExtraArchs, 1) cleanupB() } + +func readModuleHashes(t *testing.T, wasmDb ethdb.KeyValueStore) []common.Hash { + modulesSet := make(map[common.Hash]struct{}) + asmPrefix := []byte{0x00, 'w'} + it := wasmDb.NewIterator(asmPrefix, nil) + defer it.Release() + for it.Next() { + key := it.Key() + if len(key) != rawdb.WasmKeyLen { + t.Fatalf("unexpected activated module key length, len: %d, key: %v", len(key), key) + } + moduleHash := key[rawdb.WasmPrefixLen:] + if len(moduleHash) != common.HashLength { + t.Fatalf("Invalid moduleHash length in key: %v, moduleHash: %v", key, moduleHash) + } + modulesSet[common.BytesToHash(moduleHash)] = struct{}{} + } + modules := make([]common.Hash, 0, len(modulesSet)) + for module := range modulesSet { + modules = append(modules, module) + } + return modules +} + +func checkWasmStoreContent(t *testing.T, wasmDb ethdb.KeyValueStore, targets []string, numModules int) { + modules := readModuleHashes(t, wasmDb) + if len(modules) != numModules { + t.Fatalf("Unexpected number of module hashes found in wasm store, want: %d, have: %d", numModules, len(modules)) + } + for _, module := range modules { + for _, target := range targets { + wasmTarget := ethdb.WasmTarget(target) + if !rawdb.IsSupportedWasmTarget(wasmTarget) { + t.Fatalf("internal test error - unsupported target passed to checkWasmStoreContent: %v", target) + } + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("Failed to read activated asm for target: %v, module: %v", target, module) + } + }() + _ = rawdb.ReadActivatedAsm(wasmDb, wasmTarget, module) + }() + } + } +} From 2e5a6eadc03a562fa32e96781fda77ab430b55a0 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 26 Aug 2024 12:22:25 -0300 Subject: [PATCH 20/60] Renames job name related to checking submodule pin --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 12a9656e84..870f520877 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -10,7 +10,7 @@ permissions: jobs: submodule-pin-check: - name: Submodule Pin Check + name: Check Submodule Pin runs-on: ubuntu-latest steps: - name: Checkout From 18de0ad594cf3882f3d2c6a85a497080f3816f1b Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Mon, 26 Aug 2024 18:32:29 -0600 Subject: [PATCH 21/60] update default x86 architecture --- arbos/programs/native.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index fd3dec25a0..9316e8f230 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -289,7 +289,7 @@ func ResizeWasmLruCache(size uint32) { } const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon" -const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2" +const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2+lzcnt+bmi" func SetTarget(name rawdb.Target, description string, native bool) error { output := &rustBytes{} From 9e9bcc43cfe6b2d0bf0c3710e416130ac9321328 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Mon, 26 Aug 2024 19:17:02 -0600 Subject: [PATCH 22/60] testcompile: test loaded compile with default target --- arbos/programs/cgo_test.go | 8 ++++++++ arbos/programs/testcompile.go | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/arbos/programs/cgo_test.go b/arbos/programs/cgo_test.go index c0e146d98d..e16c362ef8 100644 --- a/arbos/programs/cgo_test.go +++ b/arbos/programs/cgo_test.go @@ -40,5 +40,13 @@ func TestCompileArch(t *testing.T) { if err != nil { t.Fatal(err) } + err = resetNativeTarget() + if err != nil { + t.Fatal(err) + } + err = testCompileLoad() + if err != nil { + t.Fatal(err) + } } } diff --git a/arbos/programs/testcompile.go b/arbos/programs/testcompile.go index a16bae52c0..1daf470620 100644 --- a/arbos/programs/testcompile.go +++ b/arbos/programs/testcompile.go @@ -178,6 +178,28 @@ func testCompileArch(store bool) error { return nil } +func resetNativeTarget() error { + output := &rustBytes{} + + _, err := fmt.Print("resetting native target\n") + if err != nil { + return err + } + + localCompileName := []byte("local") + + status := C.stylus_target_set(goSlice(localCompileName), + goSlice([]byte{}), + output, + cbool(true)) + + if status != 0 { + return fmt.Errorf("failed setting compilation target arm: %v", string(output.intoBytes())) + } + + return nil +} + func testCompileLoad() error { filePath := "../../target/testdata/host.bin" localTarget := rawdb.LocalTarget() From ea0cbc672d82802e91be407b595463d6d9415f8c Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Tue, 27 Aug 2024 15:05:59 +0200 Subject: [PATCH 23/60] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 0a376604c9..4ca4b4d734 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 0a376604c9ce0c588de0d943184380acd7e25db4 +Subproject commit 4ca4b4d734021df09cb42a796ca8f263d47383f9 From af6320388b2bf9928251115d7edfafb33cbcf81b Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 27 Aug 2024 14:03:04 -0500 Subject: [PATCH 24/60] Return false if nil is passed to IsNotExistError --- util/dbutil/dbutil.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/util/dbutil/dbutil.go b/util/dbutil/dbutil.go index ca0f5aaaeb..6573c5742c 100644 --- a/util/dbutil/dbutil.go +++ b/util/dbutil/dbutil.go @@ -6,7 +6,7 @@ package dbutil import ( "errors" "fmt" - "os" + "io/fs" "regexp" "github.com/cockroachdb/pebble" @@ -22,13 +22,15 @@ func IsErrNotFound(err error) bool { var pebbleNotExistErrorRegex = regexp.MustCompile("pebble: database .* does not exist") func isPebbleNotExistError(err error) bool { - return pebbleNotExistErrorRegex.MatchString(err.Error()) + return err != nil && pebbleNotExistErrorRegex.MatchString(err.Error()) } func isLeveldbNotExistError(err error) bool { - return os.IsNotExist(err) + return errors.Is(err, fs.ErrNotExist) } +// IsNotExistError returns true if the error is a "database not found" error. +// It must return false if err is nil. func IsNotExistError(err error) bool { return isLeveldbNotExistError(err) || isPebbleNotExistError(err) } From a32669038ec7d6009d3614973f8b779890aa13c4 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 27 Aug 2024 14:05:11 -0500 Subject: [PATCH 25/60] Add test case --- util/dbutil/dbutil_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/util/dbutil/dbutil_test.go b/util/dbutil/dbutil_test.go index b28f8a2c23..b303bb56b6 100644 --- a/util/dbutil/dbutil_test.go +++ b/util/dbutil/dbutil_test.go @@ -28,6 +28,9 @@ func testIsNotExistError(t *testing.T, dbEngine string, isNotExist func(error) b if isNotExist(err) { t.Fatalf("Classified other error as not exist, err: %v", err) } + if isNotExist(nil) { + t.Fatal("Classified nil as not exist") + } } func TestIsNotExistError(t *testing.T) { From eb1fd9e1bd366b89431e29b3ddcc05586f90d0b4 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Tue, 27 Aug 2024 17:55:09 -0300 Subject: [PATCH 26/60] Do not extract wasm db instead of deleting it Use the extract.Renamer function to avoid extracting the wasm DB when the init.import-wasm flag is set to false. --- cmd/nitro/init.go | 57 ++++++++++--------- cmd/nitro/init_test.go | 121 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 27 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 9a1230651b..53048d3486 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -24,7 +24,7 @@ import ( "time" "github.com/cavaliergopher/grab/v3" - extract "github.com/codeclysm/extract/v3" + "github.com/codeclysm/extract/v3" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -405,6 +405,34 @@ func databaseIsEmpty(db ethdb.Database) bool { return !it.Next() } +func extractSnapshot(archive string, location string, importWasm bool) error { + isWasmDB := regexp.MustCompile("^(./)?wasm") + reader, err := os.Open(archive) + if err != nil { + return fmt.Errorf("couln't open init '%v' archive: %w", archive, err) + } + stat, err := reader.Stat() + if err != nil { + return err + } + log.Info("extracting downloaded init archive", "size", fmt.Sprintf("%dMB", stat.Size()/1024/1024)) + var rename extract.Renamer + if !importWasm { + rename = func(path string) string { + path = strings.ToLower(path) + if isWasmDB.MatchString(path) { + return "" // do not extract wasm files + } + return path + } + } + err = extract.Archive(context.Background(), reader, location, rename) + if err != nil { + return fmt.Errorf("couln't extract init archive '%v' err: %w", archive, err) + } + return nil +} + // removes all entries with keys prefixed with prefixes and of length used in initial version of wasm store schema func purgeVersion0WasmStoreEntries(db ethdb.Database) error { prefixes, keyLength := rawdb.DeprecatedPrefixesV0() @@ -597,32 +625,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo } if initFile != "" { - reader, err := os.Open(initFile) - if err != nil { - return nil, nil, fmt.Errorf("couln't open init '%v' archive: %w", initFile, err) - } - stat, err := reader.Stat() - if err != nil { - return nil, nil, err - } - log.Info("extracting downloaded init archive", "size", fmt.Sprintf("%dMB", stat.Size()/1024/1024)) - err = extract.Archive(context.Background(), reader, stack.InstanceDir(), nil) - if err != nil { - err = fmt.Errorf("couln't extract init archive '%v' err:%w", initFile, err) - } - wasmDb := path.Join(stack.InstanceDir(), "wasm") - if dirExists(wasmDb) && !config.Init.ImportWasm { - // By default, remove the wasm DB because it contains native executables. - // Hence, it is a security concern when downloading a snapshot from a - // untrusted source. - removeErr := os.RemoveAll(wasmDb) - if removeErr != nil { - err = errors.Join(err, fmt.Errorf("failed to remove extracted wasm database: %w", removeErr)) - } else { - log.Debug("init: removed wasm database") - } - } - if err != nil { + if err := extractSnapshot(initFile, stack.InstanceDir(), config.Init.ImportWasm); err != nil { return nil, nil, err } } diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index b2773ed861..9282839f75 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -4,18 +4,21 @@ package main import ( + "archive/tar" "bytes" "context" "crypto/sha256" "encoding/hex" "errors" "fmt" + "io" "math/big" "net" "net/http" "os" "path" "path/filepath" + "slices" "strings" "testing" "time" @@ -24,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/node" + "github.com/google/go-cmp/cmp" "github.com/offchainlabs/nitro/arbnode" "github.com/offchainlabs/nitro/cmd/chaininfo" "github.com/offchainlabs/nitro/cmd/conf" @@ -554,3 +558,120 @@ func TestOpenInitializeChainDbEmptyInit(t *testing.T) { err = chainDb.Close() Require(t, err) } + +func TestExtractSnapshot(t *testing.T) { + testCases := []struct { + name string + archiveFiles []string + importWasm bool + wantFiles []string + }{ + { + name: "extractAll", + importWasm: true, + archiveFiles: []string{ + "arbitrumdata/000001.ldb", + "l2chaindata/000001.ldb", + "l2chaindata/ancients/000001.ldb", + "nodes/000001.ldb", + "wasm/000001.ldb", + }, + wantFiles: []string{ + "arbitrumdata/000001.ldb", + "l2chaindata/000001.ldb", + "l2chaindata/ancients/000001.ldb", + "nodes/000001.ldb", + "wasm/000001.ldb", + }, + }, + { + name: "extractAllButWasm", + importWasm: false, + archiveFiles: []string{ + "arbitrumdata/000001.ldb", + "l2chaindata/000001.ldb", + "nodes/000001.ldb", + "wasm/000001.ldb", + }, + wantFiles: []string{ + "arbitrumdata/000001.ldb", + "l2chaindata/000001.ldb", + "nodes/000001.ldb", + }, + }, + { + name: "extractAllButWasmWithPrefixDot", + importWasm: false, + archiveFiles: []string{ + "./arbitrumdata/000001.ldb", + "./l2chaindata/000001.ldb", + "./nodes/000001.ldb", + "./wasm/000001.ldb", + }, + wantFiles: []string{ + "arbitrumdata/000001.ldb", + "l2chaindata/000001.ldb", + "nodes/000001.ldb", + }, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + // Create archive with dummy files + archiveDir := t.TempDir() + archivePath := path.Join(archiveDir, "archive.tar") + { + // Create context to close the file handlers + archiveFile, err := os.Create(archivePath) + Require(t, err) + defer archiveFile.Close() + tarWriter := tar.NewWriter(archiveFile) + defer tarWriter.Close() + for _, relativePath := range testCase.archiveFiles { + filePath := path.Join(archiveDir, relativePath) + dir := filepath.Dir(filePath) + const dirPerm = 0700 + err := os.MkdirAll(dir, dirPerm) + Require(t, err) + const filePerm = 0600 + err = os.WriteFile(filePath, []byte{0xbe, 0xef}, filePerm) + Require(t, err) + file, err := os.Open(filePath) + Require(t, err) + info, err := file.Stat() + Require(t, err) + header, err := tar.FileInfoHeader(info, "") + Require(t, err) + header.Name = relativePath + err = tarWriter.WriteHeader(header) + Require(t, err) + _, err = io.Copy(tarWriter, file) + Require(t, err) + } + } + + // Extract archive and compare contents + targetDir := t.TempDir() + err := extractSnapshot(archivePath, targetDir, testCase.importWasm) + Require(t, err, "failed to extract snapshot") + gotFiles := []string{} + err = filepath.WalkDir(targetDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + gotFiles = append(gotFiles, path) + } + return nil + }) + Require(t, err) + slices.Sort(gotFiles) + for i, f := range testCase.wantFiles { + testCase.wantFiles[i] = path.Join(targetDir, f) + } + if diff := cmp.Diff(gotFiles, testCase.wantFiles); diff != "" { + t.Fatal("extracted files don't match", diff) + } + }) + } +} From 8ffca7e70304cede8048afa981b442b14677d3bb Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Tue, 27 Aug 2024 17:59:27 -0300 Subject: [PATCH 27/60] Do not change case when extracting DB --- cmd/nitro/init.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 53048d3486..435c73c997 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -419,8 +419,7 @@ func extractSnapshot(archive string, location string, importWasm bool) error { var rename extract.Renamer if !importWasm { rename = func(path string) string { - path = strings.ToLower(path) - if isWasmDB.MatchString(path) { + if isWasmDB.MatchString(strings.ToLower(path)) { return "" // do not extract wasm files } return path From 7b4834facf7e1c5834ea630551b9908a15183321 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 13:24:35 +0200 Subject: [PATCH 28/60] pass WasmTargets to RecordProgram call --- arbos/programs/native.go | 4 ++-- go-ethereum | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 98abf9443c..08431b1209 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -228,8 +228,8 @@ func callProgram( panic("missing asm") } - if db, ok := db.(*state.StateDB); ok { - db.RecordProgram(moduleHash) + if stateDb, ok := db.(*state.StateDB); ok { + stateDb.RecordProgram(db.Database().WasmTargets(), moduleHash) } evmApi := newApi(interpreter, tracingInfo, scope, memoryModel) diff --git a/go-ethereum b/go-ethereum index 4ca4b4d734..81114dde8a 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 4ca4b4d734021df09cb42a796ca8f263d47383f9 +Subproject commit 81114dde8a26bae90c188605c4a36d5919a4a265 From 99b34b301af91888b32b3395b58aad7eb0ad46b9 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 13:39:09 +0200 Subject: [PATCH 29/60] init test: call Validate on StylusTargetConfig --- cmd/nitro/init_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index af2242af78..eadc6234c0 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -350,6 +350,12 @@ func TestEmptyDatabaseDir(t *testing.T) { } } +func defaultStylusTargetConfigForTest(t *testing.T) *gethexec.StylusTargetConfig { + targetConfig := gethexec.DefaultStylusTargetConfig + Require(t, targetConfig.Validate()) + return &targetConfig +} + func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { t.Parallel() @@ -377,7 +383,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -394,7 +400,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -412,7 +418,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -548,7 +554,7 @@ func TestOpenInitializeChainDbEmptyInit(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, From bf735f466ce13e25006ce2f87f6362692df51659 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 14:15:46 +0200 Subject: [PATCH 30/60] don't call programs.SetTarget for TargetWavm --- execution/gethexec/executionengine.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 192408794b..56b52ed986 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -157,6 +157,9 @@ func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { for _, target := range targets { var effectiveStylusTarget string switch target { + case rawdb.TargetWavm: + // skip wavm target + continue case rawdb.TargetArm64: effectiveStylusTarget = targetConfig.Arm64 case rawdb.TargetAmd64: From 29327a8be174e5c582caef4fbe8d4fd55839688e Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 14:16:57 +0200 Subject: [PATCH 31/60] refactor StylusTargetConfig.Validate --- execution/gethexec/node.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index bc29e9b7f2..01464123fd 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -44,22 +44,24 @@ func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { func (c *StylusTargetConfig) Validate() error { localTarget := rawdb.LocalTarget() - targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)) - targetsSet := make(map[ethdb.WasmTarget]struct{}, len(c.ExtraArchs)+1) + targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) + targetsSet := make(map[ethdb.WasmTarget]bool, len(c.ExtraArchs)) for _, arch := range c.ExtraArchs { target := ethdb.WasmTarget(arch) + if targetsSet[target] { + // skip duplicate + continue + } if !rawdb.IsSupportedWasmTarget(target) { return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) } - if _, duplicate := targetsSet[target]; !duplicate { - targets = append(targets, target) - targetsSet[target] = struct{}{} - } + targets = append(targets, target) + targetsSet[target] = true } - if _, has := targetsSet[rawdb.TargetWavm]; !has { + if !targetsSet[rawdb.TargetWavm] { return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) } - if _, has := targetsSet[localTarget]; !has { + if !targetsSet[localTarget] { targets = append(targets, localTarget) } c.wasmTargets = targets From aba94150dbb0a1b4c6533762284c83948a299e89 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 14:21:21 +0200 Subject: [PATCH 32/60] error for unsupported stylus target in populateStylusTargetCache --- execution/gethexec/executionengine.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 56b52ed986..8594d5867d 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -166,11 +166,13 @@ func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { effectiveStylusTarget = targetConfig.Amd64 case rawdb.TargetHost: effectiveStylusTarget = targetConfig.Host + default: + return fmt.Errorf("unsupported stylus target: %v", target) } isNative := target == localTarget err := programs.SetTarget(target, effectiveStylusTarget, isNative) if err != nil { - return fmt.Errorf("Failed to set stylus target: %w", err) + return fmt.Errorf("failed to set stylus target: %w", err) } nativeSet = nativeSet || isNative } From a69cb63efc66e86bd5f5bce998fdc66998e536ae Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 11:43:11 -0300 Subject: [PATCH 33/60] Replace regex with filepath functions --- cmd/nitro/init.go | 15 +++++++++++++-- cmd/nitro/init_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 435c73c997..e0709bb3c9 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -405,8 +405,19 @@ func databaseIsEmpty(db ethdb.Database) bool { return !it.Next() } +func isWasmDb(path string) bool { + path = filepath.Clean(strings.ToLower(path)) + parts := strings.Split(path, string(filepath.Separator)) + if len(parts) >= 1 && parts[0] == "wasm" { + return true + } + if len(parts) >= 2 && parts[0] == "" && parts[1] == "wasm" { // Cover "/wasm" case + return true + } + return false +} + func extractSnapshot(archive string, location string, importWasm bool) error { - isWasmDB := regexp.MustCompile("^(./)?wasm") reader, err := os.Open(archive) if err != nil { return fmt.Errorf("couln't open init '%v' archive: %w", archive, err) @@ -419,7 +430,7 @@ func extractSnapshot(archive string, location string, importWasm bool) error { var rename extract.Renamer if !importWasm { rename = func(path string) string { - if isWasmDB.MatchString(strings.ToLower(path)) { + if isWasmDb(path) { return "" // do not extract wasm files } return path diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index 9282839f75..7b9153a9cf 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -675,3 +675,35 @@ func TestExtractSnapshot(t *testing.T) { }) } } + +func TestIsWasmDb(t *testing.T) { + testCases := []struct { + path string + want bool + }{ + {"wasm", true}, + {"wasm/", true}, + {"wasm/something", true}, + {"/wasm", true}, + {"./wasm", true}, + {"././wasm", true}, + {"/./wasm", true}, + {"WASM", true}, + {"wAsM", true}, + {"nitro/../wasm", true}, + {"/nitro/../wasm", true}, + {".//nitro/.//../wasm", true}, + {"not-wasm", false}, + {"l2chaindata/example@@", false}, + {"somedir/wasm", false}, + } + for _, testCase := range testCases { + name := fmt.Sprintf("%q", testCase.path) + t.Run(name, func(t *testing.T) { + got := isWasmDb(testCase.path) + if testCase.want != got { + t.Fatalf("want %v, but got %v", testCase.want, got) + } + }) + } +} From 2e9f4a5a1e77c0c63da526d89874f1225550fab0 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 12:07:46 -0300 Subject: [PATCH 34/60] Disallow any wasm prefix --- cmd/nitro/init.go | 4 ++-- cmd/nitro/init_test.go | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index e0709bb3c9..92398c9207 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -408,10 +408,10 @@ func databaseIsEmpty(db ethdb.Database) bool { func isWasmDb(path string) bool { path = filepath.Clean(strings.ToLower(path)) parts := strings.Split(path, string(filepath.Separator)) - if len(parts) >= 1 && parts[0] == "wasm" { + if len(parts) >= 1 && strings.HasPrefix(parts[0], "wasm") { return true } - if len(parts) >= 2 && parts[0] == "" && parts[1] == "wasm" { // Cover "/wasm" case + if len(parts) >= 2 && parts[0] == "" && strings.HasPrefix(parts[1], "wasm") { // Cover "/wasm" case return true } return false diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index 7b9153a9cf..50d88c9b40 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -693,6 +693,9 @@ func TestIsWasmDb(t *testing.T) { {"nitro/../wasm", true}, {"/nitro/../wasm", true}, {".//nitro/.//../wasm", true}, + {"wasm-something", true}, + {"wasm\x00something", true}, + {"wasm\x00test/example", true}, {"not-wasm", false}, {"l2chaindata/example@@", false}, {"somedir/wasm", false}, From 3f9461d132899134e527bd7fdaf84cd8efbdf773 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 28 Aug 2024 22:40:17 +0530 Subject: [PATCH 35/60] Support rebuilding of wasm store on init --- cmd/nitro/init.go | 92 ++++++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 41 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 5573bd6ab2..db9f3bc0fc 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -475,6 +475,51 @@ func validateOrUpgradeWasmStoreSchemaVersion(db ethdb.Database) error { return nil } +func rebuildLocalWasm(ctx context.Context, config *NodeConfig, l2BlockChain *core.BlockChain, chainConfig *params.ChainConfig, chainDb, wasmDb ethdb.Database, rebuildMode string) (ethdb.Database, *core.BlockChain, error) { + var err error + latestBlock := l2BlockChain.CurrentBlock() + if latestBlock == nil || latestBlock.Number.Uint64() <= chainConfig.ArbitrumChainParams.GenesisBlockNum || + types.DeserializeHeaderExtraInformation(latestBlock).ArbOSFormatVersion < params.ArbosVersion_Stylus { + // If there is only genesis block or no blocks in the blockchain, set Rebuilding of wasm store to Done + // If Stylus upgrade hasn't yet happened, skipping rebuilding of wasm store + log.Info("Setting rebuilding of wasm store to done") + if err = gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, gethexec.RebuildingDone); err != nil { + return nil, nil, fmt.Errorf("unable to set rebuilding status of wasm store to done: %w", err) + } + } else if rebuildMode != "false" { + var position common.Hash + if rebuildMode == "force" { + log.Info("Commencing force rebuilding of wasm store by setting codehash position in rebuilding to beginning") + if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { + return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) + } + } else { + position, err = gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingPositionKey) + if err != nil { + log.Info("Unable to get codehash position in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it and starting rebuilding", "err", err) + if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { + return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) + } + } + } + if position != gethexec.RebuildingDone { + startBlockHash, err := gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingStartBlockHashKey) + if err != nil { + log.Info("Unable to get start block hash in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it to latest block hash", "err", err) + if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingStartBlockHashKey, latestBlock.Hash()); err != nil { + return nil, nil, fmt.Errorf("unable to initialize start block hash in rebuilding of wasm store to latest block hash: %w", err) + } + startBlockHash = latestBlock.Hash() + } + log.Info("Starting or continuing rebuilding of wasm store", "codeHash", position, "startBlockHash", startBlockHash) + if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.Execution.RPC.MaxRecreateStateDepth, &config.Execution.StylusTarget, l2BlockChain, position, startBlockHash); err != nil { + return nil, nil, fmt.Errorf("error rebuilding of wasm store: %w", err) + } + } + } + return chainDb, l2BlockChain, nil +} + func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { if !config.Init.Force { if readOnlyDb, err := stack.OpenDatabaseWithFreezerWithExtraOptions("l2chaindata", 0, 0, config.Persistent.Ancient, "l2chaindata/", true, persistentConfig.Pebble.ExtraOptions("l2chaindata")); err == nil { @@ -523,47 +568,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return chainDb, l2BlockChain, fmt.Errorf("failed to recreate missing states: %w", err) } } - latestBlock := l2BlockChain.CurrentBlock() - if latestBlock == nil || latestBlock.Number.Uint64() <= chainConfig.ArbitrumChainParams.GenesisBlockNum || - types.DeserializeHeaderExtraInformation(latestBlock).ArbOSFormatVersion < params.ArbosVersion_Stylus { - // If there is only genesis block or no blocks in the blockchain, set Rebuilding of wasm store to Done - // If Stylus upgrade hasn't yet happened, skipping rebuilding of wasm store - log.Info("Setting rebuilding of wasm store to done") - if err = gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, gethexec.RebuildingDone); err != nil { - return nil, nil, fmt.Errorf("unable to set rebuilding status of wasm store to done: %w", err) - } - } else if config.Init.RebuildLocalWasm != "false" { - var position common.Hash - if config.Init.RebuildLocalWasm == "force" { - log.Info("Commencing force rebuilding of wasm store by setting codehash position in rebuilding to beginning") - if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { - return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) - } - } else { - position, err = gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingPositionKey) - if err != nil { - log.Info("Unable to get codehash position in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it and starting rebuilding", "err", err) - if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { - return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) - } - } - } - if position != gethexec.RebuildingDone { - startBlockHash, err := gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingStartBlockHashKey) - if err != nil { - log.Info("Unable to get start block hash in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it to latest block hash", "err", err) - if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingStartBlockHashKey, latestBlock.Hash()); err != nil { - return nil, nil, fmt.Errorf("unable to initialize start block hash in rebuilding of wasm store to latest block hash: %w", err) - } - startBlockHash = latestBlock.Hash() - } - log.Info("Starting or continuing rebuilding of wasm store", "codeHash", position, "startBlockHash", startBlockHash) - if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.Execution.RPC.MaxRecreateStateDepth, &config.Execution.StylusTarget, l2BlockChain, position, startBlockHash); err != nil { - return nil, nil, fmt.Errorf("error rebuilding of wasm store: %w", err) - } - } - } - return chainDb, l2BlockChain, nil + return rebuildLocalWasm(ctx, config, l2BlockChain, chainConfig, chainDb, wasmDb, config.Init.RebuildLocalWasm) } readOnlyDb.Close() } else if !dbutil.IsNotExistError(err) { @@ -795,6 +800,11 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return chainDb, l2BlockChain, err } + if config.Init.RebuildLocalWasm != "false" { + // In order to rebuild wasm store correctly we have to use force option + return rebuildLocalWasm(ctx, config, l2BlockChain, chainConfig, chainDb, wasmDb, "force") + } + return chainDb, l2BlockChain, nil } From 2499675af22d1c760a59dbed4b21cc6cd3821977 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 28 Aug 2024 22:46:31 +0530 Subject: [PATCH 36/60] update flag description --- cmd/conf/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index f360691693..208969fce2 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -86,7 +86,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Int64(prefix+".reorg-to-message-batch", InitConfigDefault.ReorgToMessageBatch, "rolls back the blockchain to the first batch at or before a given message index") f.Int64(prefix+".reorg-to-block-batch", InitConfigDefault.ReorgToBlockBatch, "rolls back the blockchain to the first batch at or before a given block number") f.String(prefix+".rebuild-local-wasm", InitConfigDefault.RebuildLocalWasm, "rebuild local wasm database on boot if needed (otherwise-will be done lazily). Three modes are supported \n"+ - "\"auto\"- (enabled by default) if any previous rebuilding attempt was successful then rebuilding is disabled else continues to rebuild,\n"+ + "\"auto\"- (enabled by default) if any previous rebuilding attempt was successful then rebuilding is disabled else continues to rebuild. Is equivalent to force when starting out with --init.force or downloading db snapshot via init,\n"+ "\"force\"- force rebuilding which would commence rebuilding despite the status of previous attempts,\n"+ "\"false\"- do not rebuild on startup", ) From caa421013a96664a5f71653bde8a392165a78421 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 15:19:26 -0300 Subject: [PATCH 37/60] Revert "Disallow any wasm prefix" This reverts commit 2e9f4a5a1e77c0c63da526d89874f1225550fab0. --- cmd/nitro/init.go | 4 ++-- cmd/nitro/init_test.go | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 92398c9207..e0709bb3c9 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -408,10 +408,10 @@ func databaseIsEmpty(db ethdb.Database) bool { func isWasmDb(path string) bool { path = filepath.Clean(strings.ToLower(path)) parts := strings.Split(path, string(filepath.Separator)) - if len(parts) >= 1 && strings.HasPrefix(parts[0], "wasm") { + if len(parts) >= 1 && parts[0] == "wasm" { return true } - if len(parts) >= 2 && parts[0] == "" && strings.HasPrefix(parts[1], "wasm") { // Cover "/wasm" case + if len(parts) >= 2 && parts[0] == "" && parts[1] == "wasm" { // Cover "/wasm" case return true } return false diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index 50d88c9b40..7b9153a9cf 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -693,9 +693,6 @@ func TestIsWasmDb(t *testing.T) { {"nitro/../wasm", true}, {"/nitro/../wasm", true}, {".//nitro/.//../wasm", true}, - {"wasm-something", true}, - {"wasm\x00something", true}, - {"wasm\x00test/example", true}, {"not-wasm", false}, {"l2chaindata/example@@", false}, {"somedir/wasm", false}, From 9644416f86ac09478f99b6c56fe3784229318abf Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 18:38:06 -0300 Subject: [PATCH 38/60] Improve out of gas warning log For the setTrieSlots and setTransientBytes32 HostIOs, when there is an out-of-gas error, Nitro emits a warning line with `Caused by:\n \x02`. This commit improves this error message by printing the debug representation of the API status code and the warning becomes: `Caused by:\n OutOfGas`. This commit also solves a potential panic when the API does not return the status code. --- arbitrator/arbutil/src/evm/req.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/arbitrator/arbutil/src/evm/req.rs b/arbitrator/arbutil/src/evm/req.rs index b1c8d99972..f76d1407f1 100644 --- a/arbitrator/arbutil/src/evm/req.rs +++ b/arbitrator/arbutil/src/evm/req.rs @@ -140,8 +140,13 @@ impl> EvmApi for EvmApiRequestor { } let (res, _, cost) = self.request(EvmApiMethod::SetTrieSlots, data); - if res[0] != EvmApiStatus::Success.into() { - bail!("{}", String::from_utf8_or_hex(res)); + let status = res + .first() + .copied() + .map(EvmApiStatus::from) + .unwrap_or(EvmApiStatus::Failure); + if status != EvmApiStatus::Success { + bail!("{:?}", status); } Ok(cost) } @@ -156,8 +161,13 @@ impl> EvmApi for EvmApiRequestor { data.extend(key); data.extend(value); let (res, ..) = self.request(EvmApiMethod::SetTransientBytes32, data); - if res[0] != EvmApiStatus::Success.into() { - bail!("{}", String::from_utf8_or_hex(res)); + let status = res + .first() + .copied() + .map(EvmApiStatus::from) + .unwrap_or(EvmApiStatus::Failure); + if status != EvmApiStatus::Success { + bail!("{:?}", status); } Ok(()) } From 03dd3abfd7fb27f5be4bfffa6f0cb8e9deadd8ef Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 22:20:57 -0300 Subject: [PATCH 39/60] Remove unused import --- arbitrator/arbutil/src/evm/req.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/arbitrator/arbutil/src/evm/req.rs b/arbitrator/arbutil/src/evm/req.rs index f76d1407f1..287db357f3 100644 --- a/arbitrator/arbutil/src/evm/req.rs +++ b/arbitrator/arbutil/src/evm/req.rs @@ -7,7 +7,6 @@ use crate::{ storage::{StorageCache, StorageWord}, user::UserOutcomeKind, }, - format::Utf8OrHex, pricing::EVM_API_INK, Bytes20, Bytes32, }; From f6efc1b491a39e0eb52e62dc7dc01cda829bd168 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 22:25:41 -0300 Subject: [PATCH 40/60] Add comment about case-insensitive file systems --- cmd/nitro/init.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index e0709bb3c9..b338e0a49e 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -406,7 +406,8 @@ func databaseIsEmpty(db ethdb.Database) bool { } func isWasmDb(path string) bool { - path = filepath.Clean(strings.ToLower(path)) + path = strings.ToLower(path) // lowers the path to handle case-insensitive file systems + path = filepath.Clean(path) parts := strings.Split(path, string(filepath.Separator)) if len(parts) >= 1 && parts[0] == "wasm" { return true From 972508da88fb9797c3151cbdc23349b096a098f5 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 29 Aug 2024 09:44:34 +0530 Subject: [PATCH 41/60] address PR comments --- cmd/nitro/init.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index db9f3bc0fc..6ae3c9125d 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -475,10 +475,10 @@ func validateOrUpgradeWasmStoreSchemaVersion(db ethdb.Database) error { return nil } -func rebuildLocalWasm(ctx context.Context, config *NodeConfig, l2BlockChain *core.BlockChain, chainConfig *params.ChainConfig, chainDb, wasmDb ethdb.Database, rebuildMode string) (ethdb.Database, *core.BlockChain, error) { +func rebuildLocalWasm(ctx context.Context, config *gethexec.Config, l2BlockChain *core.BlockChain, chainDb, wasmDb ethdb.Database, rebuildMode string) (ethdb.Database, *core.BlockChain, error) { var err error latestBlock := l2BlockChain.CurrentBlock() - if latestBlock == nil || latestBlock.Number.Uint64() <= chainConfig.ArbitrumChainParams.GenesisBlockNum || + if latestBlock == nil || latestBlock.Number.Uint64() <= l2BlockChain.Config().ArbitrumChainParams.GenesisBlockNum || types.DeserializeHeaderExtraInformation(latestBlock).ArbOSFormatVersion < params.ArbosVersion_Stylus { // If there is only genesis block or no blocks in the blockchain, set Rebuilding of wasm store to Done // If Stylus upgrade hasn't yet happened, skipping rebuilding of wasm store @@ -512,7 +512,7 @@ func rebuildLocalWasm(ctx context.Context, config *NodeConfig, l2BlockChain *cor startBlockHash = latestBlock.Hash() } log.Info("Starting or continuing rebuilding of wasm store", "codeHash", position, "startBlockHash", startBlockHash) - if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.Execution.RPC.MaxRecreateStateDepth, &config.Execution.StylusTarget, l2BlockChain, position, startBlockHash); err != nil { + if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.RPC.MaxRecreateStateDepth, &config.StylusTarget, l2BlockChain, position, startBlockHash); err != nil { return nil, nil, fmt.Errorf("error rebuilding of wasm store: %w", err) } } @@ -568,7 +568,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return chainDb, l2BlockChain, fmt.Errorf("failed to recreate missing states: %w", err) } } - return rebuildLocalWasm(ctx, config, l2BlockChain, chainConfig, chainDb, wasmDb, config.Init.RebuildLocalWasm) + return rebuildLocalWasm(ctx, &config.Execution, l2BlockChain, chainDb, wasmDb, config.Init.RebuildLocalWasm) } readOnlyDb.Close() } else if !dbutil.IsNotExistError(err) { @@ -800,12 +800,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return chainDb, l2BlockChain, err } - if config.Init.RebuildLocalWasm != "false" { - // In order to rebuild wasm store correctly we have to use force option - return rebuildLocalWasm(ctx, config, l2BlockChain, chainConfig, chainDb, wasmDb, "force") - } - - return chainDb, l2BlockChain, nil + return rebuildLocalWasm(ctx, &config.Execution, l2BlockChain, chainDb, wasmDb, config.Init.RebuildLocalWasm) } func testTxIndexUpdated(chainDb ethdb.Database, lastBlock uint64) bool { From a710d0c4a9203af55e9a43b4687647e3d1444a55 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 29 Aug 2024 09:46:29 +0530 Subject: [PATCH 42/60] undo flag description --- cmd/conf/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index 208969fce2..f360691693 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -86,7 +86,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Int64(prefix+".reorg-to-message-batch", InitConfigDefault.ReorgToMessageBatch, "rolls back the blockchain to the first batch at or before a given message index") f.Int64(prefix+".reorg-to-block-batch", InitConfigDefault.ReorgToBlockBatch, "rolls back the blockchain to the first batch at or before a given block number") f.String(prefix+".rebuild-local-wasm", InitConfigDefault.RebuildLocalWasm, "rebuild local wasm database on boot if needed (otherwise-will be done lazily). Three modes are supported \n"+ - "\"auto\"- (enabled by default) if any previous rebuilding attempt was successful then rebuilding is disabled else continues to rebuild. Is equivalent to force when starting out with --init.force or downloading db snapshot via init,\n"+ + "\"auto\"- (enabled by default) if any previous rebuilding attempt was successful then rebuilding is disabled else continues to rebuild,\n"+ "\"force\"- force rebuilding which would commence rebuilding despite the status of previous attempts,\n"+ "\"false\"- do not rebuild on startup", ) From 138bcb7c5c3c5ab4427be249d48430afbc0a5699 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 29 Aug 2024 10:47:20 -0500 Subject: [PATCH 43/60] Add safety check that min basefee must be positive for gas estimation --- precompiles/ArbOwner.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index 066fc0a4c4..8b87445e0e 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -69,6 +69,9 @@ func (con ArbOwner) SetL2BaseFee(c ctx, evm mech, priceInWei huge) error { // SetMinimumL2BaseFee sets the minimum base fee needed for a transaction to succeed func (con ArbOwner) SetMinimumL2BaseFee(c ctx, evm mech, priceInWei huge) error { + if c.txProcessor.MsgIsNonMutating() && priceInWei.Sign() == 0 { + return errors.New("minimum base fee must be nonzero") + } return c.State.L2PricingState().SetMinBaseFeeWei(priceInWei) } From 7a4fe007e5e3edb198130265ad167dc39d531073 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 29 Aug 2024 21:21:50 +0200 Subject: [PATCH 44/60] simplify StylusTargetConfig.Validate --- execution/gethexec/node.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 01464123fd..21c2b4bece 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -43,26 +43,21 @@ func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { } func (c *StylusTargetConfig) Validate() error { - localTarget := rawdb.LocalTarget() - targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) targetsSet := make(map[ethdb.WasmTarget]bool, len(c.ExtraArchs)) for _, arch := range c.ExtraArchs { target := ethdb.WasmTarget(arch) - if targetsSet[target] { - // skip duplicate - continue - } if !rawdb.IsSupportedWasmTarget(target) { return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) } - targets = append(targets, target) targetsSet[target] = true } if !targetsSet[rawdb.TargetWavm] { return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) } - if !targetsSet[localTarget] { - targets = append(targets, localTarget) + targetsSet[rawdb.LocalTarget()] = true + targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) + for target := range targetsSet { + targets = append(targets, target) } c.wasmTargets = targets return nil From 01c91c72db803410631f51b8614cd69021c8e786 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 30 Aug 2024 10:11:46 -0300 Subject: [PATCH 45/60] Use pull_request_target instead of pull_request in submodule pin check --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 870f520877..f045f71f68 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request: + pull_request_target: branches: [ master ] types: [synchronize, opened, reopened] From ae67f4e575ac90b97119429ebf2dccc5fcc0ef17 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Fri, 30 Aug 2024 12:11:40 -0600 Subject: [PATCH 46/60] redis consimer: workers and beffers --- validator/valnode/redis/consumer.go | 69 ++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 16 deletions(-) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index fb7db1e870..8458ec021a 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -3,6 +3,7 @@ package redis import ( "context" "fmt" + "runtime" "time" "github.com/ethereum/go-ethereum/common" @@ -22,8 +23,9 @@ type ValidationServer struct { spawner validator.ValidationSpawner // consumers stores moduleRoot to consumer mapping. - consumers map[common.Hash]*pubsub.Consumer[*validator.ValidationInput, validator.GoGlobalState] - streamTimeout time.Duration + consumers map[common.Hash]*pubsub.Consumer[*validator.ValidationInput, validator.GoGlobalState] + + config *ValidationServerConfig } func NewValidationServer(cfg *ValidationServerConfig, spawner validator.ValidationSpawner) (*ValidationServer, error) { @@ -44,9 +46,9 @@ func NewValidationServer(cfg *ValidationServerConfig, spawner validator.Validati consumers[mr] = c } return &ValidationServer{ - consumers: consumers, - spawner: spawner, - streamTimeout: cfg.StreamTimeout, + consumers: consumers, + spawner: spawner, + config: cfg, }, nil } @@ -54,6 +56,19 @@ func (s *ValidationServer) Start(ctx_in context.Context) { s.StopWaiter.Start(ctx_in, s) // Channel that all consumers use to indicate their readiness. readyStreams := make(chan struct{}, len(s.consumers)) + type workUnit struct { + req *pubsub.Message[*validator.ValidationInput] + moduleRoot common.Hash + } + workers := s.config.Workers + if workers == 0 { + workers = runtime.NumCPU() + } + bufferSize := 0 + if s.config.BufferReads { + bufferSize = workers + } + workQueue := make(chan workUnit, bufferSize) for moduleRoot, c := range s.consumers { c := c moduleRoot := moduleRoot @@ -93,17 +108,11 @@ func (s *ValidationServer) Start(ctx_in context.Context) { // There's nothing in the queue. return time.Second } - valRun := s.spawner.Launch(req.Value, moduleRoot) - res, err := valRun.Await(ctx) - if err != nil { - log.Error("Error validating", "request value", req.Value, "error", err) - return 0 - } - if err := c.SetResult(ctx, req.ID, res); err != nil { - log.Error("Error setting result for request", "id", req.ID, "result", res, "error", err) - return 0 + select { + case <-ctx.Done(): + case workQueue <- workUnit{req, moduleRoot}: } - return time.Second + return 0 }) }) } @@ -113,7 +122,7 @@ func (s *ValidationServer) Start(ctx_in context.Context) { case <-readyStreams: log.Trace("At least one stream is ready") return // Don't block Start if at least one of the stream is ready. - case <-time.After(s.streamTimeout): + case <-time.After(s.config.StreamTimeout): log.Error("Waiting for redis streams timed out") case <-ctx.Done(): log.Info("Context done while waiting redis streams to be ready, failed to start") @@ -121,6 +130,26 @@ func (s *ValidationServer) Start(ctx_in context.Context) { } } }) + for i := 0; i < workers; i++ { + s.StopWaiter.LaunchThread(func(ctx context.Context) { + for { + var work workUnit + select { + case <-ctx.Done(): + return + case work = <-workQueue: + } + valRun := s.spawner.Launch(work.req.Value, work.moduleRoot) + res, err := valRun.Await(ctx) + if err != nil { + log.Error("Error validating", "request value", work.req.Value, "error", err) + } + if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { + log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } + } + }) + } } type ValidationServerConfig struct { @@ -131,6 +160,8 @@ type ValidationServerConfig struct { // Timeout on polling for existence of each redis stream. StreamTimeout time.Duration `koanf:"stream-timeout"` StreamPrefix string `koanf:"stream-prefix"` + Workers int `koanf:"workers"` + BufferReads bool `koanf:"buffer-reads"` } var DefaultValidationServerConfig = ValidationServerConfig{ @@ -139,6 +170,8 @@ var DefaultValidationServerConfig = ValidationServerConfig{ ConsumerConfig: pubsub.DefaultConsumerConfig, ModuleRoots: []string{}, StreamTimeout: 10 * time.Minute, + Workers: 1, + BufferReads: true, } var TestValidationServerConfig = ValidationServerConfig{ @@ -147,6 +180,8 @@ var TestValidationServerConfig = ValidationServerConfig{ ConsumerConfig: pubsub.TestConsumerConfig, ModuleRoots: []string{}, StreamTimeout: time.Minute, + Workers: 1, + BufferReads: true, } func ValidationServerConfigAddOptions(prefix string, f *pflag.FlagSet) { @@ -155,6 +190,8 @@ func ValidationServerConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".redis-url", DefaultValidationServerConfig.RedisURL, "url of redis server") f.String(prefix+".stream-prefix", DefaultValidationServerConfig.StreamPrefix, "prefix for stream name") f.Duration(prefix+".stream-timeout", DefaultValidationServerConfig.StreamTimeout, "Timeout on polling for existence of redis streams") + f.Int(prefix+".workers", DefaultValidationServerConfig.Workers, "number of validation threads (0 to use number of CPUs)") + f.Bool(prefix+".buffer-reads", DefaultValidationServerConfig.BufferReads, "buffer reads (read next while working)") } func (cfg *ValidationServerConfig) Enabled() bool { From d7ee2eed4b5e07c609d95d6cba60e625d9df67ae Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Fri, 30 Aug 2024 15:41:57 -0600 Subject: [PATCH 47/60] redis consumer: use tokens for buffer, default 0 --- validator/valnode/redis/consumer.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 8458ec021a..7456466533 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -64,11 +64,15 @@ func (s *ValidationServer) Start(ctx_in context.Context) { if workers == 0 { workers = runtime.NumCPU() } - bufferSize := 0 + workQueue := make(chan workUnit, workers) + tokensCount := workers if s.config.BufferReads { - bufferSize = workers + tokensCount += workers + } + requestTokenQueue := make(chan struct{}, tokensCount) + for i := 0; i < tokensCount; i++ { + requestTokenQueue <- struct{}{} } - workQueue := make(chan workUnit, bufferSize) for moduleRoot, c := range s.consumers { c := c moduleRoot := moduleRoot @@ -99,6 +103,11 @@ func (s *ValidationServer) Start(ctx_in context.Context) { case <-ready: // Wait until the stream exists and start consuming iteratively. } s.StopWaiter.CallIteratively(func(ctx context.Context) time.Duration { + select { + case <-ctx.Done(): + return 0 + case <-requestTokenQueue: + } req, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) @@ -147,6 +156,11 @@ func (s *ValidationServer) Start(ctx_in context.Context) { if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) } + select { + case <-ctx.Done(): + return + case requestTokenQueue <- struct{}{}: + } } }) } @@ -170,7 +184,7 @@ var DefaultValidationServerConfig = ValidationServerConfig{ ConsumerConfig: pubsub.DefaultConsumerConfig, ModuleRoots: []string{}, StreamTimeout: 10 * time.Minute, - Workers: 1, + Workers: 0, BufferReads: true, } From 0d08e481f039c778ff0f9a7c60f38167eadb29f2 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 30 Aug 2024 20:46:02 -0300 Subject: [PATCH 48/60] Also uses DataPoster when only creating validator wallet contract and exiting --- arbnode/node.go | 23 ++++++++++ cmd/nitro/nitro.go | 15 ++++++- staker/validatorwallet/contract.go | 72 +++++++++++++++--------------- system_tests/fast_confirm_test.go | 54 +++++++++++----------- system_tests/staker_test.go | 25 ++++++++--- 5 files changed, 119 insertions(+), 70 deletions(-) diff --git a/arbnode/node.go b/arbnode/node.go index 93b58e800f..e13f92f30f 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -339,6 +339,29 @@ func checkArbDbSchemaVersion(arbDb ethdb.Database) error { return nil } +func DataposterOnlyUsedToCreateValidatorWalletContract( + ctx context.Context, + l1Reader *headerreader.HeaderReader, + transactOpts *bind.TransactOpts, + cfg *dataposter.DataPosterConfig, + parentChainID *big.Int, +) (*dataposter.DataPoster, error) { + return dataposter.NewDataPoster(ctx, + &dataposter.DataPosterOpts{ + HeaderReader: l1Reader, + Auth: transactOpts, + Config: func() *dataposter.DataPosterConfig { + cfg.UseNoOpStorage = true + return cfg + }, + MetadataRetriever: func(ctx context.Context, blockNum *big.Int) ([]byte, error) { + return nil, nil + }, + ParentChainID: parentChainID, + }, + ) +} + func StakerDataposter( ctx context.Context, db ethdb.Database, l1Reader *headerreader.HeaderReader, transactOpts *bind.TransactOpts, cfgFetcher ConfigFetcher, syncMonitor *SyncMonitor, diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 8c26b95a92..bad48f430c 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -371,8 +371,21 @@ func mainImpl() int { if err != nil { log.Crit("error getting rollup addresses config", "err", err) } + + dataPoster, err := arbnode.DataposterOnlyUsedToCreateValidatorWalletContract( + ctx, + l1Reader, + l1TransactionOptsBatchPoster, + &nodeConfig.Node.Staker.DataPoster, + new(big.Int).SetUint64(nodeConfig.ParentChain.ID), + ) + if err != nil { + log.Crit("error creating data poster to create validator wallet contract", "err", err) + } + getExtraGas := func() uint64 { return nodeConfig.Node.Staker.ExtraGas } + // #nosec G115 - addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, nil) + addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, dataPoster, getExtraGas) if err != nil { log.Crit("error creating validator wallet contract", "error", err, "address", l1TransactionOptsValidator.From.Hex()) } diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 495e796cd2..3db35d8232 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -155,16 +155,19 @@ func (v *Contract) From() common.Address { } // nil value == 0 value -func (v *Contract) getAuth(ctx context.Context, value *big.Int) (*bind.TransactOpts, error) { - newAuth := *v.auth - newAuth.Context = ctx - newAuth.Value = value - nonce, err := v.L1Client().NonceAt(ctx, v.auth.From, nil) +func getAuthWithUpdatedNonce(ctx context.Context, l1Reader *headerreader.HeaderReader, auth bind.TransactOpts, value *big.Int) (*bind.TransactOpts, error) { + auth.Context = ctx + auth.Value = value + nonce, err := l1Reader.Client().NonceAt(ctx, auth.From, nil) if err != nil { return nil, err } - newAuth.Nonce = new(big.Int).SetUint64(nonce) - return &newAuth, nil + auth.Nonce = new(big.Int).SetUint64(nonce) + return &auth, nil +} + +func (v *Contract) getAuth(ctx context.Context, value *big.Int) (*bind.TransactOpts, error) { + return getAuthWithUpdatedNonce(ctx, v.l1Reader, *v.auth, value) } func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction, gasRefunder common.Address) (*types.Transaction, error) { @@ -183,9 +186,12 @@ func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), *v.Address(), data, gas, auth.Value) } -// Exposed for tests -func (v *Contract) CreateWalletContract( +func createWalletContract( ctx context.Context, + l1Reader *headerreader.HeaderReader, + auth *bind.TransactOpts, + dataPoster *dataposter.DataPoster, + getExtraGas func() uint64, validatorWalletFactoryAddr common.Address, ) (*types.Transaction, error) { var initialExecutorAllowedDests []common.Address @@ -194,24 +200,19 @@ func (v *Contract) CreateWalletContract( return nil, err } - auth, err := v.getAuth(ctx, nil) - if err != nil { - return nil, err - } - gas, err := gasForTxData( ctx, - v.l1Reader, + l1Reader, auth, &validatorWalletFactoryAddr, txData, - v.getExtraGas, + getExtraGas, ) if err != nil { return nil, fmt.Errorf("getting gas for tx data: %w", err) } - return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) + return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) } func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) error { @@ -225,15 +226,10 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err return nil } if v.address.Load() == nil { - auth, err := v.getAuth(ctx, nil) - if err != nil { - return err - } - - // By passing v.CreateWalletContract as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. + // By passing v.dataPoster as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. // DataPoster keeps in its internal state information related to the transactions sent through it, which is used to infer the expected nonce in a transaction for example. // If a transaction is sent using the Staker's DataPoster key, but not through the Staker's DataPoster object, DataPoster's internal state will be outdated, which can compromise the expected nonce inference. - addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing, v.CreateWalletContract) + addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, v.auth, v.l1Reader, createIfMissing, v.dataPoster, v.getExtraGas) if err != nil { return err } @@ -439,6 +435,11 @@ func (b *Contract) DataPoster() *dataposter.DataPoster { return b.dataPoster } +// Exported for testing +func (b *Contract) GetExtraGas() func() uint64 { + return b.getExtraGas +} + func GetValidatorWalletContract( ctx context.Context, validatorWalletFactoryAddr common.Address, @@ -446,7 +447,8 @@ func GetValidatorWalletContract( transactAuth *bind.TransactOpts, l1Reader *headerreader.HeaderReader, createIfMissing bool, - createWalletFunc func(context.Context, common.Address) (*types.Transaction, error), + dataPoster *dataposter.DataPoster, + getExtraGas func() uint64, ) (*common.Address, error) { client := l1Reader.Client() @@ -483,18 +485,14 @@ func GetValidatorWalletContract( return nil, nil } - var tx *types.Transaction - if createWalletFunc == nil { - var initialExecutorAllowedDests []common.Address - tx, err = walletCreator.CreateWallet(transactAuth, initialExecutorAllowedDests) - if err != nil { - return nil, err - } - } else { - tx, err = createWalletFunc(ctx, validatorWalletFactoryAddr) - if err != nil { - return nil, err - } + transactAuth, err = getAuthWithUpdatedNonce(ctx, l1Reader, *transactAuth, nil) + if err != nil { + return nil, err + } + + tx, err := createWalletContract(ctx, l1Reader, transactAuth, dataPoster, getExtraGas, validatorWalletFactoryAddr) + if err != nil { + return nil, err } receipt, err := l1Reader.WaitForTxApproval(ctx, tx) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index ae624be1e9..4598827418 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -115,10 +115,10 @@ func TestFastConfirmation(t *testing.T) { Require(t, err) valConfig.Strategy = "MakeNodes" - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) if valWalletAddr == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) @@ -278,15 +278,6 @@ func TestFastConfirmationWithSafe(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - if valWalletAddrA == *valWalletAddrCheck { - Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) - } - rollup, err := rollupgen.NewRollupAdminLogic(l2nodeA.DeployInfo.Rollup, builder.L1.Client) Require(t, err) @@ -295,28 +286,13 @@ func TestFastConfirmationWithSafe(t *testing.T) { rollupABI, err := abi.JSON(strings.NewReader(rollupgen.RollupAdminLogicABI)) Require(t, err, "unable to parse rollup ABI") - safeAddress := deploySafe(t, builder.L1, builder.L1.Client, deployAuth, []common.Address{valWalletAddrA, srv.Address}) - setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address, safeAddress}, []bool{true, true, true, true}) - Require(t, err, "unable to generate setValidator calldata") - tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) - Require(t, err, "unable to set validators") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - setMinAssertPeriodCalldata, err := rollupABI.Pack("setMinimumAssertionPeriod", big.NewInt(1)) Require(t, err, "unable to generate setMinimumAssertionPeriod calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) + tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) Require(t, err, "unable to set minimum assertion period") _, err = builder.L1.EnsureTxSucceeded(tx) Require(t, err) - setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", safeAddress) - Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) - Require(t, err, "unable to set anytrust fast confirmer") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - valConfigA := staker.TestL1ValidatorConfig valConfigA.EnableFastConfirmation = true @@ -339,6 +315,30 @@ func TestFastConfirmationWithSafe(t *testing.T) { Require(t, err) valConfigA.Strategy = "MakeNodes" + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + Require(t, err) + valWalletAddrA := *valWalletAddrAPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + Require(t, err) + if valWalletAddrA == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) + } + + safeAddress := deploySafe(t, builder.L1, builder.L1.Client, deployAuth, []common.Address{valWalletAddrA, srv.Address}) + setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address, safeAddress}, []bool{true, true, true, true}) + Require(t, err, "unable to generate setValidator calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) + Require(t, err, "unable to set validators") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + + setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", safeAddress) + Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) + Require(t, err, "unable to set anytrust fast confirmer") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + _, valStack := createTestValidationNode(t, ctx, &valnode.TestValidationConfig) blockValidatorConfig := staker.TestBlockValidatorConfig diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index 3fdf82ca74..e61dcedc2b 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -176,10 +176,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) valConfigA.Strategy = "MakeNodes" } - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) @@ -476,7 +476,7 @@ func TestStakersCooperative(t *testing.T) { stakerTestImpl(t, false, false) } -func TestGetValidatorWalletContractWithoutDataPoster(t *testing.T) { +func TestGetValidatorWalletContractWithDataposterOnlyUsedToCreateValidatorWalletContract(t *testing.T) { t.Parallel() ctx, cancelCtx := context.WithCancel(context.Background()) @@ -492,10 +492,25 @@ func TestGetValidatorWalletContractWithoutDataPoster(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "ValidatorA", balance, builder.L1Info) l1auth := builder.L1Info.GetDefaultTransactOpts("ValidatorA", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + parentChainID, err := builder.L1.Client.ChainID(ctx) + Require(t, err) + + dataPoster, err := arbnode.DataposterOnlyUsedToCreateValidatorWalletContract( + ctx, + builder.L2.ConsensusNode.L1Reader, + &l1auth, + &builder.nodeConfig.Staker.DataPoster, + parentChainID, + ) + if err != nil { + log.Crit("error creating data poster to create validator wallet contract", "err", err) + } + getExtraGas := func() uint64 { return builder.nodeConfig.Staker.ExtraGas } + + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) From 6eae487866df45f46db437141e30254c0eefa4c6 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 30 Aug 2024 20:47:15 -0300 Subject: [PATCH 49/60] Creates validator wallet contract with transaction with value set to zero --- staker/validatorwallet/contract.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 3db35d8232..8a522b44a3 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -212,7 +212,7 @@ func createWalletContract( return nil, fmt.Errorf("getting gas for tx data: %w", err) } - return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) + return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, common.Big0) } func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) error { From caadac62a7be06a2296a0c1c0db0330e5745d800 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 30 Aug 2024 21:05:48 -0300 Subject: [PATCH 50/60] Improves error message when getting gas to create validator wallet --- staker/validatorwallet/contract.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 8a522b44a3..1910db1d63 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -209,7 +209,7 @@ func createWalletContract( getExtraGas, ) if err != nil { - return nil, fmt.Errorf("getting gas for tx data: %w", err) + return nil, fmt.Errorf("getting gas for tx data when creating validator wallet, validatorWalletFactory=%v: %w", validatorWalletFactoryAddr, err) } return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, common.Big0) From 716bfea74964a6f2e51a2a7f119a1d9f9d314149 Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Mon, 2 Sep 2024 12:23:03 +0200 Subject: [PATCH 51/60] Add Mac OSX environment variables The unfortunate thing is that it would still be good for developers to set these in their shell environment so that if they do end up running any "go" commands directly from the CLI, they won't be bombarded with all the linker warnings. But, at least this makes the default experience of using the make files a little nicer. --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index e46bdbbe61..0a71d64f12 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,14 @@ ifneq ($(origin GOLANG_LDFLAGS),undefined) GOLANG_PARAMS = -ldflags="-extldflags '-ldl' $(GOLANG_LDFLAGS)" endif +UNAME_S := $(shell uname -s) + +# In Mac OSX, there are a lot of warnings emitted if these environment variables aren't set. +ifeq ($(UNAME_S), Darwin) + export MACOSX_DEPLOYMENT_TARGET := $(shell sw_vers -productVersion) + export CGO_LDFLAGS := -Wl,-no_warn_duplicate_libraries +endif + precompile_names = AddressTable Aggregator BLS Debug FunctionTable GasInfo Info osTest Owner RetryableTx Statistics Sys precompiles = $(patsubst %,./solgen/generated/%.go, $(precompile_names)) From 33337b8cd074c4a2ddf38dbcbe574e9712d1123f Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 10:39:38 -0300 Subject: [PATCH 52/60] Renames getAuthWithUpdatedNonce to getAuthWithUpdatedNonceFromL1 --- staker/validatorwallet/contract.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 1910db1d63..83dedcad84 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -155,7 +155,7 @@ func (v *Contract) From() common.Address { } // nil value == 0 value -func getAuthWithUpdatedNonce(ctx context.Context, l1Reader *headerreader.HeaderReader, auth bind.TransactOpts, value *big.Int) (*bind.TransactOpts, error) { +func getAuthWithUpdatedNonceFromL1(ctx context.Context, l1Reader *headerreader.HeaderReader, auth bind.TransactOpts, value *big.Int) (*bind.TransactOpts, error) { auth.Context = ctx auth.Value = value nonce, err := l1Reader.Client().NonceAt(ctx, auth.From, nil) @@ -167,7 +167,7 @@ func getAuthWithUpdatedNonce(ctx context.Context, l1Reader *headerreader.HeaderR } func (v *Contract) getAuth(ctx context.Context, value *big.Int) (*bind.TransactOpts, error) { - return getAuthWithUpdatedNonce(ctx, v.l1Reader, *v.auth, value) + return getAuthWithUpdatedNonceFromL1(ctx, v.l1Reader, *v.auth, value) } func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction, gasRefunder common.Address) (*types.Transaction, error) { @@ -485,7 +485,7 @@ func GetValidatorWalletContract( return nil, nil } - transactAuth, err = getAuthWithUpdatedNonce(ctx, l1Reader, *transactAuth, nil) + transactAuth, err = getAuthWithUpdatedNonceFromL1(ctx, l1Reader, *transactAuth, nil) if err != nil { return nil, err } From 9150808093cbfbc2fb98c9bb44e34b3b4ad927a7 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 16:12:49 -0300 Subject: [PATCH 53/60] Don't modify config inside config fetcher --- arbnode/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbnode/node.go b/arbnode/node.go index e13f92f30f..a9da4ea24b 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -346,12 +346,12 @@ func DataposterOnlyUsedToCreateValidatorWalletContract( cfg *dataposter.DataPosterConfig, parentChainID *big.Int, ) (*dataposter.DataPoster, error) { + cfg.UseNoOpStorage = true return dataposter.NewDataPoster(ctx, &dataposter.DataPosterOpts{ HeaderReader: l1Reader, Auth: transactOpts, Config: func() *dataposter.DataPosterConfig { - cfg.UseNoOpStorage = true return cfg }, MetadataRetriever: func(ctx context.Context, blockNum *big.Int) ([]byte, error) { From 1bc34f20ff8f21a7c860dc059393309fd11d1d00 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 2 Sep 2024 14:23:02 -0500 Subject: [PATCH 54/60] Close opened files explicitly instead of relying on the GC --- arbnode/resourcemanager/resource_management.go | 2 ++ cmd/nitro/init.go | 1 + 2 files changed, 3 insertions(+) diff --git a/arbnode/resourcemanager/resource_management.go b/arbnode/resourcemanager/resource_management.go index aba823cc25..249b689443 100644 --- a/arbnode/resourcemanager/resource_management.go +++ b/arbnode/resourcemanager/resource_management.go @@ -256,6 +256,7 @@ func readIntFromFile(fileName string) (int, error) { if err != nil { return 0, err } + defer file.Close() var limit int if _, err = fmt.Fscanf(file, "%d", &limit); err != nil { @@ -269,6 +270,7 @@ func readFromMemStats(fileName string, re *regexp.Regexp) (int, error) { if err != nil { return 0, err } + defer file.Close() scanner := bufio.NewScanner(file) for scanner.Scan() { diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 26cd698efc..51d895c069 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -423,6 +423,7 @@ func extractSnapshot(archive string, location string, importWasm bool) error { if err != nil { return fmt.Errorf("couln't open init '%v' archive: %w", archive, err) } + defer reader.Close() stat, err := reader.Stat() if err != nil { return err From 0bf0b9accf2ca8b6e6ef81d2ac6f9fa67bd6bdaa Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 18:07:15 -0300 Subject: [PATCH 55/60] Uses auth from data poster in GetValidatorWalletContract --- cmd/nitro/nitro.go | 2 +- staker/validatorwallet/contract.go | 4 ++-- system_tests/fast_confirm_test.go | 8 ++++---- system_tests/staker_test.go | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index bad48f430c..0cf81c45e9 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -385,7 +385,7 @@ func mainImpl() int { getExtraGas := func() uint64 { return nodeConfig.Node.Staker.ExtraGas } // #nosec G115 - addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, dataPoster, getExtraGas) + addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1Reader, true, dataPoster, getExtraGas) if err != nil { log.Crit("error creating validator wallet contract", "error", err, "address", l1TransactionOptsValidator.From.Hex()) } diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 83dedcad84..6346029c3a 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -229,7 +229,7 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err // By passing v.dataPoster as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. // DataPoster keeps in its internal state information related to the transactions sent through it, which is used to infer the expected nonce in a transaction for example. // If a transaction is sent using the Staker's DataPoster key, but not through the Staker's DataPoster object, DataPoster's internal state will be outdated, which can compromise the expected nonce inference. - addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, v.auth, v.l1Reader, createIfMissing, v.dataPoster, v.getExtraGas) + addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, v.l1Reader, createIfMissing, v.dataPoster, v.getExtraGas) if err != nil { return err } @@ -444,13 +444,13 @@ func GetValidatorWalletContract( ctx context.Context, validatorWalletFactoryAddr common.Address, fromBlock int64, - transactAuth *bind.TransactOpts, l1Reader *headerreader.HeaderReader, createIfMissing bool, dataPoster *dataposter.DataPoster, getExtraGas func() uint64, ) (*common.Address, error) { client := l1Reader.Client() + transactAuth := dataPoster.Auth() // TODO: If we just save a mapping in the wallet creator we won't need log search walletCreator, err := rollupgen.NewValidatorWalletCreator(validatorWalletFactoryAddr, client) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index 4598827418..dae2699b9f 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -115,10 +115,10 @@ func TestFastConfirmation(t *testing.T) { Require(t, err) valConfig.Strategy = "MakeNodes" - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) if valWalletAddr == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) @@ -315,10 +315,10 @@ func TestFastConfirmationWithSafe(t *testing.T) { Require(t, err) valConfigA.Strategy = "MakeNodes" - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index e61dcedc2b..67ce260529 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -176,10 +176,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) valConfigA.Strategy = "MakeNodes" } - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) @@ -507,10 +507,10 @@ func TestGetValidatorWalletContractWithDataposterOnlyUsedToCreateValidatorWallet } getExtraGas := func() uint64 { return builder.nodeConfig.Staker.ExtraGas } - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) From 857073870e455354ba13aa99c6bf06b81ba846ba Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 18:08:05 -0300 Subject: [PATCH 56/60] Uses validator auth instead of batch poster auth when creating data poster to create validator wallet contract --- cmd/nitro/nitro.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 0cf81c45e9..d4b2a87a30 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -375,7 +375,7 @@ func mainImpl() int { dataPoster, err := arbnode.DataposterOnlyUsedToCreateValidatorWalletContract( ctx, l1Reader, - l1TransactionOptsBatchPoster, + l1TransactionOptsValidator, &nodeConfig.Node.Staker.DataPoster, new(big.Int).SetUint64(nodeConfig.ParentChain.ID), ) From 8e4e8f9dafcb40057ab59516e2dbc7604ef6e39d Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 3 Sep 2024 10:34:17 -0500 Subject: [PATCH 57/60] Use a macro to define native WASM imports --- arbitrator/stylus/src/host.rs | 88 +++++++++++---- arbitrator/stylus/src/native.rs | 186 +++++++++----------------------- 2 files changed, 118 insertions(+), 156 deletions(-) diff --git a/arbitrator/stylus/src/host.rs b/arbitrator/stylus/src/host.rs index 1afc1b4e51..ca48da35db 100644 --- a/arbitrator/stylus/src/host.rs +++ b/arbitrator/stylus/src/host.rs @@ -441,26 +441,76 @@ pub(crate) fn pay_for_memory_grow>( hostio!(env, pay_for_memory_grow(pages)) } -pub(crate) fn console_log_text>( - mut env: WasmEnvMut, - ptr: GuestPtr, - len: u32, -) -> MaybeEscape { - hostio!(env, console_log_text(ptr, len)) -} +pub(crate) mod console { + use super::*; -pub(crate) fn console_log, T: Into>( - mut env: WasmEnvMut, - value: T, -) -> MaybeEscape { - hostio!(env, console_log(value)) -} + pub(crate) fn log_txt>( + mut env: WasmEnvMut, + ptr: GuestPtr, + len: u32, + ) -> MaybeEscape { + hostio!(env, console_log_text(ptr, len)) + } -pub(crate) fn console_tee, T: Into + Copy>( - mut env: WasmEnvMut, - value: T, -) -> Result { - hostio!(env, console_tee(value)) + pub(crate) fn log_i32>( + mut env: WasmEnvMut, + value: u32, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_i32>( + mut env: WasmEnvMut, + value: u32, + ) -> Result { + hostio!(env, console_tee(value)) + } + + pub(crate) fn log_i64>( + mut env: WasmEnvMut, + value: u64, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_i64>( + mut env: WasmEnvMut, + value: u64, + ) -> Result { + hostio!(env, console_tee(value)) + } + + pub(crate) fn log_f32>( + mut env: WasmEnvMut, + value: f32, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_f32>( + mut env: WasmEnvMut, + value: f32, + ) -> Result { + hostio!(env, console_tee(value)) + } + + pub(crate) fn log_f64>( + mut env: WasmEnvMut, + value: f64, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_f64>( + mut env: WasmEnvMut, + value: f64, + ) -> Result { + hostio!(env, console_tee(value)) + } } -pub(crate) fn null_host>(_: WasmEnvMut) {} +pub(crate) mod debug { + use super::*; + + pub(crate) fn null_host>(_: WasmEnvMut) {} +} diff --git a/arbitrator/stylus/src/native.rs b/arbitrator/stylus/src/native.rs index cc1d191fe2..7a82314fbc 100644 --- a/arbitrator/stylus/src/native.rs +++ b/arbitrator/stylus/src/native.rs @@ -33,7 +33,7 @@ use std::{ ops::{Deref, DerefMut}, }; use wasmer::{ - imports, AsStoreMut, Function, FunctionEnv, Instance, Memory, Module, Pages, Store, Target, + AsStoreMut, Function, FunctionEnv, Imports, Instance, Memory, Module, Pages, Store, Target, TypedFunction, Value, WasmTypeList, }; use wasmer_vm::VMExtern; @@ -151,68 +151,58 @@ impl> NativeInstance { fn from_module(module: Module, mut store: Store, env: WasmEnv) -> Result { let debug_funcs = env.compile.debug.debug_funcs; let func_env = FunctionEnv::new(&mut store, env); + let mut imports = Imports::new(); macro_rules! func { - ($func:expr) => { - Function::new_typed_with_env(&mut store, &func_env, $func) + ($rust_mod:path, $func:ident) => {{ + use $rust_mod as rust_mod; + Function::new_typed_with_env(&mut store, &func_env, rust_mod::$func) + }}; + } + macro_rules! define_imports { + ($($wasm_mod:literal => $rust_mod:path { $( $import:ident ),* $(,)? },)* $(,)?) => { + $( + $( + define_imports!(@@imports $wasm_mod, func!($rust_mod, $import), $import, "arbitrator_forward__"); + )* + )* + }; + (@@imports $wasm_mod:literal, $func:expr, $import:ident, $($p:expr),*) => { + define_imports!(@imports $wasm_mod, $func, $import, $($p),*, ""); + }; + (@imports $wasm_mod:literal, $func:expr, $import:ident, $($p:expr),*) => { + $( + imports.define($wasm_mod, concat!($p, stringify!($import)), $func); + )* }; } - let mut imports = imports! { - "vm_hooks" => { - "read_args" => func!(host::read_args), - "write_result" => func!(host::write_result), - "exit_early" => func!(host::exit_early), - "storage_load_bytes32" => func!(host::storage_load_bytes32), - "storage_cache_bytes32" => func!(host::storage_cache_bytes32), - "storage_flush_cache" => func!(host::storage_flush_cache), - "transient_load_bytes32" => func!(host::transient_load_bytes32), - "transient_store_bytes32" => func!(host::transient_store_bytes32), - "call_contract" => func!(host::call_contract), - "delegate_call_contract" => func!(host::delegate_call_contract), - "static_call_contract" => func!(host::static_call_contract), - "create1" => func!(host::create1), - "create2" => func!(host::create2), - "read_return_data" => func!(host::read_return_data), - "return_data_size" => func!(host::return_data_size), - "emit_log" => func!(host::emit_log), - "account_balance" => func!(host::account_balance), - "account_code" => func!(host::account_code), - "account_codehash" => func!(host::account_codehash), - "account_code_size" => func!(host::account_code_size), - "evm_gas_left" => func!(host::evm_gas_left), - "evm_ink_left" => func!(host::evm_ink_left), - "block_basefee" => func!(host::block_basefee), - "chainid" => func!(host::chainid), - "block_coinbase" => func!(host::block_coinbase), - "block_gas_limit" => func!(host::block_gas_limit), - "block_number" => func!(host::block_number), - "block_timestamp" => func!(host::block_timestamp), - "contract_address" => func!(host::contract_address), - "math_div" => func!(host::math_div), - "math_mod" => func!(host::math_mod), - "math_pow" => func!(host::math_pow), - "math_add_mod" => func!(host::math_add_mod), - "math_mul_mod" => func!(host::math_mul_mod), - "msg_reentrant" => func!(host::msg_reentrant), - "msg_sender" => func!(host::msg_sender), - "msg_value" => func!(host::msg_value), - "tx_gas_price" => func!(host::tx_gas_price), - "tx_ink_price" => func!(host::tx_ink_price), - "tx_origin" => func!(host::tx_origin), - "pay_for_memory_grow" => func!(host::pay_for_memory_grow), - "native_keccak256" => func!(host::native_keccak256), + define_imports!( + "vm_hooks" => host { + read_args, write_result, exit_early, + storage_load_bytes32, storage_cache_bytes32, storage_flush_cache, transient_load_bytes32, transient_store_bytes32, + call_contract, delegate_call_contract, static_call_contract, create1, create2, read_return_data, return_data_size, + emit_log, + account_balance, account_code, account_codehash, account_code_size, + evm_gas_left, evm_ink_left, + block_basefee, chainid, block_coinbase, block_gas_limit, block_number, block_timestamp, + contract_address, + math_div, math_mod, math_pow, math_add_mod, math_mul_mod, + msg_reentrant, msg_sender, msg_value, + tx_gas_price, tx_ink_price, tx_origin, + pay_for_memory_grow, + native_keccak256, }, - }; + ); if debug_funcs { - imports.define("console", "log_txt", func!(host::console_log_text)); - imports.define("console", "log_i32", func!(host::console_log::)); - imports.define("console", "log_i64", func!(host::console_log::)); - imports.define("console", "log_f32", func!(host::console_log::)); - imports.define("console", "log_f64", func!(host::console_log::)); - imports.define("console", "tee_i32", func!(host::console_tee::)); - imports.define("console", "tee_i64", func!(host::console_tee::)); - imports.define("console", "tee_f32", func!(host::console_tee::)); - imports.define("console", "tee_f64", func!(host::console_tee::)); - imports.define("debug", "null_host", func!(host::null_host)); + define_imports!( + "console" => host::console { + log_txt, + log_i32, log_i64, log_f32, log_f64, + tee_i32, tee_i64, tee_f32, tee_f64, + }, + "debug" => host::debug { + null_host, + }, + ); } let instance = Instance::new(&mut store, &module, &imports)?; let exports = &instance.exports; @@ -351,86 +341,8 @@ impl> StartlessMachine for NativeInstance { } pub fn module(wasm: &[u8], compile: CompileConfig, target: Target) -> Result> { - let mut store = compile.store(target); + let store = compile.store(target); let module = Module::new(&store, wasm)?; - macro_rules! stub { - (u8 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> u8 { panic!("incomplete import") }) - }; - (u32 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> u32 { panic!("incomplete import") }) - }; - (u64 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> u64 { panic!("incomplete import") }) - }; - (f32 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> f32 { panic!("incomplete import") }) - }; - (f64 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> f64 { panic!("incomplete import") }) - }; - ($($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> () { panic!("incomplete import") }) - }; - } - let mut imports = imports! { - "vm_hooks" => { - "read_args" => stub!(|_: u32|), - "write_result" => stub!(|_: u32, _: u32|), - "exit_early" => stub!(|_: u32|), - "storage_load_bytes32" => stub!(|_: u32, _: u32|), - "storage_cache_bytes32" => stub!(|_: u32, _: u32|), - "storage_flush_cache" => stub!(|_: u32|), - "transient_load_bytes32" => stub!(|_: u32, _: u32|), - "transient_store_bytes32" => stub!(|_: u32, _: u32|), - "call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u32, _: u64, _: u32|), - "delegate_call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u64, _: u32|), - "static_call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u64, _: u32|), - "create1" => stub!(|_: u32, _: u32, _: u32, _: u32, _: u32|), - "create2" => stub!(|_: u32, _: u32, _: u32, _: u32, _: u32, _: u32|), - "read_return_data" => stub!(u32 <- |_: u32, _: u32, _: u32|), - "return_data_size" => stub!(u32 <- ||), - "emit_log" => stub!(|_: u32, _: u32, _: u32|), - "account_balance" => stub!(|_: u32, _: u32|), - "account_code" => stub!(u32 <- |_: u32, _: u32, _: u32, _: u32|), - "account_codehash" => stub!(|_: u32, _: u32|), - "account_code_size" => stub!(u32 <- |_: u32|), - "evm_gas_left" => stub!(u64 <- ||), - "evm_ink_left" => stub!(u64 <- ||), - "block_basefee" => stub!(|_: u32|), - "chainid" => stub!(u64 <- ||), - "block_coinbase" => stub!(|_: u32|), - "block_gas_limit" => stub!(u64 <- ||), - "block_number" => stub!(u64 <- ||), - "block_timestamp" => stub!(u64 <- ||), - "contract_address" => stub!(|_: u32|), - "math_div" => stub!(|_: u32, _: u32|), - "math_mod" => stub!(|_: u32, _: u32|), - "math_pow" => stub!(|_: u32, _: u32|), - "math_add_mod" => stub!(|_: u32, _: u32, _: u32|), - "math_mul_mod" => stub!(|_: u32, _: u32, _: u32|), - "msg_reentrant" => stub!(u32 <- ||), - "msg_sender" => stub!(|_: u32|), - "msg_value" => stub!(|_: u32|), - "tx_gas_price" => stub!(|_: u32|), - "tx_ink_price" => stub!(u32 <- ||), - "tx_origin" => stub!(|_: u32|), - "pay_for_memory_grow" => stub!(|_: u16|), - "native_keccak256" => stub!(|_: u32, _: u32, _: u32|), - }, - }; - if compile.debug.debug_funcs { - imports.define("console", "log_txt", stub!(|_: u32, _: u32|)); - imports.define("console", "log_i32", stub!(|_: u32|)); - imports.define("console", "log_i64", stub!(|_: u64|)); - imports.define("console", "log_f32", stub!(|_: f32|)); - imports.define("console", "log_f64", stub!(|_: f64|)); - imports.define("console", "tee_i32", stub!(u32 <- |_: u32|)); - imports.define("console", "tee_i64", stub!(u64 <- |_: u64|)); - imports.define("console", "tee_f32", stub!(f32 <- |_: f32|)); - imports.define("console", "tee_f64", stub!(f64 <- |_: f64|)); - imports.define("debug", "null_host", stub!(||)); - } let module = module.serialize()?; Ok(module.to_vec()) From b913d6b53779489593fe4289d4a2335c86ce2b51 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 3 Sep 2024 10:51:48 -0500 Subject: [PATCH 58/60] Remove unused import --- arbitrator/stylus/src/host.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/arbitrator/stylus/src/host.rs b/arbitrator/stylus/src/host.rs index ca48da35db..fbe5657c5f 100644 --- a/arbitrator/stylus/src/host.rs +++ b/arbitrator/stylus/src/host.rs @@ -13,7 +13,6 @@ use arbutil::{ }; use caller_env::GuestPtr; use eyre::Result; -use prover::value::Value; use std::{ fmt::Display, mem::{self, MaybeUninit}, From d39f362da5c86964a68992b984a732ad618bf16c Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 4 Sep 2024 14:58:22 -0600 Subject: [PATCH 59/60] redis consumer fix --- validator/valnode/redis/consumer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 7456466533..6432741eba 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -111,10 +111,12 @@ func (s *ValidationServer) Start(ctx_in context.Context) { req, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) + requestTokenQueue <- struct{}{} return 0 } if req == nil { - // There's nothing in the queue. + // There's nothing in the queue + requestTokenQueue <- struct{}{} return time.Second } select { From 8d61504e109a89b12112ddbd4be94eb2753b0802 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 4 Sep 2024 15:00:53 -0600 Subject: [PATCH 60/60] redis consumer: dont set result if error --- validator/valnode/redis/consumer.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 6432741eba..2b025600cc 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -154,9 +154,10 @@ func (s *ValidationServer) Start(ctx_in context.Context) { res, err := valRun.Await(ctx) if err != nil { log.Error("Error validating", "request value", work.req.Value, "error", err) - } - if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { - log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } else { + if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { + log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } } select { case <-ctx.Done():