Merge branch 'ondemand-module-compile' into run-mode-struct

arbos/programs/native.go

@@ -71,16 +71,16 @@ func activateProgram(
 	burner burn.Burner,
 	runMode *core.MessageRunMode,
 ) (*activationInfo, error) {
-	info, asmMap, err := activateProgramInternal(db, program, codehash, wasm, page_limit, stylusVersion, arbosVersionForGas, debug, burner.GasLeft(), runMode)
+	moduleActivationMandatory := true
+	info, asmMap, err := activateProgramInternal(program, codehash, wasm, page_limit, stylusVersion, arbosVersionForGas, debug, burner.GasLeft(), runMode, moduleActivationMandatory)
 	if err != nil {
 		return nil, err
 	}
 	db.ActivateWasm(info.moduleHash, asmMap)
 	return info, nil
 }
 
-func activateProgramInternal(
-	db vm.StateDB,
+func activateModule(
 	addressForLogging common.Address,
 	codehash common.Hash,
 	wasm []byte,
@@ -89,8 +89,7 @@ func activateProgramInternal(
 	arbosVersionForGas uint64,
 	debug bool,
 	gasLeft *uint64,
-	runMode *core.MessageRunMode,
-) (*activationInfo, map[ethdb.WasmTarget][]byte, error) {
+) (*activationInfo, []byte, error) {
 	output := &rustBytes{}
 	moduleHash := &bytes32{}
 	stylusData := &C.StylusData{}
@@ -108,76 +107,147 @@ func activateProgramInternal(
 		stylusData,
 		(*u64)(gasLeft),
 	))
-
 	module, msg, err := status_mod.toResult(output.intoBytes(), debug)
 	if err != nil {
 		if debug {
 			log.Warn("activation failed", "err", err, "msg", msg, "program", addressForLogging)
 		}
 		if errors.Is(err, vm.ErrExecutionReverted) {
 			return nil, nil, fmt.Errorf("%w: %s", ErrProgramActivation, msg)
+		} else {
+			return nil, nil, err
 		}
-		return nil, nil, err
 	}
-	hash := moduleHash.toHash()
+	info := &activationInfo{
+		moduleHash:    moduleHash.toHash(),
+		initGas:       uint16(stylusData.init_cost),
+		cachedInitGas: uint16(stylusData.cached_init_cost),
+		asmEstimate:   uint32(stylusData.asm_estimate),
+		footprint:     uint16(stylusData.footprint),
+	}
+	return info, module, nil
+}
+
+func compileNative(
+	wasm []byte,
+	stylusVersion uint16,
+	debug bool,
+	target ethdb.WasmTarget,
+) ([]byte, error) {
+	output := &rustBytes{}
+	status_asm := C.stylus_compile(
+		goSlice(wasm),
+		u16(stylusVersion),
+		cbool(debug),
+		goSlice([]byte(target)),
+		output,
+	)
+	asm := output.intoBytes()
+	if status_asm != 0 {
+		return nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm))
+	}
+	return asm, nil
+}
+
+func activateProgramInternal(
+	addressForLogging common.Address,
+	codehash common.Hash,
+	wasm []byte,
+	page_limit uint16,
+	stylusVersion uint16,
+	arbosVersionForGas uint64,
+	debug bool,
+	gasLeft *uint64,
+	runMode *core.MessageRunMode,
+	moduleActivationMandatory bool,
+) (*activationInfo, map[ethdb.WasmTarget][]byte, error) {
 	targets := runMode.WasmTargets()
+	var wavmFound bool
+	for _, target := range targets {
+		if target == rawdb.TargetWavm {
+			wavmFound = true
+			break
+		}
+	}
 	type result struct {
 		target ethdb.WasmTarget
 		asm    []byte
 		err    error
 	}
+	asmMap := make(map[ethdb.WasmTarget][]byte, len(targets))
+
+	// info can be set in separate thread, make sure to wait before reading
+	var info *activationInfo
+	var moduleActivationStarted bool
+	if moduleActivationMandatory {
+		moduleActivationStarted = true
+		var err error
+		var module []byte
+		info, module, err = activateModule(addressForLogging, codehash, wasm, page_limit, stylusVersion, arbosVersionForGas, debug, gasLeft)
+		if err != nil {
+			return nil, nil, err
+		}
+		if wavmFound {
+			asmMap[rawdb.TargetWavm] = module
+		}
+	}
+
 	results := make(chan result, len(targets))
 	for _, target := range targets {
 		target := target
 		if target == rawdb.TargetWavm {
-			results <- result{target, module, nil}
+			if moduleActivationStarted {
+				// skip if already started or activated because of moduleActivationMandatory
+				results <- result{target, nil, nil}
+				continue
+			}
+			go func() {
+				var err error
+				var module []byte
+				info, module, err = activateModule(addressForLogging, codehash, wasm, page_limit, stylusVersion, arbosVersionForGas, debug, gasLeft)
+				results <- result{target, module, err}
+			}()
+			moduleActivationStarted = true
 		} else {
 			go func() {
-				output := &rustBytes{}
-				status_asm := C.stylus_compile(
-					goSlice(wasm),
-					u16(stylusVersion),
-					cbool(debug),
-					goSlice([]byte(target)),
-					output,
-				)
-				asm := output.intoBytes()
-				if status_asm != 0 {
-					results <- result{target, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm))}
-					return
-				}
-				results <- result{target, asm, nil}
+				asm, err := compileNative(wasm, stylusVersion, debug, target)
+				results <- result{target, asm, err}
 			}()
 		}
 	}
-	asmMap := make(map[ethdb.WasmTarget][]byte, len(targets))
+	var err error
 	for range targets {
 		res := <-results
-		if res.err != nil {
-			err = errors.Join(res.err, err)
+		if res.asm == nil {
+			continue
+		} else if res.err != nil {
+			err = errors.Join(res.err, fmt.Errorf("%s:%w", res.target, err))
 		} else {
 			asmMap[res.target] = res.asm
 		}
 	}
-	if err != nil {
-		log.Error(
-			"Compilation failed for one or more targets despite activation succeeding",
-			"address", addressForLogging,
-			"codeHash", codeHash,
-			"moduleHash", hash,
-			"targets", targets,
-			"err", err,
-		)
+	if err != nil && moduleActivationMandatory {
+		if info != nil {
+			log.Error(
+				"Compilation failed for one or more targets despite activation succeeding",
+				"address", addressForLogging,
+				"codehash", codehash,
+				"moduleHash", info.moduleHash,
+				"targets", targets,
+				"err", err,
+			)
+		} else {
+			log.Error(
+				"Compilation failed for one or more targets despite activation succeeding",
+				"address", addressForLogging,
+				"codehash", codehash,
+				"targets", targets,
+				"err", err,
+			)
+		}
 		panic(fmt.Sprintf("Compilation of %v failed for one or more targets despite activation succeeding: %v", addressForLogging, err))
 	}
 
-	info := &activationInfo{
-		moduleHash:    hash,
-		initGas:       uint16(stylusData.init_cost),
-		cachedInitGas: uint16(stylusData.cached_init_cost),
-		asmEstimate:   uint32(stylusData.asm_estimate),
-		footprint:     uint16(stylusData.footprint),
-	}
 	return info, asmMap, err
 }
 
@@ -200,7 +270,8 @@ func getLocalAsm(statedb vm.StateDB, moduleHash common.Hash, addressForLogging c
 	zeroGas := uint64(0)
 
 	// we know program is activated, so it must be in correct version and not use too much memory
-	info, asmMap, err := activateProgramInternal(statedb, addressForLogging, codeHash, wasm, pagelimit, program.version, zeroArbosVersion, debugMode, &zeroGas, runMode)
+	moduleActivationMandatory := true // TODO: do we need to do the module activation?
+	info, asmMap, err := activateProgramInternal(addressForLogging, codeHash, wasm, pagelimit, program.version, zeroArbosVersion, debugMode, &zeroGas, runMode, moduleActivationMandatory)
 	if err != nil {
 		log.Error("failed to reactivate program", "address", addressForLogging, "expected moduleHash", moduleHash, "err", err)
 		return nil, fmt.Errorf("failed to reactivate program address: %v err: %w", addressForLogging, err)

arbos/programs/wasmstorehelper.go

@@ -63,13 +63,14 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash
 
 	// We know program is activated, so it must be in correct version and not use too much memory
 	// Empty program address is supplied because we dont have access to this during rebuilding of wasm store
-	info, asmMap, err := activateProgramInternal(statedb, common.Address{}, codeHash, wasm, progParams.PageLimit, program.version, zeroArbosVersion, debugMode, &zeroGas, core.NewMessageReplayMode(targets))
+	moduleActivationMandatory := false
+	info, asmMap, err := activateProgramInternal(common.Address{}, codeHash, wasm, progParams.PageLimit, program.version, zeroArbosVersion, debugMode, &zeroGas, core.NewMessageReplayMode(targets), moduleActivationMandatory)
 	if err != nil {
 		log.Error("failed to reactivate program while rebuilding wasm store", "expected moduleHash", moduleHash, "err", err)
 		return fmt.Errorf("failed to reactivate program while rebuilding wasm store: %w", err)
 	}
 
-	if info.moduleHash != moduleHash {
+	if info != nil && info.moduleHash != moduleHash {
 		log.Error("failed to reactivate program while rebuilding wasm store", "expected moduleHash", moduleHash, "got", info.moduleHash)
 		return fmt.Errorf("failed to reactivate program while rebuilding wasm store, expected ModuleHash: %v", moduleHash)
 	}

execution/gethexec/node.go

@@ -53,9 +53,6 @@ func (c *StylusTargetConfig) Validate() error {
 		}
 		targetsSet[target] = true
 	}
-	if !targetsSet[rawdb.TargetWavm] {
-		return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs)
-	}
 	targetsSet[rawdb.LocalTarget()] = true
 	targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1)
 	for target := range targetsSet {

system_tests/common_test.go

@@ -1320,6 +1320,7 @@ func createNonL1BlockChainWithStackConfig(
 	if execConfig == nil {
 		execConfig = ExecConfigDefaultTest(t)
 	}
+	Require(t, execConfig.Validate())
 
 	stack, err := node.New(stackConfig)
 	Require(t, err)
@@ -1407,6 +1408,8 @@ func Create2ndNodeWithConfig(
 	if execConfig == nil {
 		execConfig = ExecConfigDefaultNonSequencerTest(t)
 	}
+	Require(t, execConfig.Validate())
+
 	feedErrChan := make(chan error, 10)
 	parentChainRpcClient := parentChainStack.Attach()
 	parentChainClient := ethclient.NewClient(parentChainRpcClient)
@@ -1440,7 +1443,6 @@ func Create2ndNodeWithConfig(
 
 	AddValNodeIfNeeded(t, ctx, nodeConfig, true, "", valnodeConfig.Wasm.RootPath)
 
-	Require(t, execConfig.Validate())
 	Require(t, nodeConfig.Validate())
 	configFetcher := func() *gethexec.Config { return execConfig }
 	currentExec, err := gethexec.CreateExecutionNode(ctx, chainStack, chainDb, blockchain, parentChainClient, configFetcher)