From 272139690e028d3bdebdb6bcb1824fec23cefd0f Mon Sep 17 00:00:00 2001 From: Yury Akudovich Date: Thu, 31 Oct 2024 14:47:04 +0100 Subject: [PATCH 1/3] feat(prover): Add queue metric to report autoscaler view of the queue. (#3206) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Add `queue` metric to report autoscaler view of the queue. Add Copy trait for QueueReportFields and remove unneeded clone or references. ## Why ❔ The `queue` metric will be used in dashboards. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [x] Code has been formatted via `zkstack dev fmt` and `zkstack dev lint`. ref ZKD-1855 --- core/lib/config/src/configs/prover_autoscaler.rs | 2 +- prover/crates/bin/prover_autoscaler/src/global/queuer.rs | 6 +++--- prover/crates/bin/prover_autoscaler/src/global/scaler.rs | 8 +++++--- prover/crates/bin/prover_autoscaler/src/metrics.rs | 4 +++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/core/lib/config/src/configs/prover_autoscaler.rs b/core/lib/config/src/configs/prover_autoscaler.rs index ab6b8fdf202..4191208b96e 100644 --- a/core/lib/config/src/configs/prover_autoscaler.rs +++ b/core/lib/config/src/configs/prover_autoscaler.rs @@ -100,7 +100,7 @@ pub enum Gpu { // TODO: generate this enum by QueueReport from https://github.com/matter-labs/zksync-era/blob/main/prover/crates/bin/prover_job_monitor/src/autoscaler_queue_reporter.rs#L23 // and remove allowing of non_camel_case_types by generating field name parser. -#[derive(Debug, Display, PartialEq, Eq, Hash, Clone, Deserialize, EnumString, Default)] +#[derive(Debug, Display, PartialEq, Eq, Hash, Clone, Copy, Deserialize, EnumString, Default)] #[allow(non_camel_case_types)] pub enum QueueReportFields { #[strum(ascii_case_insensitive)] diff --git a/prover/crates/bin/prover_autoscaler/src/global/queuer.rs b/prover/crates/bin/prover_autoscaler/src/global/queuer.rs index e2cd1c6a4fb..7255f479647 100644 --- a/prover/crates/bin/prover_autoscaler/src/global/queuer.rs +++ b/prover/crates/bin/prover_autoscaler/src/global/queuer.rs @@ -24,7 +24,7 @@ pub struct Queuer { pub prover_job_monitor_url: String, } -fn target_to_queue(target: &QueueReportFields, report: &QueueReport) -> u64 { +fn target_to_queue(target: QueueReportFields, report: &QueueReport) -> u64 { let res = match target { QueueReportFields::basic_witness_jobs => report.basic_witness_jobs.all(), QueueReportFields::leaf_witness_jobs => report.leaf_witness_jobs.all(), @@ -65,8 +65,8 @@ impl Queuer { .flat_map(|versioned_report| { jobs.iter().map(move |j| { ( - (versioned_report.version.to_string(), j.clone()), - target_to_queue(j, &versioned_report.report), + (versioned_report.version.to_string(), *j), + target_to_queue(*j, &versioned_report.report), ) }) }) diff --git a/prover/crates/bin/prover_autoscaler/src/global/scaler.rs b/prover/crates/bin/prover_autoscaler/src/global/scaler.rs index 362fbbac074..dc652999da5 100644 --- a/prover/crates/bin/prover_autoscaler/src/global/scaler.rs +++ b/prover/crates/bin/prover_autoscaler/src/global/scaler.rs @@ -124,7 +124,7 @@ impl Scaler { let mut simple_scalers = Vec::default(); let mut jobs = vec![QueueReportFields::prover_jobs]; for c in &config.scaler_targets { - jobs.push(c.queue_report_field.clone()); + jobs.push(c.queue_report_field); simple_scalers.push(SimpleScaler::new( c, config.cluster_priorities.clone(), @@ -429,7 +429,7 @@ impl SimpleScaler { long_pending_duration: chrono::Duration, ) -> Self { Self { - queue_report_field: config.queue_report_field.clone(), + queue_report_field: config.queue_report_field, deployment: config.deployment.clone(), cluster_priorities, max_replicas: config.max_replicas.clone(), @@ -671,6 +671,7 @@ impl Task for Scaler { .get(&(ppv.to_string(), QueueReportFields::prover_jobs)) .cloned() .unwrap_or(0); + AUTOSCALER_METRICS.queue[&(ns.clone(), "prover".into())].set(q); tracing::debug!("Running eval for namespace {ns} and PPV {ppv} found queue {q}"); if q > 0 || is_namespace_running(ns, &guard.clusters) { let provers = self.prover_scaler.run(ns, q, &guard.clusters); @@ -684,9 +685,10 @@ impl Task for Scaler { // Simple Scalers. for scaler in &self.simple_scalers { let q = queue - .get(&(ppv.to_string(), scaler.queue_report_field.clone())) + .get(&(ppv.to_string(), scaler.queue_report_field)) .cloned() .unwrap_or(0); + AUTOSCALER_METRICS.queue[&(ns.clone(), scaler.deployment.clone())].set(q); tracing::debug!("Running eval for namespace {ns}, PPV {ppv}, simple scaler {} found queue {q}", scaler.deployment); if q > 0 || is_namespace_running(ns, &guard.clusters) { let replicas = scaler.run(ns, q, &guard.clusters); diff --git a/prover/crates/bin/prover_autoscaler/src/metrics.rs b/prover/crates/bin/prover_autoscaler/src/metrics.rs index 853e3db000f..39860a9e8f0 100644 --- a/prover/crates/bin/prover_autoscaler/src/metrics.rs +++ b/prover/crates/bin/prover_autoscaler/src/metrics.rs @@ -16,7 +16,9 @@ pub(crate) struct AutoscalerMetrics { #[metrics(labels = ["target", "status"])] pub calls: LabeledFamily<(String, u16), Counter, 2>, #[metrics(labels = ["target_cluster"])] - pub scale_errors: LabeledFamily, 1>, + pub scale_errors: LabeledFamily>, + #[metrics(labels = ["target_namespace", "job"])] + pub queue: LabeledFamily<(String, String), Gauge, 2>, } #[vise::register] From 9ad1f0d77e5a5b411f7866ef6a1819373c07f91b Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Thu, 31 Oct 2024 16:09:18 +0200 Subject: [PATCH 2/3] feat(vm): Support EVM emulation in fast VM (#3163) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Supports EVM emulation if it is signaled via `SystemEnv`. ## Why ❔ Fast VM should have feature parity with the legacy VM. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zkstack dev fmt` and `zkstack dev lint`. --- core/lib/multivm/src/versions/shadow/tests.rs | 48 ++ .../src/versions/testonly/evm_emulator.rs | 493 ++++++++++++++++++ core/lib/multivm/src/versions/testonly/mod.rs | 1 + .../src/versions/vm_fast/circuits_tracer.rs | 2 +- .../src/versions/vm_fast/evm_deploy_tracer.rs | 84 +++ core/lib/multivm/src/versions/vm_fast/mod.rs | 4 +- .../versions/vm_fast/tests/evm_emulator.rs | 53 ++ .../multivm/src/versions/vm_fast/tests/mod.rs | 11 +- .../lib/multivm/src/versions/vm_fast/utils.rs | 13 + core/lib/multivm/src/versions/vm_fast/vm.rs | 97 +++- .../versions/vm_latest/tests/evm_emulator.rs | 482 +---------------- core/lib/vm_interface/src/utils/shadow.rs | 10 + 12 files changed, 802 insertions(+), 496 deletions(-) create mode 100644 core/lib/multivm/src/versions/testonly/evm_emulator.rs create mode 100644 core/lib/multivm/src/versions/vm_fast/evm_deploy_tracer.rs create mode 100644 core/lib/multivm/src/versions/vm_fast/tests/evm_emulator.rs create mode 100644 core/lib/multivm/src/versions/vm_fast/utils.rs diff --git a/core/lib/multivm/src/versions/shadow/tests.rs b/core/lib/multivm/src/versions/shadow/tests.rs index 6a39a28f763..e6fb05e2406 100644 --- a/core/lib/multivm/src/versions/shadow/tests.rs +++ b/core/lib/multivm/src/versions/shadow/tests.rs @@ -197,6 +197,54 @@ mod default_aa { } } +mod evm_emulator { + use test_casing::{test_casing, Product}; + + use crate::versions::testonly::evm_emulator::*; + + #[test] + fn tracing_evm_contract_deployment() { + test_tracing_evm_contract_deployment::(); + } + + #[test] + fn mock_emulator_basics() { + test_mock_emulator_basics::(); + } + + #[test_casing(2, [false, true])] + #[test] + fn mock_emulator_with_payment(deploy_emulator: bool) { + test_mock_emulator_with_payment::(deploy_emulator); + } + + #[test_casing(4, Product(([false, true], [false, true])))] + #[test] + fn mock_emulator_with_recursion(deploy_emulator: bool, is_external: bool) { + test_mock_emulator_with_recursion::(deploy_emulator, is_external); + } + + #[test] + fn calling_to_mock_emulator_from_native_contract() { + test_calling_to_mock_emulator_from_native_contract::(); + } + + #[test] + fn mock_emulator_with_deployment() { + test_mock_emulator_with_deployment::(); + } + + #[test] + fn mock_emulator_with_delegate_call() { + test_mock_emulator_with_delegate_call::(); + } + + #[test] + fn mock_emulator_with_static_call() { + test_mock_emulator_with_static_call::(); + } +} + mod gas_limit { use crate::versions::testonly::gas_limit::*; diff --git a/core/lib/multivm/src/versions/testonly/evm_emulator.rs b/core/lib/multivm/src/versions/testonly/evm_emulator.rs new file mode 100644 index 00000000000..6de394842aa --- /dev/null +++ b/core/lib/multivm/src/versions/testonly/evm_emulator.rs @@ -0,0 +1,493 @@ +use std::collections::HashMap; + +use ethabi::Token; +use zksync_contracts::{load_contract, read_bytecode, SystemContractCode}; +use zksync_system_constants::{ + CONTRACT_DEPLOYER_ADDRESS, KNOWN_CODES_STORAGE_ADDRESS, L2_BASE_TOKEN_ADDRESS, +}; +use zksync_test_account::TxType; +use zksync_types::{ + get_code_key, get_known_code_key, + utils::{key_for_eth_balance, storage_key_for_eth_balance}, + AccountTreeId, Address, Execute, StorageKey, H256, U256, +}; +use zksync_utils::{ + bytecode::{hash_bytecode, hash_evm_bytecode}, + bytes_to_be_words, h256_to_u256, +}; + +use super::{default_system_env, TestedVm, VmTester, VmTesterBuilder}; +use crate::interface::{ + storage::InMemoryStorage, TxExecutionMode, VmExecutionResultAndLogs, VmInterfaceExt, +}; + +const MOCK_DEPLOYER_PATH: &str = "etc/contracts-test-data/artifacts-zk/contracts/mock-evm/mock-evm.sol/MockContractDeployer.json"; +const MOCK_KNOWN_CODE_STORAGE_PATH: &str = "etc/contracts-test-data/artifacts-zk/contracts/mock-evm/mock-evm.sol/MockKnownCodeStorage.json"; +const MOCK_EMULATOR_PATH: &str = + "etc/contracts-test-data/artifacts-zk/contracts/mock-evm/mock-evm.sol/MockEvmEmulator.json"; +const RECURSIVE_CONTRACT_PATH: &str = "etc/contracts-test-data/artifacts-zk/contracts/mock-evm/mock-evm.sol/NativeRecursiveContract.json"; +const INCREMENTING_CONTRACT_PATH: &str = "etc/contracts-test-data/artifacts-zk/contracts/mock-evm/mock-evm.sol/IncrementingContract.json"; + +fn override_system_contracts(storage: &mut InMemoryStorage) { + let mock_deployer = read_bytecode(MOCK_DEPLOYER_PATH); + let mock_deployer_hash = hash_bytecode(&mock_deployer); + let mock_known_code_storage = read_bytecode(MOCK_KNOWN_CODE_STORAGE_PATH); + let mock_known_code_storage_hash = hash_bytecode(&mock_known_code_storage); + + storage.set_value(get_code_key(&CONTRACT_DEPLOYER_ADDRESS), mock_deployer_hash); + storage.set_value( + get_known_code_key(&mock_deployer_hash), + H256::from_low_u64_be(1), + ); + storage.set_value( + get_code_key(&KNOWN_CODES_STORAGE_ADDRESS), + mock_known_code_storage_hash, + ); + storage.set_value( + get_known_code_key(&mock_known_code_storage_hash), + H256::from_low_u64_be(1), + ); + storage.store_factory_dep(mock_deployer_hash, mock_deployer); + storage.store_factory_dep(mock_known_code_storage_hash, mock_known_code_storage); +} + +#[derive(Debug)] +struct EvmTestBuilder { + deploy_emulator: bool, + storage: InMemoryStorage, + evm_contract_addresses: Vec
, +} + +impl EvmTestBuilder { + fn new(deploy_emulator: bool, evm_contract_address: Address) -> Self { + Self { + deploy_emulator, + storage: InMemoryStorage::with_system_contracts(hash_bytecode), + evm_contract_addresses: vec![evm_contract_address], + } + } + + fn with_mock_deployer(mut self) -> Self { + override_system_contracts(&mut self.storage); + self + } + + fn with_evm_address(mut self, address: Address) -> Self { + self.evm_contract_addresses.push(address); + self + } + + fn build(self) -> VmTester { + let mock_emulator = read_bytecode(MOCK_EMULATOR_PATH); + let mut storage = self.storage; + let mut system_env = default_system_env(); + if self.deploy_emulator { + let evm_bytecode: Vec<_> = (0..32).collect(); + let evm_bytecode_hash = hash_evm_bytecode(&evm_bytecode); + storage.set_value( + get_known_code_key(&evm_bytecode_hash), + H256::from_low_u64_be(1), + ); + for evm_address in self.evm_contract_addresses { + storage.set_value(get_code_key(&evm_address), evm_bytecode_hash); + } + + system_env.base_system_smart_contracts.evm_emulator = Some(SystemContractCode { + hash: hash_bytecode(&mock_emulator), + code: bytes_to_be_words(mock_emulator), + }); + } else { + let emulator_hash = hash_bytecode(&mock_emulator); + storage.set_value(get_known_code_key(&emulator_hash), H256::from_low_u64_be(1)); + storage.store_factory_dep(emulator_hash, mock_emulator); + + for evm_address in self.evm_contract_addresses { + storage.set_value(get_code_key(&evm_address), emulator_hash); + // Set `isUserSpace` in the emulator storage to `true`, so that it skips emulator-specific checks + storage.set_value( + StorageKey::new(AccountTreeId::new(evm_address), H256::zero()), + H256::from_low_u64_be(1), + ); + } + } + + VmTesterBuilder::new() + .with_system_env(system_env) + .with_storage(storage) + .with_execution_mode(TxExecutionMode::VerifyExecute) + .with_rich_accounts(1) + .build::() + } +} + +pub(crate) fn test_tracing_evm_contract_deployment() { + let mut storage = InMemoryStorage::with_system_contracts(hash_bytecode); + override_system_contracts(&mut storage); + + let mut system_env = default_system_env(); + // The EVM emulator will not be accessed, so we set it to a dummy value. + system_env.base_system_smart_contracts.evm_emulator = + Some(system_env.base_system_smart_contracts.default_aa.clone()); + let mut vm = VmTesterBuilder::new() + .with_system_env(system_env) + .with_storage(storage) + .with_execution_mode(TxExecutionMode::VerifyExecute) + .with_rich_accounts(1) + .build::(); + let account = &mut vm.rich_accounts[0]; + + let args = [Token::Bytes((0..32).collect())]; + let evm_bytecode = ethabi::encode(&args); + let expected_bytecode_hash = hash_evm_bytecode(&evm_bytecode); + let execute = Execute::for_deploy(expected_bytecode_hash, vec![0; 32], &args); + let deploy_tx = account.get_l2_tx_for_execute(execute, None); + let (_, vm_result) = vm + .vm + .execute_transaction_with_bytecode_compression(deploy_tx, true); + assert!(!vm_result.result.is_failed(), "{:?}", vm_result.result); + + let new_known_factory_deps = vm_result.new_known_factory_deps.unwrap(); + assert_eq!(new_known_factory_deps.len(), 2); // the deployed EraVM contract + EVM contract + assert_eq!( + new_known_factory_deps[&expected_bytecode_hash], + evm_bytecode + ); +} + +pub(crate) fn test_mock_emulator_basics() { + let called_address = Address::repeat_byte(0x23); + let mut vm = EvmTestBuilder::new(true, called_address).build::(); + let account = &mut vm.rich_accounts[0]; + let tx = account.get_l2_tx_for_execute( + Execute { + contract_address: Some(called_address), + calldata: vec![], + value: 0.into(), + factory_deps: vec![], + }, + None, + ); + + let (_, vm_result) = vm + .vm + .execute_transaction_with_bytecode_compression(tx, true); + assert!(!vm_result.result.is_failed(), "{:?}", vm_result.result); +} + +const RECIPIENT_ADDRESS: Address = Address::repeat_byte(0x12); + +/// `deploy_emulator = false` here and below tests the mock emulator as an ordinary contract (i.e., sanity-checks its logic). +pub(crate) fn test_mock_emulator_with_payment(deploy_emulator: bool) { + let mock_emulator_abi = load_contract(MOCK_EMULATOR_PATH); + let mut vm = EvmTestBuilder::new(deploy_emulator, RECIPIENT_ADDRESS).build::(); + + let mut current_balance = U256::zero(); + for i in 1_u64..=5 { + let transferred_value = (1_000_000_000 * i).into(); + let vm_result = test_payment( + &mut vm, + &mock_emulator_abi, + &mut current_balance, + transferred_value, + ); + + let balance_storage_logs = vm_result.logs.storage_logs.iter().filter_map(|log| { + (*log.log.key.address() == L2_BASE_TOKEN_ADDRESS) + .then_some((*log.log.key.key(), h256_to_u256(log.log.value))) + }); + let balances: HashMap<_, _> = balance_storage_logs.collect(); + assert_eq!( + balances[&key_for_eth_balance(&RECIPIENT_ADDRESS)], + current_balance + ); + } +} + +fn test_payment( + vm: &mut VmTester, + mock_emulator_abi: ðabi::Contract, + balance: &mut U256, + transferred_value: U256, +) -> VmExecutionResultAndLogs { + *balance += transferred_value; + let test_payment_fn = mock_emulator_abi.function("testPayment").unwrap(); + let account = &mut vm.rich_accounts[0]; + let tx = account.get_l2_tx_for_execute( + Execute { + contract_address: Some(RECIPIENT_ADDRESS), + calldata: test_payment_fn + .encode_input(&[Token::Uint(transferred_value), Token::Uint(*balance)]) + .unwrap(), + value: transferred_value, + factory_deps: vec![], + }, + None, + ); + + let (_, vm_result) = vm + .vm + .execute_transaction_with_bytecode_compression(tx, true); + assert!(!vm_result.result.is_failed(), "{vm_result:?}"); + vm_result +} + +pub(crate) fn test_mock_emulator_with_recursion( + deploy_emulator: bool, + is_external: bool, +) { + let mock_emulator_abi = load_contract(MOCK_EMULATOR_PATH); + let recipient_address = Address::repeat_byte(0x12); + let mut vm = EvmTestBuilder::new(deploy_emulator, recipient_address).build::(); + let account = &mut vm.rich_accounts[0]; + + let test_recursion_fn = mock_emulator_abi + .function(if is_external { + "testExternalRecursion" + } else { + "testRecursion" + }) + .unwrap(); + let mut expected_value = U256::one(); + let depth = 50_u32; + for i in 2..=depth { + expected_value *= i; + } + + let factory_deps = if is_external { + vec![read_bytecode(RECURSIVE_CONTRACT_PATH)] + } else { + vec![] + }; + let tx = account.get_l2_tx_for_execute( + Execute { + contract_address: Some(recipient_address), + calldata: test_recursion_fn + .encode_input(&[Token::Uint(depth.into()), Token::Uint(expected_value)]) + .unwrap(), + value: 0.into(), + factory_deps, + }, + None, + ); + let (_, vm_result) = vm + .vm + .execute_transaction_with_bytecode_compression(tx, true); + assert!(!vm_result.result.is_failed(), "{vm_result:?}"); +} + +pub(crate) fn test_calling_to_mock_emulator_from_native_contract() { + let recipient_address = Address::repeat_byte(0x12); + let mut vm = EvmTestBuilder::new(true, recipient_address).build::(); + let account = &mut vm.rich_accounts[0]; + + // Deploy a native contract. + let native_contract = read_bytecode(RECURSIVE_CONTRACT_PATH); + let native_contract_abi = load_contract(RECURSIVE_CONTRACT_PATH); + let deploy_tx = account.get_deploy_tx( + &native_contract, + Some(&[Token::Address(recipient_address)]), + TxType::L2, + ); + let (_, vm_result) = vm + .vm + .execute_transaction_with_bytecode_compression(deploy_tx.tx, true); + assert!(!vm_result.result.is_failed(), "{:?}", vm_result.result); + + // Call from the native contract to the EVM emulator. + let test_fn = native_contract_abi.function("recurse").unwrap(); + let test_tx = account.get_l2_tx_for_execute( + Execute { + contract_address: Some(deploy_tx.address), + calldata: test_fn.encode_input(&[Token::Uint(50.into())]).unwrap(), + value: Default::default(), + factory_deps: vec![], + }, + None, + ); + let (_, vm_result) = vm + .vm + .execute_transaction_with_bytecode_compression(test_tx, true); + assert!(!vm_result.result.is_failed(), "{:?}", vm_result.result); +} + +pub(crate) fn test_mock_emulator_with_deployment() { + let contract_address = Address::repeat_byte(0xaa); + let mut vm = EvmTestBuilder::new(true, contract_address) + .with_mock_deployer() + .build::(); + let account = &mut vm.rich_accounts[0]; + + let mock_emulator_abi = load_contract(MOCK_EMULATOR_PATH); + let new_evm_bytecode = vec![0xfe; 96]; + let new_evm_bytecode_hash = hash_evm_bytecode(&new_evm_bytecode); + + let test_fn = mock_emulator_abi.function("testDeploymentAndCall").unwrap(); + let test_tx = account.get_l2_tx_for_execute( + Execute { + contract_address: Some(contract_address), + calldata: test_fn + .encode_input(&[ + Token::FixedBytes(new_evm_bytecode_hash.0.into()), + Token::Bytes(new_evm_bytecode.clone()), + ]) + .unwrap(), + value: 0.into(), + factory_deps: vec![], + }, + None, + ); + let (_, vm_result) = vm + .vm + .execute_transaction_with_bytecode_compression(test_tx, true); + assert!(!vm_result.result.is_failed(), "{vm_result:?}"); + + let factory_deps = vm_result.new_known_factory_deps.unwrap(); + assert_eq!( + factory_deps, + HashMap::from([(new_evm_bytecode_hash, new_evm_bytecode)]) + ); +} + +pub(crate) fn test_mock_emulator_with_delegate_call() { + let evm_contract_address = Address::repeat_byte(0xaa); + let other_evm_contract_address = Address::repeat_byte(0xbb); + let mut builder = EvmTestBuilder::new(true, evm_contract_address); + builder.storage.set_value( + storage_key_for_eth_balance(&evm_contract_address), + H256::from_low_u64_be(1_000_000), + ); + builder.storage.set_value( + storage_key_for_eth_balance(&other_evm_contract_address), + H256::from_low_u64_be(2_000_000), + ); + let mut vm = builder + .with_evm_address(other_evm_contract_address) + .build::(); + let account = &mut vm.rich_accounts[0]; + + // Deploy a native contract. + let native_contract = read_bytecode(INCREMENTING_CONTRACT_PATH); + let native_contract_abi = load_contract(INCREMENTING_CONTRACT_PATH); + let deploy_tx = account.get_deploy_tx(&native_contract, None, TxType::L2); + let (_, vm_result) = vm + .vm + .execute_transaction_with_bytecode_compression(deploy_tx.tx, true); + assert!(!vm_result.result.is_failed(), "{:?}", vm_result.result); + + let test_fn = native_contract_abi.function("testDelegateCall").unwrap(); + // Delegate to the native contract from EVM. + test_delegate_call(&mut vm, test_fn, evm_contract_address, deploy_tx.address); + // Delegate to EVM from the native contract. + test_delegate_call(&mut vm, test_fn, deploy_tx.address, evm_contract_address); + // Delegate to EVM from EVM. + test_delegate_call( + &mut vm, + test_fn, + evm_contract_address, + other_evm_contract_address, + ); +} + +fn test_delegate_call( + vm: &mut VmTester, + test_fn: ðabi::Function, + from: Address, + to: Address, +) { + let account = &mut vm.rich_accounts[0]; + let test_tx = account.get_l2_tx_for_execute( + Execute { + contract_address: Some(from), + calldata: test_fn.encode_input(&[Token::Address(to)]).unwrap(), + value: 0.into(), + factory_deps: vec![], + }, + None, + ); + let (_, vm_result) = vm + .vm + .execute_transaction_with_bytecode_compression(test_tx, true); + assert!(!vm_result.result.is_failed(), "{vm_result:?}"); +} + +pub(crate) fn test_mock_emulator_with_static_call() { + let evm_contract_address = Address::repeat_byte(0xaa); + let other_evm_contract_address = Address::repeat_byte(0xbb); + let mut builder = EvmTestBuilder::new(true, evm_contract_address); + builder.storage.set_value( + storage_key_for_eth_balance(&evm_contract_address), + H256::from_low_u64_be(1_000_000), + ); + builder.storage.set_value( + storage_key_for_eth_balance(&other_evm_contract_address), + H256::from_low_u64_be(2_000_000), + ); + // Set differing read values for tested contracts. The slot index is defined in the contract. + let value_slot = H256::from_low_u64_be(0x123); + builder.storage.set_value( + StorageKey::new(AccountTreeId::new(evm_contract_address), value_slot), + H256::from_low_u64_be(100), + ); + builder.storage.set_value( + StorageKey::new(AccountTreeId::new(other_evm_contract_address), value_slot), + H256::from_low_u64_be(200), + ); + let mut vm = builder + .with_evm_address(other_evm_contract_address) + .build::(); + let account = &mut vm.rich_accounts[0]; + + // Deploy a native contract. + let native_contract = read_bytecode(INCREMENTING_CONTRACT_PATH); + let native_contract_abi = load_contract(INCREMENTING_CONTRACT_PATH); + let deploy_tx = account.get_deploy_tx(&native_contract, None, TxType::L2); + let (_, vm_result) = vm + .vm + .execute_transaction_with_bytecode_compression(deploy_tx.tx, true); + assert!(!vm_result.result.is_failed(), "{:?}", vm_result.result); + + let test_fn = native_contract_abi.function("testStaticCall").unwrap(); + // Call to the native contract from EVM. + test_static_call(&mut vm, test_fn, evm_contract_address, deploy_tx.address, 0); + // Call to EVM from the native contract. + test_static_call( + &mut vm, + test_fn, + deploy_tx.address, + evm_contract_address, + 100, + ); + // Call to EVM from EVM. + test_static_call( + &mut vm, + test_fn, + evm_contract_address, + other_evm_contract_address, + 200, + ); +} + +fn test_static_call( + vm: &mut VmTester, + test_fn: ðabi::Function, + from: Address, + to: Address, + expected_value: u64, +) { + let account = &mut vm.rich_accounts[0]; + let test_tx = account.get_l2_tx_for_execute( + Execute { + contract_address: Some(from), + calldata: test_fn + .encode_input(&[Token::Address(to), Token::Uint(expected_value.into())]) + .unwrap(), + value: 0.into(), + factory_deps: vec![], + }, + None, + ); + let (_, vm_result) = vm + .vm + .execute_transaction_with_bytecode_compression(test_tx, true); + assert!(!vm_result.result.is_failed(), "{vm_result:?}"); +} diff --git a/core/lib/multivm/src/versions/testonly/mod.rs b/core/lib/multivm/src/versions/testonly/mod.rs index eece1d475bb..309c0edff58 100644 --- a/core/lib/multivm/src/versions/testonly/mod.rs +++ b/core/lib/multivm/src/versions/testonly/mod.rs @@ -39,6 +39,7 @@ pub(super) mod bytecode_publishing; pub(super) mod circuits; pub(super) mod code_oracle; pub(super) mod default_aa; +pub(super) mod evm_emulator; pub(super) mod gas_limit; pub(super) mod get_used_contracts; pub(super) mod is_write_initial; diff --git a/core/lib/multivm/src/versions/vm_fast/circuits_tracer.rs b/core/lib/multivm/src/versions/vm_fast/circuits_tracer.rs index f588f20ab25..1999db47fb9 100644 --- a/core/lib/multivm/src/versions/vm_fast/circuits_tracer.rs +++ b/core/lib/multivm/src/versions/vm_fast/circuits_tracer.rs @@ -7,7 +7,7 @@ use crate::vm_latest::tracers::circuits_capacity::*; /// VM tracer tracking [`CircuitStatistic`]s. Statistics generally depend on the number of time some opcodes were invoked, /// and, for precompiles, invocation complexity (e.g., how many hashing cycles `keccak256` required). #[derive(Debug, Default, Clone, PartialEq)] -pub struct CircuitsTracer { +pub(super) struct CircuitsTracer { main_vm_cycles: u32, ram_permutation_cycles: u32, storage_application_cycles: u32, diff --git a/core/lib/multivm/src/versions/vm_fast/evm_deploy_tracer.rs b/core/lib/multivm/src/versions/vm_fast/evm_deploy_tracer.rs new file mode 100644 index 00000000000..d869796cd2c --- /dev/null +++ b/core/lib/multivm/src/versions/vm_fast/evm_deploy_tracer.rs @@ -0,0 +1,84 @@ +//! Tracer tracking deployment of EVM bytecodes during VM execution. + +use std::{cell::RefCell, collections::HashMap, rc::Rc}; + +use zksync_system_constants::{CONTRACT_DEPLOYER_ADDRESS, KNOWN_CODES_STORAGE_ADDRESS}; +use zksync_types::U256; +use zksync_utils::{bytecode::hash_evm_bytecode, h256_to_u256}; +use zksync_vm2::interface::{ + CallframeInterface, CallingMode, GlobalStateInterface, Opcode, OpcodeType, Tracer, +}; + +use super::utils::read_fat_pointer; + +/// Container for dynamic bytecodes added by [`EvmDeployTracer`]. +#[derive(Debug, Clone, Default)] +pub(super) struct DynamicBytecodes(Rc>>>); + +impl DynamicBytecodes { + pub(super) fn take(&self, hash: U256) -> Option> { + self.0.borrow_mut().remove(&hash) + } + + fn insert(&self, hash: U256, bytecode: Vec) { + self.0.borrow_mut().insert(hash, bytecode); + } +} + +/// Tracer that tracks EVM bytecode deployments. +/// +/// Unlike EraVM bytecodes, EVM bytecodes are *dynamic*; they are not necessarily known before transaction execution. +/// (EraVM bytecodes must be present in the storage or be mentioned in the `factory_deps` field of a transaction.) +/// Hence, it's necessary to track which EVM bytecodes were deployed so that they are persisted after VM execution. +#[derive(Debug)] +pub(super) struct EvmDeployTracer { + tracked_signature: [u8; 4], + bytecodes: DynamicBytecodes, +} + +impl EvmDeployTracer { + pub(super) fn new(bytecodes: DynamicBytecodes) -> Self { + let tracked_signature = + ethabi::short_signature("publishEVMBytecode", &[ethabi::ParamType::Bytes]); + Self { + tracked_signature, + bytecodes, + } + } + + fn handle_far_call(&self, state: &mut impl GlobalStateInterface) { + let from = state.current_frame().caller(); + let to = state.current_frame().code_address(); + if from != CONTRACT_DEPLOYER_ADDRESS || to != KNOWN_CODES_STORAGE_ADDRESS { + return; + } + + let data = read_fat_pointer(state, state.read_register(1).0); + if data.len() < 4 { + return; + } + let (signature, data) = data.split_at(4); + if signature != self.tracked_signature { + return; + } + + match ethabi::decode(&[ethabi::ParamType::Bytes], data) { + Ok(decoded) => { + // `unwrap`s should be safe since the function signature is checked above. + let published_bytecode = decoded.into_iter().next().unwrap().into_bytes().unwrap(); + let bytecode_hash = h256_to_u256(hash_evm_bytecode(&published_bytecode)); + self.bytecodes.insert(bytecode_hash, published_bytecode); + } + Err(err) => tracing::error!("Unable to decode `publishEVMBytecode` call: {err}"), + } + } +} + +impl Tracer for EvmDeployTracer { + #[inline(always)] + fn after_instruction(&mut self, state: &mut S) { + if matches!(OP::VALUE, Opcode::FarCall(CallingMode::Normal)) { + self.handle_far_call(state); + } + } +} diff --git a/core/lib/multivm/src/versions/vm_fast/mod.rs b/core/lib/multivm/src/versions/vm_fast/mod.rs index 35789c6cdc9..de6e7bd4ef6 100644 --- a/core/lib/multivm/src/versions/vm_fast/mod.rs +++ b/core/lib/multivm/src/versions/vm_fast/mod.rs @@ -1,11 +1,12 @@ pub use zksync_vm2::interface; -pub use self::{circuits_tracer::CircuitsTracer, vm::Vm}; +pub use self::vm::Vm; mod bootloader_state; mod bytecode; mod circuits_tracer; mod events; +mod evm_deploy_tracer; mod glue; mod hook; mod initial_bootloader_memory; @@ -14,4 +15,5 @@ mod refund; #[cfg(test)] mod tests; mod transaction_data; +mod utils; mod vm; diff --git a/core/lib/multivm/src/versions/vm_fast/tests/evm_emulator.rs b/core/lib/multivm/src/versions/vm_fast/tests/evm_emulator.rs new file mode 100644 index 00000000000..cb7d54dba29 --- /dev/null +++ b/core/lib/multivm/src/versions/vm_fast/tests/evm_emulator.rs @@ -0,0 +1,53 @@ +use test_casing::{test_casing, Product}; + +use crate::{ + versions::testonly::evm_emulator::{ + test_calling_to_mock_emulator_from_native_contract, test_mock_emulator_basics, + test_mock_emulator_with_delegate_call, test_mock_emulator_with_deployment, + test_mock_emulator_with_payment, test_mock_emulator_with_recursion, + test_mock_emulator_with_static_call, test_tracing_evm_contract_deployment, + }, + vm_fast::Vm, +}; + +#[test] +fn tracing_evm_contract_deployment() { + test_tracing_evm_contract_deployment::>(); +} + +#[test] +fn mock_emulator_basics() { + test_mock_emulator_basics::>(); +} + +#[test_casing(2, [false, true])] +#[test] +fn mock_emulator_with_payment(deploy_emulator: bool) { + test_mock_emulator_with_payment::>(deploy_emulator); +} + +#[test_casing(4, Product(([false, true], [false, true])))] +#[test] +fn mock_emulator_with_recursion(deploy_emulator: bool, is_external: bool) { + test_mock_emulator_with_recursion::>(deploy_emulator, is_external); +} + +#[test] +fn calling_to_mock_emulator_from_native_contract() { + test_calling_to_mock_emulator_from_native_contract::>(); +} + +#[test] +fn mock_emulator_with_deployment() { + test_mock_emulator_with_deployment::>(); +} + +#[test] +fn mock_emulator_with_delegate_call() { + test_mock_emulator_with_delegate_call::>(); +} + +#[test] +fn mock_emulator_with_static_call() { + test_mock_emulator_with_static_call::>(); +} diff --git a/core/lib/multivm/src/versions/vm_fast/tests/mod.rs b/core/lib/multivm/src/versions/vm_fast/tests/mod.rs index 2b4665f8224..27192f46d8d 100644 --- a/core/lib/multivm/src/versions/vm_fast/tests/mod.rs +++ b/core/lib/multivm/src/versions/vm_fast/tests/mod.rs @@ -8,11 +8,11 @@ use zksync_vm_interface::{ VmExecutionMode, VmExecutionResultAndLogs, VmInterface, }; -use super::Vm; +use super::{circuits_tracer::CircuitsTracer, Vm}; use crate::{ interface::storage::{ImmutableStorageView, InMemoryStorage}, versions::testonly::TestedVm, - vm_fast::CircuitsTracer, + vm_fast::evm_deploy_tracer::{DynamicBytecodes, EvmDeployTracer}, }; mod block_tip; @@ -21,6 +21,7 @@ mod bytecode_publishing; mod circuits; mod code_oracle; mod default_aa; +mod evm_emulator; mod gas_limit; mod get_used_contracts; mod is_write_initial; @@ -122,9 +123,13 @@ impl TestedVm for Vm> { } fn manually_decommit(&mut self, code_hash: H256) -> bool { + let mut tracer = ( + ((), CircuitsTracer::default()), + EvmDeployTracer::new(DynamicBytecodes::default()), + ); let (_, is_fresh) = self.inner.world_diff_mut().decommit_opcode( &mut self.world, - &mut ((), CircuitsTracer::default()), + &mut tracer, h256_to_u256(code_hash), ); is_fresh diff --git a/core/lib/multivm/src/versions/vm_fast/utils.rs b/core/lib/multivm/src/versions/vm_fast/utils.rs new file mode 100644 index 00000000000..20a6545d338 --- /dev/null +++ b/core/lib/multivm/src/versions/vm_fast/utils.rs @@ -0,0 +1,13 @@ +use zksync_types::U256; +use zksync_vm2::{interface::StateInterface, FatPointer}; + +pub(super) fn read_fat_pointer(state: &S, raw: U256) -> Vec { + let pointer = FatPointer::from(raw); + let length = pointer.length - pointer.offset; + let start = pointer.start + pointer.offset; + let mut result = vec![0; length as usize]; + for i in 0..length { + result[i as usize] = state.read_heap_byte(pointer.memory_page, start + i); + } + result +} diff --git a/core/lib/multivm/src/versions/vm_fast/vm.rs b/core/lib/multivm/src/versions/vm_fast/vm.rs index 6ebc4b9c571..b7113bf8f19 100644 --- a/core/lib/multivm/src/versions/vm_fast/vm.rs +++ b/core/lib/multivm/src/versions/vm_fast/vm.rs @@ -27,6 +27,7 @@ use super::{ bootloader_state::{BootloaderState, BootloaderStateSnapshot}, bytecode::compress_bytecodes, circuits_tracer::CircuitsTracer, + evm_deploy_tracer::{DynamicBytecodes, EvmDeployTracer}, hook::Hook, initial_bootloader_memory::bootloader_initial_memory, transaction_data::TransactionData, @@ -54,13 +55,14 @@ use crate::{ get_result_success_first_slot, get_vm_hook_params_start_position, get_vm_hook_position, OPERATOR_REFUNDS_OFFSET, TX_GAS_LIMIT_OFFSET, VM_HOOK_PARAMS_COUNT, }, + utils::extract_bytecodes_marked_as_known, MultiVMSubversion, }, }; const VM_VERSION: MultiVMSubversion = MultiVMSubversion::IncreasedBootloaderMemory; -type FullTracer = (Tr, CircuitsTracer); +type FullTracer = ((Tr, CircuitsTracer), EvmDeployTracer); #[derive(Debug)] struct VmRunResult { @@ -93,12 +95,12 @@ impl VmRunResult { /// and implement [`Default`] (the latter is necessary to complete batches). [`CircuitsTracer`] is currently always enabled; /// you don't need to specify it explicitly. pub struct Vm { - pub(crate) world: World>, - pub(crate) inner: VirtualMachine, World>>, + pub(super) world: World>, + pub(super) inner: VirtualMachine, World>>, gas_for_account_validation: u32, - pub(crate) bootloader_state: BootloaderState, - pub(crate) batch_env: L1BatchEnv, - pub(crate) system_env: SystemEnv, + pub(super) bootloader_state: BootloaderState, + pub(super) batch_env: L1BatchEnv, + pub(super) system_env: SystemEnv, snapshot: Option, #[cfg(test)] enforced_state_diffs: Option>, @@ -112,16 +114,22 @@ impl Vm { system_env.version ); - let default_aa_code_hash = system_env + let default_aa_code_hash = system_env.base_system_smart_contracts.default_aa.hash; + let evm_emulator_hash = system_env .base_system_smart_contracts - .default_aa - .hash - .into(); + .evm_emulator + .as_ref() + .map(|evm| evm.hash) + .unwrap_or(system_env.base_system_smart_contracts.default_aa.hash); - let program_cache = HashMap::from([World::convert_system_contract_code( + let mut program_cache = HashMap::from([World::convert_system_contract_code( &system_env.base_system_smart_contracts.default_aa, false, )]); + if let Some(evm_emulator) = &system_env.base_system_smart_contracts.evm_emulator { + let (bytecode_hash, program) = World::convert_system_contract_code(evm_emulator, false); + program_cache.insert(bytecode_hash, program); + } let (_, bootloader) = World::convert_system_contract_code( &system_env.base_system_smart_contracts.bootloader, @@ -136,9 +144,8 @@ impl Vm { &[], system_env.bootloader_gas_limit, Settings { - default_aa_code_hash, - // this will change after 1.5 - evm_interpreter_code_hash: default_aa_code_hash, + default_aa_code_hash: default_aa_code_hash.into(), + evm_interpreter_code_hash: evm_emulator_hash.into(), hook_address: get_vm_hook_position(VM_VERSION) * 32, }, ); @@ -173,7 +180,7 @@ impl Vm { fn run( &mut self, execution_mode: VmExecutionMode, - tracer: &mut (Tr, CircuitsTracer), + tracer: &mut FullTracer, track_refunds: bool, ) -> VmRunResult { let mut refunds = Refunds { @@ -571,9 +578,13 @@ impl Vm { let start = self.inner.world_diff().snapshot(); let gas_before = self.gas_remaining(); - let mut full_tracer = (mem::take(tracer), CircuitsTracer::default()); + let mut full_tracer = ( + (mem::take(tracer), CircuitsTracer::default()), + EvmDeployTracer::new(self.world.dynamic_bytecodes.clone()), + ); let result = self.run(execution_mode, &mut full_tracer, track_refunds); - *tracer = full_tracer.0; // place the tracer back + let ((external_tracer, circuits_tracer), _) = full_tracer; + *tracer = external_tracer; // place the tracer back let ignore_world_diff = matches!(execution_mode, VmExecutionMode::OneTx) && result.should_ignore_vm_logs(); @@ -630,6 +641,11 @@ impl Vm { let gas_remaining = self.gas_remaining(); let gas_used = gas_before - gas_remaining; + // We need to filter out bytecodes the deployment of which may have been reverted; the tracer is not aware of reverts. + // To do this, we check bytecodes against deployer events. + let factory_deps_marked_as_known = extract_bytecodes_marked_as_known(&logs.events); + let new_known_factory_deps = self.world.decommit_bytecodes(&factory_deps_marked_as_known); + VmExecutionResultAndLogs { result: result.execution_result, logs, @@ -639,13 +655,13 @@ impl Vm { gas_remaining, computational_gas_used: gas_used, // since 1.5.0, this always has the same value as `gas_used` pubdata_published: result.pubdata_published, - circuit_statistic: full_tracer.1.circuit_statistic(), + circuit_statistic: circuits_tracer.circuit_statistic(), contracts_used: 0, cycles_used: 0, total_log_queries: 0, }, refunds: result.refunds, - new_known_factory_deps: None, + new_known_factory_deps: Some(new_known_factory_deps), } } } @@ -797,6 +813,7 @@ impl fmt::Debug for Vm { #[derive(Debug)] pub(crate) struct World { pub(crate) storage: S, + dynamic_bytecodes: DynamicBytecodes, program_cache: HashMap>, pub(crate) bytecode_cache: HashMap>, } @@ -805,8 +822,9 @@ impl World { fn new(storage: S, program_cache: HashMap>) -> Self { Self { storage, + dynamic_bytecodes: DynamicBytecodes::default(), program_cache, - bytecode_cache: Default::default(), + bytecode_cache: HashMap::default(), } } @@ -819,6 +837,20 @@ impl World { Program::from_words(code.code.clone(), is_bootloader), ) } + + fn decommit_bytecodes(&self, hashes: &[H256]) -> HashMap> { + let bytecodes = hashes.iter().map(|&hash| { + let int_hash = h256_to_u256(hash); + let bytecode = self + .bytecode_cache + .get(&int_hash) + .cloned() + .or_else(|| self.dynamic_bytecodes.take(int_hash)) + .unwrap_or_else(|| panic!("Bytecode with hash {hash:?} not found")); + (hash, bytecode) + }); + bytecodes.collect() + } } impl zksync_vm2::StorageInterface for World { @@ -872,15 +904,34 @@ impl zksync_vm2::StorageInterface for World { } } +/// It may look like that an append-only cache for EVM bytecodes / `Program`s can lead to the following scenario: +/// +/// 1. A transaction deploys an EVM bytecode with hash `H`, then reverts. +/// 2. A following transaction in the same VM run queries a bytecode with hash `H` and gets it. +/// +/// This would be incorrect behavior because bytecode deployments must be reverted along with transactions. +/// +/// In reality, this cannot happen because both `decommit()` and `decommit_code()` calls perform storage-based checks +/// before a decommit: +/// +/// - `decommit_code()` is called from the `CodeOracle` system contract, which checks that the decommitted bytecode is known. +/// - `decommit()` is called during far calls, which obtains address -> bytecode hash mapping beforehand. +/// +/// Thus, if storage is reverted correctly, additional EVM bytecodes occupy the cache, but are unreachable. impl zksync_vm2::World for World { fn decommit(&mut self, hash: U256) -> Program { self.program_cache .entry(hash) .or_insert_with(|| { let bytecode = self.bytecode_cache.entry(hash).or_insert_with(|| { - self.storage - .load_factory_dep(u256_to_h256(hash)) - .expect("vm tried to decommit nonexistent bytecode") + // Since we put the bytecode in the cache anyway, it's safe to *take* it out from `dynamic_bytecodes` + // and put it in `bytecode_cache`. + self.dynamic_bytecodes + .take(hash) + .or_else(|| self.storage.load_factory_dep(u256_to_h256(hash))) + .unwrap_or_else(|| { + panic!("VM tried to decommit nonexistent bytecode: {hash:?}"); + }) }); Program::new(bytecode, false) }) diff --git a/core/lib/multivm/src/versions/vm_latest/tests/evm_emulator.rs b/core/lib/multivm/src/versions/vm_latest/tests/evm_emulator.rs index 4d6e77aed51..b9b96c67098 100644 --- a/core/lib/multivm/src/versions/vm_latest/tests/evm_emulator.rs +++ b/core/lib/multivm/src/versions/vm_latest/tests/evm_emulator.rs @@ -1,507 +1,53 @@ -use std::collections::HashMap; - -use ethabi::Token; use test_casing::{test_casing, Product}; -use zksync_contracts::{load_contract, read_bytecode, SystemContractCode}; -use zksync_system_constants::{ - CONTRACT_DEPLOYER_ADDRESS, KNOWN_CODES_STORAGE_ADDRESS, L2_BASE_TOKEN_ADDRESS, -}; -use zksync_test_account::TxType; -use zksync_types::{ - get_code_key, get_known_code_key, - utils::{key_for_eth_balance, storage_key_for_eth_balance}, - AccountTreeId, Address, Execute, StorageKey, H256, U256, -}; -use zksync_utils::{ - be_words_to_bytes, - bytecode::{hash_bytecode, hash_evm_bytecode}, - bytes_to_be_words, h256_to_u256, -}; -use super::TestedLatestVm; use crate::{ - interface::{ - storage::InMemoryStorage, TxExecutionMode, VmExecutionResultAndLogs, VmInterfaceExt, + versions::testonly::evm_emulator::{ + test_calling_to_mock_emulator_from_native_contract, test_mock_emulator_basics, + test_mock_emulator_with_delegate_call, test_mock_emulator_with_deployment, + test_mock_emulator_with_payment, test_mock_emulator_with_recursion, + test_mock_emulator_with_static_call, test_tracing_evm_contract_deployment, }, - versions::testonly::{default_system_env, VmTester, VmTesterBuilder}, + vm_latest::{HistoryEnabled, Vm}, }; -const MOCK_DEPLOYER_PATH: &str = "etc/contracts-test-data/artifacts-zk/contracts/mock-evm/mock-evm.sol/MockContractDeployer.json"; -const MOCK_KNOWN_CODE_STORAGE_PATH: &str = "etc/contracts-test-data/artifacts-zk/contracts/mock-evm/mock-evm.sol/MockKnownCodeStorage.json"; -const MOCK_EMULATOR_PATH: &str = - "etc/contracts-test-data/artifacts-zk/contracts/mock-evm/mock-evm.sol/MockEvmEmulator.json"; -const RECURSIVE_CONTRACT_PATH: &str = "etc/contracts-test-data/artifacts-zk/contracts/mock-evm/mock-evm.sol/NativeRecursiveContract.json"; -const INCREMENTING_CONTRACT_PATH: &str = "etc/contracts-test-data/artifacts-zk/contracts/mock-evm/mock-evm.sol/IncrementingContract.json"; - -fn override_system_contracts(storage: &mut InMemoryStorage) { - let mock_deployer = read_bytecode(MOCK_DEPLOYER_PATH); - let mock_deployer_hash = hash_bytecode(&mock_deployer); - let mock_known_code_storage = read_bytecode(MOCK_KNOWN_CODE_STORAGE_PATH); - let mock_known_code_storage_hash = hash_bytecode(&mock_known_code_storage); - - storage.set_value(get_code_key(&CONTRACT_DEPLOYER_ADDRESS), mock_deployer_hash); - storage.set_value( - get_known_code_key(&mock_deployer_hash), - H256::from_low_u64_be(1), - ); - storage.set_value( - get_code_key(&KNOWN_CODES_STORAGE_ADDRESS), - mock_known_code_storage_hash, - ); - storage.set_value( - get_known_code_key(&mock_known_code_storage_hash), - H256::from_low_u64_be(1), - ); - storage.store_factory_dep(mock_deployer_hash, mock_deployer); - storage.store_factory_dep(mock_known_code_storage_hash, mock_known_code_storage); -} - -#[derive(Debug)] -struct EvmTestBuilder { - deploy_emulator: bool, - storage: InMemoryStorage, - evm_contract_addresses: Vec
, -} - -impl EvmTestBuilder { - fn new(deploy_emulator: bool, evm_contract_address: Address) -> Self { - Self { - deploy_emulator, - storage: InMemoryStorage::with_system_contracts(hash_bytecode), - evm_contract_addresses: vec![evm_contract_address], - } - } - - fn with_mock_deployer(mut self) -> Self { - override_system_contracts(&mut self.storage); - self - } - - fn with_evm_address(mut self, address: Address) -> Self { - self.evm_contract_addresses.push(address); - self - } - - fn build(self) -> VmTester { - let mock_emulator = read_bytecode(MOCK_EMULATOR_PATH); - let mut storage = self.storage; - let mut system_env = default_system_env(); - if self.deploy_emulator { - let evm_bytecode: Vec<_> = (0..32).collect(); - let evm_bytecode_hash = hash_evm_bytecode(&evm_bytecode); - storage.set_value( - get_known_code_key(&evm_bytecode_hash), - H256::from_low_u64_be(1), - ); - for evm_address in self.evm_contract_addresses { - storage.set_value(get_code_key(&evm_address), evm_bytecode_hash); - } - - system_env.base_system_smart_contracts.evm_emulator = Some(SystemContractCode { - hash: hash_bytecode(&mock_emulator), - code: bytes_to_be_words(mock_emulator), - }); - } else { - let emulator_hash = hash_bytecode(&mock_emulator); - storage.set_value(get_known_code_key(&emulator_hash), H256::from_low_u64_be(1)); - storage.store_factory_dep(emulator_hash, mock_emulator); - - for evm_address in self.evm_contract_addresses { - storage.set_value(get_code_key(&evm_address), emulator_hash); - // Set `isUserSpace` in the emulator storage to `true`, so that it skips emulator-specific checks - storage.set_value( - StorageKey::new(AccountTreeId::new(evm_address), H256::zero()), - H256::from_low_u64_be(1), - ); - } - } - - VmTesterBuilder::new() - .with_system_env(system_env) - .with_storage(storage) - .with_execution_mode(TxExecutionMode::VerifyExecute) - .with_rich_accounts(1) - .build() - } -} - #[test] fn tracing_evm_contract_deployment() { - let mut storage = InMemoryStorage::with_system_contracts(hash_bytecode); - override_system_contracts(&mut storage); - - let mut system_env = default_system_env(); - // The EVM emulator will not be accessed, so we set it to a dummy value. - system_env.base_system_smart_contracts.evm_emulator = - Some(system_env.base_system_smart_contracts.default_aa.clone()); - let mut vm = VmTesterBuilder::new() - .with_system_env(system_env) - .with_storage(storage) - .with_execution_mode(TxExecutionMode::VerifyExecute) - .with_rich_accounts(1) - .build::(); - let account = &mut vm.rich_accounts[0]; - - let args = [Token::Bytes((0..32).collect())]; - let evm_bytecode = ethabi::encode(&args); - let expected_bytecode_hash = hash_evm_bytecode(&evm_bytecode); - let execute = Execute::for_deploy(expected_bytecode_hash, vec![0; 32], &args); - let deploy_tx = account.get_l2_tx_for_execute(execute, None); - let (_, vm_result) = vm - .vm - .execute_transaction_with_bytecode_compression(deploy_tx, true); - assert!(!vm_result.result.is_failed(), "{:?}", vm_result.result); - - // Check that the surrogate EVM bytecode was added to the decommitter. - let known_bytecodes = vm.vm.state.decommittment_processor.known_bytecodes.inner(); - let known_evm_bytecode = - be_words_to_bytes(&known_bytecodes[&h256_to_u256(expected_bytecode_hash)]); - assert_eq!(known_evm_bytecode, evm_bytecode); - - let new_known_factory_deps = vm_result.new_known_factory_deps.unwrap(); - assert_eq!(new_known_factory_deps.len(), 2); // the deployed EraVM contract + EVM contract - assert_eq!( - new_known_factory_deps[&expected_bytecode_hash], - evm_bytecode - ); + test_tracing_evm_contract_deployment::>(); } #[test] fn mock_emulator_basics() { - let called_address = Address::repeat_byte(0x23); - let mut vm = EvmTestBuilder::new(true, called_address).build(); - let account = &mut vm.rich_accounts[0]; - let tx = account.get_l2_tx_for_execute( - Execute { - contract_address: Some(called_address), - calldata: vec![], - value: 0.into(), - factory_deps: vec![], - }, - None, - ); - - let (_, vm_result) = vm - .vm - .execute_transaction_with_bytecode_compression(tx, true); - assert!(!vm_result.result.is_failed(), "{:?}", vm_result.result); + test_mock_emulator_basics::>(); } -const RECIPIENT_ADDRESS: Address = Address::repeat_byte(0x12); - -/// `deploy_emulator = false` here and below tests the mock emulator as an ordinary contract (i.e., sanity-checks its logic). #[test_casing(2, [false, true])] #[test] fn mock_emulator_with_payment(deploy_emulator: bool) { - let mock_emulator_abi = load_contract(MOCK_EMULATOR_PATH); - let mut vm = EvmTestBuilder::new(deploy_emulator, RECIPIENT_ADDRESS).build(); - - let mut current_balance = U256::zero(); - for i in 1_u64..=5 { - let transferred_value = (1_000_000_000 * i).into(); - let vm_result = test_payment( - &mut vm, - &mock_emulator_abi, - &mut current_balance, - transferred_value, - ); - - let balance_storage_logs = vm_result.logs.storage_logs.iter().filter_map(|log| { - (*log.log.key.address() == L2_BASE_TOKEN_ADDRESS) - .then_some((*log.log.key.key(), h256_to_u256(log.log.value))) - }); - let balances: HashMap<_, _> = balance_storage_logs.collect(); - assert_eq!( - balances[&key_for_eth_balance(&RECIPIENT_ADDRESS)], - current_balance - ); - } -} - -fn test_payment( - vm: &mut VmTester, - mock_emulator_abi: ðabi::Contract, - balance: &mut U256, - transferred_value: U256, -) -> VmExecutionResultAndLogs { - *balance += transferred_value; - let test_payment_fn = mock_emulator_abi.function("testPayment").unwrap(); - let account = &mut vm.rich_accounts[0]; - let tx = account.get_l2_tx_for_execute( - Execute { - contract_address: Some(RECIPIENT_ADDRESS), - calldata: test_payment_fn - .encode_input(&[Token::Uint(transferred_value), Token::Uint(*balance)]) - .unwrap(), - value: transferred_value, - factory_deps: vec![], - }, - None, - ); - - let (_, vm_result) = vm - .vm - .execute_transaction_with_bytecode_compression(tx, true); - assert!(!vm_result.result.is_failed(), "{vm_result:?}"); - vm_result + test_mock_emulator_with_payment::>(deploy_emulator); } #[test_casing(4, Product(([false, true], [false, true])))] #[test] fn mock_emulator_with_recursion(deploy_emulator: bool, is_external: bool) { - let mock_emulator_abi = load_contract(MOCK_EMULATOR_PATH); - let recipient_address = Address::repeat_byte(0x12); - let mut vm = EvmTestBuilder::new(deploy_emulator, recipient_address).build(); - let account = &mut vm.rich_accounts[0]; - - let test_recursion_fn = mock_emulator_abi - .function(if is_external { - "testExternalRecursion" - } else { - "testRecursion" - }) - .unwrap(); - let mut expected_value = U256::one(); - let depth = 50_u32; - for i in 2..=depth { - expected_value *= i; - } - - let factory_deps = if is_external { - vec![read_bytecode(RECURSIVE_CONTRACT_PATH)] - } else { - vec![] - }; - let tx = account.get_l2_tx_for_execute( - Execute { - contract_address: Some(recipient_address), - calldata: test_recursion_fn - .encode_input(&[Token::Uint(depth.into()), Token::Uint(expected_value)]) - .unwrap(), - value: 0.into(), - factory_deps, - }, - None, - ); - let (_, vm_result) = vm - .vm - .execute_transaction_with_bytecode_compression(tx, true); - assert!(!vm_result.result.is_failed(), "{vm_result:?}"); + test_mock_emulator_with_recursion::>(deploy_emulator, is_external); } #[test] fn calling_to_mock_emulator_from_native_contract() { - let recipient_address = Address::repeat_byte(0x12); - let mut vm = EvmTestBuilder::new(true, recipient_address).build(); - let account = &mut vm.rich_accounts[0]; - - // Deploy a native contract. - let native_contract = read_bytecode(RECURSIVE_CONTRACT_PATH); - let native_contract_abi = load_contract(RECURSIVE_CONTRACT_PATH); - let deploy_tx = account.get_deploy_tx( - &native_contract, - Some(&[Token::Address(recipient_address)]), - TxType::L2, - ); - let (_, vm_result) = vm - .vm - .execute_transaction_with_bytecode_compression(deploy_tx.tx, true); - assert!(!vm_result.result.is_failed(), "{:?}", vm_result.result); - - // Call from the native contract to the EVM emulator. - let test_fn = native_contract_abi.function("recurse").unwrap(); - let test_tx = account.get_l2_tx_for_execute( - Execute { - contract_address: Some(deploy_tx.address), - calldata: test_fn.encode_input(&[Token::Uint(50.into())]).unwrap(), - value: Default::default(), - factory_deps: vec![], - }, - None, - ); - let (_, vm_result) = vm - .vm - .execute_transaction_with_bytecode_compression(test_tx, true); - assert!(!vm_result.result.is_failed(), "{:?}", vm_result.result); + test_calling_to_mock_emulator_from_native_contract::>(); } #[test] fn mock_emulator_with_deployment() { - let contract_address = Address::repeat_byte(0xaa); - let mut vm = EvmTestBuilder::new(true, contract_address) - .with_mock_deployer() - .build(); - let account = &mut vm.rich_accounts[0]; - - let mock_emulator_abi = load_contract(MOCK_EMULATOR_PATH); - let new_evm_bytecode = vec![0xfe; 96]; - let new_evm_bytecode_hash = hash_evm_bytecode(&new_evm_bytecode); - - let test_fn = mock_emulator_abi.function("testDeploymentAndCall").unwrap(); - let test_tx = account.get_l2_tx_for_execute( - Execute { - contract_address: Some(contract_address), - calldata: test_fn - .encode_input(&[ - Token::FixedBytes(new_evm_bytecode_hash.0.into()), - Token::Bytes(new_evm_bytecode.clone()), - ]) - .unwrap(), - value: 0.into(), - factory_deps: vec![], - }, - None, - ); - let (_, vm_result) = vm - .vm - .execute_transaction_with_bytecode_compression(test_tx, true); - assert!(!vm_result.result.is_failed(), "{vm_result:?}"); - - let factory_deps = vm_result.new_known_factory_deps.unwrap(); - assert_eq!( - factory_deps, - HashMap::from([(new_evm_bytecode_hash, new_evm_bytecode)]) - ); + test_mock_emulator_with_deployment::>(); } #[test] fn mock_emulator_with_delegate_call() { - let evm_contract_address = Address::repeat_byte(0xaa); - let other_evm_contract_address = Address::repeat_byte(0xbb); - let mut builder = EvmTestBuilder::new(true, evm_contract_address); - builder.storage.set_value( - storage_key_for_eth_balance(&evm_contract_address), - H256::from_low_u64_be(1_000_000), - ); - builder.storage.set_value( - storage_key_for_eth_balance(&other_evm_contract_address), - H256::from_low_u64_be(2_000_000), - ); - let mut vm = builder.with_evm_address(other_evm_contract_address).build(); - let account = &mut vm.rich_accounts[0]; - - // Deploy a native contract. - let native_contract = read_bytecode(INCREMENTING_CONTRACT_PATH); - let native_contract_abi = load_contract(INCREMENTING_CONTRACT_PATH); - let deploy_tx = account.get_deploy_tx(&native_contract, None, TxType::L2); - let (_, vm_result) = vm - .vm - .execute_transaction_with_bytecode_compression(deploy_tx.tx, true); - assert!(!vm_result.result.is_failed(), "{:?}", vm_result.result); - - let test_fn = native_contract_abi.function("testDelegateCall").unwrap(); - // Delegate to the native contract from EVM. - test_delegate_call(&mut vm, test_fn, evm_contract_address, deploy_tx.address); - // Delegate to EVM from the native contract. - test_delegate_call(&mut vm, test_fn, deploy_tx.address, evm_contract_address); - // Delegate to EVM from EVM. - test_delegate_call( - &mut vm, - test_fn, - evm_contract_address, - other_evm_contract_address, - ); -} - -fn test_delegate_call( - vm: &mut VmTester, - test_fn: ðabi::Function, - from: Address, - to: Address, -) { - let account = &mut vm.rich_accounts[0]; - let test_tx = account.get_l2_tx_for_execute( - Execute { - contract_address: Some(from), - calldata: test_fn.encode_input(&[Token::Address(to)]).unwrap(), - value: 0.into(), - factory_deps: vec![], - }, - None, - ); - let (_, vm_result) = vm - .vm - .execute_transaction_with_bytecode_compression(test_tx, true); - assert!(!vm_result.result.is_failed(), "{vm_result:?}"); + test_mock_emulator_with_delegate_call::>(); } #[test] fn mock_emulator_with_static_call() { - let evm_contract_address = Address::repeat_byte(0xaa); - let other_evm_contract_address = Address::repeat_byte(0xbb); - let mut builder = EvmTestBuilder::new(true, evm_contract_address); - builder.storage.set_value( - storage_key_for_eth_balance(&evm_contract_address), - H256::from_low_u64_be(1_000_000), - ); - builder.storage.set_value( - storage_key_for_eth_balance(&other_evm_contract_address), - H256::from_low_u64_be(2_000_000), - ); - // Set differing read values for tested contracts. The slot index is defined in the contract. - let value_slot = H256::from_low_u64_be(0x123); - builder.storage.set_value( - StorageKey::new(AccountTreeId::new(evm_contract_address), value_slot), - H256::from_low_u64_be(100), - ); - builder.storage.set_value( - StorageKey::new(AccountTreeId::new(other_evm_contract_address), value_slot), - H256::from_low_u64_be(200), - ); - let mut vm = builder.with_evm_address(other_evm_contract_address).build(); - let account = &mut vm.rich_accounts[0]; - - // Deploy a native contract. - let native_contract = read_bytecode(INCREMENTING_CONTRACT_PATH); - let native_contract_abi = load_contract(INCREMENTING_CONTRACT_PATH); - let deploy_tx = account.get_deploy_tx(&native_contract, None, TxType::L2); - let (_, vm_result) = vm - .vm - .execute_transaction_with_bytecode_compression(deploy_tx.tx, true); - assert!(!vm_result.result.is_failed(), "{:?}", vm_result.result); - - let test_fn = native_contract_abi.function("testStaticCall").unwrap(); - // Call to the native contract from EVM. - test_static_call(&mut vm, test_fn, evm_contract_address, deploy_tx.address, 0); - // Call to EVM from the native contract. - test_static_call( - &mut vm, - test_fn, - deploy_tx.address, - evm_contract_address, - 100, - ); - // Call to EVM from EVM. - test_static_call( - &mut vm, - test_fn, - evm_contract_address, - other_evm_contract_address, - 200, - ); -} - -fn test_static_call( - vm: &mut VmTester, - test_fn: ðabi::Function, - from: Address, - to: Address, - expected_value: u64, -) { - let account = &mut vm.rich_accounts[0]; - let test_tx = account.get_l2_tx_for_execute( - Execute { - contract_address: Some(from), - calldata: test_fn - .encode_input(&[Token::Address(to), Token::Uint(expected_value.into())]) - .unwrap(), - value: 0.into(), - factory_deps: vec![], - }, - None, - ); - let (_, vm_result) = vm - .vm - .execute_transaction_with_bytecode_compression(test_tx, true); - assert!(!vm_result.result.is_failed(), "{vm_result:?}"); + test_mock_emulator_with_static_call::>(); } diff --git a/core/lib/vm_interface/src/utils/shadow.rs b/core/lib/vm_interface/src/utils/shadow.rs index d12d85fa2e3..060c0429854 100644 --- a/core/lib/vm_interface/src/utils/shadow.rs +++ b/core/lib/vm_interface/src/utils/shadow.rs @@ -189,6 +189,16 @@ impl CheckDivergence for VmExecutionResultAndLogs { &self.statistics.computational_gas_used, &other.statistics.computational_gas_used, ); + + if let (Some(these_deps), Some(other_deps)) = + (&self.new_known_factory_deps, &other.new_known_factory_deps) + { + // Order deps to have a more reasonable diff on a mismatch + let these_deps = these_deps.iter().collect::>(); + let other_deps = other_deps.iter().collect::>(); + errors.check_match("new_known_factory_deps", &these_deps, &other_deps); + } + errors } } From f1328c033e037a8229e7b3bb99fd581e64847f60 Mon Sep 17 00:00:00 2001 From: Joonatan Saarhelo Date: Thu, 31 Oct 2024 14:54:02 +0000 Subject: [PATCH 3/3] chore: make artificially shortened executions count as successes (#3204) Currently, successful validations are reported as reverts, which is super confusing. Why this is safe: - `TracerExecutionStopReason::Finished` is only emitted on success, Errors always emit Abort instead - Some of the removed code was just dead code. `self.result` was checked twice. I'd rather fix the strange behaviour than emulate it to make ShadowVm line up. --- .../src/versions/vm_latest/tracers/default_tracers.rs | 5 ++++- .../src/versions/vm_latest/tracers/result_tracer.rs | 9 ++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/core/lib/multivm/src/versions/vm_latest/tracers/default_tracers.rs b/core/lib/multivm/src/versions/vm_latest/tracers/default_tracers.rs index 2ae5e81a328..7156acce152 100755 --- a/core/lib/multivm/src/versions/vm_latest/tracers/default_tracers.rs +++ b/core/lib/multivm/src/versions/vm_latest/tracers/default_tracers.rs @@ -228,7 +228,10 @@ impl Tracer for DefaultExecutionTracer { ); match hook { - VmHook::TxHasEnded => self.tx_has_been_processed = true, + VmHook::TxHasEnded if matches!(self.execution_mode, VmExecutionMode::OneTx) => { + self.result_tracer.tx_finished_in_one_tx_mode = true; + self.tx_has_been_processed = true; + } VmHook::NoValidationEntered => self.in_account_validation = false, VmHook::AccountValidationEntered => self.in_account_validation = true, VmHook::FinalBatchInfo => self.final_batch_info_requested = true, diff --git a/core/lib/multivm/src/versions/vm_latest/tracers/result_tracer.rs b/core/lib/multivm/src/versions/vm_latest/tracers/result_tracer.rs index 6ba00f4a099..0687c8393c6 100644 --- a/core/lib/multivm/src/versions/vm_latest/tracers/result_tracer.rs +++ b/core/lib/multivm/src/versions/vm_latest/tracers/result_tracer.rs @@ -104,6 +104,8 @@ pub(crate) struct ResultTracer { far_call_tracker: FarCallTracker, subversion: MultiVMSubversion, + pub(crate) tx_finished_in_one_tx_mode: bool, + _phantom: PhantomData, } @@ -115,6 +117,7 @@ impl ResultTracer { execution_mode, far_call_tracker: Default::default(), subversion, + tx_finished_in_one_tx_mode: false, _phantom: PhantomData, } } @@ -297,7 +300,7 @@ impl ResultTracer { let has_failed = tx_has_failed(state, bootloader_state.current_tx() as u32, self.subversion); - if has_failed { + if self.tx_finished_in_one_tx_mode && has_failed { self.result = Some(Result::Error { error_reason: VmRevertReason::General { msg: "Transaction reverted with empty reason. Possibly out of gas" @@ -306,9 +309,9 @@ impl ResultTracer { }, }); } else { - self.result = Some(self.result.clone().unwrap_or(Result::Success { + self.result = Some(Result::Success { return_data: vec![], - })); + }); } } }