Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(db): SyncDatabase for cross-chain state access #44

Merged
merged 37 commits into from
Oct 11, 2024
Merged
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
58731cd
compiled SyncDatabse
CeciliaZ030 Sep 21, 2024
25a064a
post_block_balance_increments + ProviderError
CeciliaZ030 Sep 21, 2024
2134fc1
clippy & fmt
CeciliaZ030 Sep 21, 2024
79ee267
most tests passed
CeciliaZ030 Sep 23, 2024
22fee2b
most tests pass
CeciliaZ030 Sep 23, 2024
04b8f42
update patch to git
CeciliaZ030 Sep 23, 2024
a42b1a7
fix more chain_id
CeciliaZ030 Sep 27, 2024
e5bcd94
prints
CeciliaZ030 Sep 27, 2024
af50cf9
execution_outcome.all_states() fixs blockhash
CeciliaZ030 Sep 27, 2024
9208f26
current_state
CeciliaZ030 Sep 27, 2024
462135b
Merge branch 'print-shit-out' into gwyneth-cecilia
CeciliaZ030 Sep 27, 2024
a0091f7
no extra println
CeciliaZ030 Sep 27, 2024
27cfd6b
force execution outcome to have chain_id
CeciliaZ030 Sep 27, 2024
4f1d86c
fmt
CeciliaZ030 Sep 30, 2024
1f7ea4e
update
CeciliaZ030 Oct 1, 2024
684054c
clippy & clean up
CeciliaZ030 Oct 1, 2024
b3f7718
clean up more
CeciliaZ030 Oct 1, 2024
88d055d
comments
CeciliaZ030 Oct 4, 2024
ce68b12
fix bench util
CeciliaZ030 Oct 4, 2024
71c7078
fix tx_env chain_id
CeciliaZ030 Oct 7, 2024
cfa0c55
rebase make install procedure to newest base
Oct 8, 2024
011d89f
rbuilder compat changes
Brechtpd Oct 8, 2024
b699a7f
Merge remote-tracking branch 'origin/gwyneth-cecilia' into rbuilderV3
Brechtpd Oct 8, 2024
a764b2f
fix updated packages giving small incompatibility issues
Brechtpd Oct 8, 2024
6bf500f
fix tx chain_id again
CeciliaZ030 Oct 9, 2024
34856df
rbuilder fixed db/rpc and fixes
Brechtpd Oct 9, 2024
7faee55
static blockscout ports
Oct 10, 2024
681003a
add rbuilder start after make install
Oct 10, 2024
7c010b2
cleanup
Brechtpd Oct 10, 2024
166cd4e
Merge remote-tracking branch 'origin/rbuilderV2' into rbuilderV3
Brechtpd Oct 10, 2024
c39bbbf
stream rbuilder logs into tmp/rbuilder.log
Oct 11, 2024
40ccc2a
Merge remote-tracking branch 'origin/rbuilderV2' into rbuilderV3
Brechtpd Oct 11, 2024
0b7ef07
add script for shoot up a second node
Oct 11, 2024
b98ecfa
Add Gwyneth instructions
Brechtpd Oct 11, 2024
b8caa86
Merge pull request #45 from taikoxyz/rbuilderV2
Brechtpd Oct 11, 2024
d2beb2f
Merge branch 'gwyneth-cecilia' into rbuilderV3
Brechtpd Oct 11, 2024
be2e95a
Merge pull request #46 from taikoxyz/rbuilderV3
Brechtpd Oct 11, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
prints
CeciliaZ030 committed Sep 27, 2024
commit e5bcd94dabe22252063a4bc688b6ab50410ce61d
1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -266,6 +266,7 @@ codegen-units = 8
inherits = "release"
debug = 2
strip = false
panic = "unwind"

# Make sure debug symbols are in the bench profile
[profile.bench]
10 changes: 7 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -16,7 +16,7 @@ FROM chef AS builder
COPY --from=planner /app/recipe.json recipe.json

# Build profile, release by default
ARG BUILD_PROFILE=release
ARG BUILD_PROFILE=profiling
ENV BUILD_PROFILE=$BUILD_PROFILE

# Extra Cargo flags
@@ -27,12 +27,16 @@ ENV RUSTFLAGS="$RUSTFLAGS"
ARG FEATURES=""
ENV FEATURES=$FEATURES

# Extra Cargo features
ARG RUST_BACKTRACE=1
ENV RUST_BACKTRACE=$RUST_BACKTRACE

# Builds dependencies
RUN cargo chef cook --profile $BUILD_PROFILE --features "$FEATURES" --recipe-path recipe.json
RUN RUST_BACKTRACE=1 cargo chef cook --profile $BUILD_PROFILE --features "$FEATURES" --recipe-path recipe.json

# Build application
COPY . .
RUN cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin reth
RUN RUST_BACKTRACE=1 cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin reth

# ARG is not resolved in COPY so we have to hack around it by copying the
# binary to a temporary location
2 changes: 1 addition & 1 deletion bin/reth/Cargo.toml
Original file line number Diff line number Diff line change
@@ -129,7 +129,7 @@ libc = "0.2"
reth-discv4.workspace = true

[features]
default = ["jemalloc"]
default = ["jemalloc", "min-debug-logs"]

dev = ["reth-cli-commands/dev"]

1 change: 1 addition & 0 deletions crates/blockchain-tree/src/blockchain_tree.rs
Original file line number Diff line number Diff line change
@@ -390,6 +390,7 @@ where
block: SealedBlockWithSenders,
block_validation_kind: BlockValidationKind,
) -> Result<BlockStatus, InsertBlockErrorKind> {
println!("BlockchainTree: try_append_canonical_chain {:?} \n tx {:?}", block.state_root, block.transactions().count());
let parent = block.parent_num_hash();
let block_num_hash = block.num_hash();
debug!(target: "blockchain_tree", head = ?block_num_hash.hash, ?parent, "Appending block to canonical chain");
13 changes: 9 additions & 4 deletions crates/blockchain-tree/src/chain.rs
Original file line number Diff line number Diff line change
@@ -18,8 +18,7 @@ use reth_primitives::{
BlockHash, BlockNumber, ForkBlock, GotExpected, SealedBlockWithSenders, SealedHeader, U256,
};
use reth_provider::{
providers::{BundleStateProvider, ConsistentDbView},
FullExecutionDataProvider, ProviderError, StateRootProvider,
providers::{BundleStateProvider, ConsistentDbView}, ChainSpecProvider, FullExecutionDataProvider, ProviderError, StateRootProvider
};
use reth_revm::database::{StateProviderDatabase, SyncStateProviderDatabase};
use reth_trie::{updates::TrieUpdates, HashedPostState};
@@ -79,6 +78,7 @@ impl AppendableChain {
DB: Database + Clone,
E: BlockExecutorProvider,
{
println!("AppendableChain::new_canonical_fork");
let execution_outcome = ExecutionOutcome::default();
let empty = BTreeMap::new();

@@ -117,6 +117,7 @@ impl AppendableChain {
DB: Database + Clone,
E: BlockExecutorProvider,
{
println!("AppendableChain::new_chain_fork");
let parent_number =
block.number.checked_sub(1).ok_or(BlockchainTreeError::GenesisBlockHasNoParent)?;
let parent = self.blocks().get(&parent_number).ok_or(
@@ -182,7 +183,8 @@ impl AppendableChain {
DB: Database + Clone,
E: BlockExecutorProvider,
{
println!("validate_and_execute");
println!("AppendableChain::validate_and_execute");
println!("sealed block: {:?}", block.header.state_root);
// some checks are done before blocks comes here.
externals.consensus.validate_header_against_parent(&block, parent_block)?;

@@ -207,7 +209,9 @@ impl AppendableChain {

let provider = BundleStateProvider::new(state_provider, bundle_state_data_provider);

let db = SyncStateProviderDatabase::new(None, StateProviderDatabase::new(&provider));
let db = SyncStateProviderDatabase::new(
Some(externals.provider_factory.chain_spec().chain.id()),
StateProviderDatabase::new(&provider));
let executor = externals.executor_factory.executor(db);
let block_hash = block.hash();
let block = block.unseal();
@@ -289,6 +293,7 @@ impl AppendableChain {
DB: Database + Clone,
E: BlockExecutorProvider,
{
println!("AppendableChain::append_block");
let parent_block = self.chain.tip();

let bundle_state_data = BundleStateDataRef {
1 change: 1 addition & 0 deletions crates/engine/tree/src/tree/mod.rs
Original file line number Diff line number Diff line change
@@ -1712,6 +1712,7 @@ where
&mut self,
block: SealedBlockWithSenders,
) -> Result<InsertPayloadOk2, InsertBlockErrorKindTwo> {
println!("insert_block_inner");
if self.block_by_hash(block.hash())?.is_some() {
return Ok(InsertPayloadOk2::AlreadySeen(BlockStatus2::Valid))
}
1 change: 1 addition & 0 deletions crates/engine/util/src/reorg.rs
Original file line number Diff line number Diff line change
@@ -269,6 +269,7 @@ where

debug!(target: "engine::stream::reorg", number = reorg_target.number, hash = %previous_hash, "Selected reorg target");

println!("reorg_target_parent: {:?}", reorg_target_parent.number);
// Configure state
let state_provider = provider.state_by_block_hash(reorg_target.parent_hash)?;
let mut state = State::builder()
7 changes: 4 additions & 3 deletions crates/ethereum/evm/src/execute.rs
Original file line number Diff line number Diff line change
@@ -148,6 +148,7 @@ where
DB: SyncDatabase,
DB::Error: Into<ProviderError> + Display,
{
print!("execute_state_transitions");
// apply pre execution changes
apply_beacon_root_contract_call(
&self.evm_config,
@@ -308,7 +309,7 @@ where
block: &BlockWithSenders,
total_difficulty: U256,
) -> Result<EthExecuteOutput, BlockExecutionError> {
println!("execute_without_verification");
println!("EthBlockExecutor::execute_without_verification");

// 1. prepare state on new block
self.on_new_block(&block.header);
@@ -318,8 +319,8 @@ where
let env = self.evm_env_for_block(&block.header, total_difficulty);
println!("evm_env_for_block");
let output = {
let evm = self.executor.evm_config.evm_with_env(&mut self.state, env);
println!("evm_config.evm_with_env");
let evm = self.executor.evm_config.evm_with_env(&mut self.state, env.clone());
println!("evm_config.evm_with_env -- env: {:?}", env);
self.executor.execute_state_transitions(block, evm)
}?;

3 changes: 2 additions & 1 deletion crates/ethereum/node/src/node.rs
Original file line number Diff line number Diff line change
@@ -54,7 +54,7 @@ impl EthereumNode {
PayloadBuilderAttributes = EthPayloadBuilderAttributes,
>,
{
println!("EthereumNode.components");
println!("EthereumNode.components -- {:?}", EthEvmConfig::default());
ComponentsBuilder::default()
.node_types::<Node>()
.pool(EthereumPoolBuilder::default())
@@ -227,6 +227,7 @@ where
ctx: &BuilderContext<Node>,
pool: Pool,
) -> eyre::Result<PayloadBuilderHandle<Node::Engine>> {
println!("EthereumPayloadBuilder.spawn_payload_service");
let payload_builder =
reth_ethereum_payload_builder::EthereumPayloadBuilder::new(self.evm_config);
let conf = ctx.payload_builder_config();
27 changes: 20 additions & 7 deletions crates/ethereum/payload/src/lib.rs
Original file line number Diff line number Diff line change
@@ -99,6 +99,7 @@ where
initialized_cfg,
..
} = config;
println!("build_empty_payload");

debug!(target: "payload_builder", parent_hash = ?parent_block.hash(), parent_number = parent_block.number, "building empty payload");

@@ -262,6 +263,7 @@ where
let block = Block { header, body: vec![], ommers: vec![], withdrawals, requests };
let sealed_block = block.seal_slow();

println!("END build_empty_payload {:?}", sealed_block.header.state_root);
Ok(EthBuiltPayload::new(attributes.payload_id(), sealed_block, U256::ZERO))
}
}
@@ -293,9 +295,13 @@ where
chain_spec,
..
} = config;
println!("default_ethereum_payload_builder \n{:?}", initialized_cfg);

let state_provider = client.state_by_block_hash(parent_block.hash())?;
let state = SyncStateProviderDatabase::new(None, StateProviderDatabase::new(state_provider));
let state = SyncStateProviderDatabase::new(
Some(chain_spec.chain.id()),
StateProviderDatabase::new(state_provider),
);
let mut sync_cached_reads = to_sync_cached_reads(cached_reads, chain_spec.chain.id());
let mut db = State::builder()
.with_database_ref(sync_cached_reads.as_db(state))
@@ -354,6 +360,12 @@ where
PayloadBuilderError::Internal(err.into())
})?;

println!("while let Some(pool_tx) = best_txs.next(): {:?}",
pool.best_transactions_with_attributes(BestTransactionsAttributes::new(
base_fee,
initialized_block_env.get_blob_gasprice().map(|gasprice| gasprice as u64),
)).count()
);
let mut receipts = Vec::new();
while let Some(pool_tx) = best_txs.next() {
// ensure we still have capacity for this transaction
@@ -362,12 +374,12 @@ where
// which also removes all dependent transaction from the iterator before we can
// continue
best_txs.mark_invalid(&pool_tx);
continue
continue;
CeciliaZ030 marked this conversation as resolved.
Show resolved Hide resolved
}

// check if the job was cancelled, if so we can exit early
if cancel.is_cancelled() {
return Ok(BuildOutcome::Cancelled)
return Ok(BuildOutcome::Cancelled);
}

// convert tx to a signed transaction
@@ -384,7 +396,7 @@ where
// for regular transactions above.
trace!(target: "payload_builder", tx=?tx.hash, ?sum_blob_gas_used, ?tx_blob_gas, "skipping blob transaction because it would exceed the max data gas per block");
best_txs.mark_invalid(&pool_tx);
continue
continue;
}
}

@@ -412,11 +424,11 @@ where
best_txs.mark_invalid(&pool_tx);
}

continue
continue;
}
err => {
// this is an error that we should treat as fatal for this attempt
return Err(PayloadBuilderError::EvmExecutionError(err))
return Err(PayloadBuilderError::EvmExecutionError(err));
}
}
}
@@ -468,7 +480,7 @@ where
return Ok(BuildOutcome::Aborted {
fees: total_fees,
cached_reads: sync_cached_reads.into(),
})
});
}

// calculate the requests and the requests root
@@ -585,6 +597,7 @@ where
let sealed_block = block.seal_slow();
debug!(target: "payload_builder", ?sealed_block, "sealed built block");

println!("END default_ethereum_payload_builder {:?}", sealed_block.header.state_root);
let mut payload = EthBuiltPayload::new(attributes.id, sealed_block, total_fees);

// extend the payload with the blob sidecars from the executed txs
1 change: 1 addition & 0 deletions crates/evm/src/lib.rs
Original file line number Diff line number Diff line change
@@ -57,6 +57,7 @@ pub trait ConfigureEvm: ConfigureEvmEnv {
db: DB,
env: EnvWithHandlerCfg,
) -> Evm<'_, Self::DefaultExternalContext<'_>, DB> {
// println!("evm_with_env {:?}", env);
let mut evm = self.evm(db);
evm.modify_spec_id(env.spec_id());
evm.context.evm.env = env.env;
4 changes: 4 additions & 0 deletions crates/evm/src/system_calls.rs
Original file line number Diff line number Diff line change
@@ -44,6 +44,7 @@ where
DB::Error: Display,
EvmConfig: ConfigureEvm,
{
println!("pre_block_blockhashes_contract_call");
// Apply the pre-block EIP-2935 contract call
let mut evm_pre_block = Evm::builder()
.with_db(db)
@@ -145,6 +146,7 @@ where
DB::Error: Display,
EvmConfig: ConfigureEvm,
{
println!("pre_block_beacon_root_contract_call");
// apply pre-block EIP-4788 contract call
let mut evm_pre_block = Evm::builder()
.with_db(db)
@@ -187,6 +189,7 @@ where
DB::Error: core::fmt::Display,
EvmConfig: ConfigureEvm,
{
println!("is_cancun_active_at_timestamp");
if !chain_spec.is_cancun_active_at_timestamp(block_timestamp) {
return Ok(())
}
@@ -216,6 +219,7 @@ where
BEACON_ROOTS_ADDRESS,
parent_beacon_block_root.0.into(),
);
println!("fill_tx_env_system_contract_call");

let mut state = match evm.transact() {
Ok(res) => res.state,
3 changes: 3 additions & 0 deletions crates/exex/exex/src/backfill/job.rs
Original file line number Diff line number Diff line change
@@ -64,6 +64,8 @@ where
}

fn execute_range(&mut self) -> Result<Chain, BlockExecutionError> {
println!("BackfillJob::execute_range");
println!("~~None");
let db = SyncStateProviderDatabase::new(
None,
StateProviderDatabase::new(
@@ -206,6 +208,7 @@ where
.ok_or_else(|| ProviderError::HeaderNotFound(block_number.into()))?;

// Configure the executor to use the previous block's state.
println!("~~None");
let executor = self.executor.executor(SyncStateProviderDatabase::new(
None,
StateProviderDatabase::new(
5 changes: 1 addition & 4 deletions crates/exex/exex/src/backfill/test_utils.rs
Original file line number Diff line number Diff line change
@@ -48,10 +48,7 @@ where
let provider = provider_factory.provider()?;
let db = SyncStateProviderDatabase::new(
Some(chain_spec.chain.id()),
StateProviderDatabase::new(LatestStateProviderRef::new(
provider.tx_ref(),
provider.static_file_provider().clone(),
))
StateProviderDatabase::new(LatestStateProviderRef::new(provider.tx_ref(),provider.static_file_provider().clone()))
);

// Execute the block to produce a block execution output
2 changes: 2 additions & 0 deletions crates/gwyneth/src/builder.rs
Original file line number Diff line number Diff line change
@@ -67,6 +67,7 @@ where
SyncProvider: StateProvider,
{
// Brecht: ethereum payload builder
println!("Cecilia: default_gwyneth_payload_builder");

let BuildArguments { client, pool, mut cached_reads, config, cancel, best_payload } = args;
let PayloadConfig {
@@ -85,6 +86,7 @@ where

let (l1_id, l1_provider) = attributes.l1_provider.unwrap();
let l1_box: Box<dyn StateProvider> = Box::new(l1_provider);
println!("Cecilia: sync_state.add_db l1_id: {:?}", l1_id);
let l1_state = StateProviderDatabase::new(l1_box);
sync_state.add_db(l1_id, l1_state);

1 change: 1 addition & 0 deletions crates/node/builder/src/launch/mod.rs
Original file line number Diff line number Diff line change
@@ -379,6 +379,7 @@ where
let (tx, rx) = oneshot::channel();
info!(target: "reth::cli", "Starting consensus engine");
ctx.task_executor().spawn_critical_blocking("consensus engine", async move {
println!("Cecilia: Starting consensus engine");
let res = beacon_consensus_engine.await;
let _ = tx.send(res);
});
Loading