Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: assertoor implementation #1324

Draft
wants to merge 26 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
a2fb27f
Initial assertoor test
rodrigo-o Oct 8, 2024
7382a6c
Added the assertoor task to the CI
rodrigo-o Oct 8, 2024
96fe806
added branch to the ethereum package url
rodrigo-o Oct 8, 2024
dbab954
removed jobs from CI and put them in their own file
rodrigo-o Oct 8, 2024
87ced43
test without our fork
rodrigo-o Oct 8, 2024
32f783e
add our eth pkg fork again
rodrigo-o Oct 8, 2024
b21ca2d
check now that branch is correctly picked up
rodrigo-o Oct 9, 2024
cce3bbd
Change file path to config
rodrigo-o Oct 9, 2024
5449526
changing config file to the root of the project
rodrigo-o Oct 9, 2024
89abeb6
Change in the ethereum-package branch
rodrigo-o Oct 9, 2024
0be4b29
Testing with the participant config
rodrigo-o Oct 9, 2024
7414d9b
Making the file more near to network_params for testing
rodrigo-o Oct 9, 2024
df8d22d
Added syncing and peers endpoint to valdiate assertoor execution
rodrigo-o Oct 9, 2024
a18768f
Push consensus file to take it form the repo
rodrigo-o Oct 11, 2024
dd3c1bb
Some fixes and logging
rodrigo-o Oct 15, 2024
0ad9cd5
Added an initial implementation of the config/spec
rodrigo-o Oct 16, 2024
147200e
cleanup and removed diffs
rodrigo-o Oct 16, 2024
e3d47ab
Merge branch 'main' into initial-assertoor-implementation
rodrigo-o Oct 16, 2024
13a5bc1
format
rodrigo-o Oct 16, 2024
85aa3f6
Merge branch 'sentry-alert-on-head-not-updated' into initial-assertoo…
rodrigo-o Oct 16, 2024
192ac92
Moved sync_status to Libp2p and made sync blocks depend on the store
rodrigo-o Oct 16, 2024
558933f
Merge branch 'main' into initial-assertoor-implementation
rodrigo-o Oct 17, 2024
f45a91b
Added some additional info to libp2p status and removed unneded alias
rodrigo-o Oct 18, 2024
292aed8
initial headers/head implementation
rodrigo-o Oct 18, 2024
e7937cc
Small change to the log_requests plug and removal of old loggers
rodrigo-o Oct 18, 2024
6919410
fixed /states/{state_id}/finality_checkpoints for genesis executions
rodrigo-o Oct 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions lib/beacon_api/controllers/error_controller.ex
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
defmodule BeaconApi.ErrorController do
require Logger
use BeaconApi, :controller

@spec bad_request(Plug.Conn.t(), binary()) :: Plug.Conn.t()
def bad_request(conn, message) do
Logger.error("Bad request: #{message}, path: #{conn.request_path}")

conn
|> put_status(400)
|> json(%{
Expand All @@ -13,6 +16,8 @@ defmodule BeaconApi.ErrorController do

@spec not_found(Plug.Conn.t(), any) :: Plug.Conn.t()
def not_found(conn, _params) do
Logger.error("Not found resource, path: #{conn.request_path}")

conn
|> put_status(404)
|> json(%{
Expand All @@ -23,6 +28,8 @@ defmodule BeaconApi.ErrorController do

@spec internal_error(Plug.Conn.t(), any) :: Plug.Conn.t()
def internal_error(conn, _params) do
Logger.error("Internal server error, path: #{conn.request_path}")

conn
|> put_status(500)
|> json(%{
Expand Down
34 changes: 33 additions & 1 deletion lib/beacon_api/controllers/v1/beacon_controller.ex
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,15 @@ defmodule BeaconApi.V1.BeaconController do
def open_api_operation(:get_finality_checkpoints),
do: ApiSpec.spec().paths["/eth/v1/beacon/states/{state_id}/finality_checkpoints"].get

def open_api_operation(:get_headers_by_block),
do: ApiSpec.spec().paths["/eth/v1/beacon/headers/{block_id}"].get

@spec get_genesis(Plug.Conn.t(), any) :: Plug.Conn.t()
def get_genesis(conn, _params) do
conn
|> json(%{
"data" => %{
"genesis_time" => StoreDb.fetch_genesis_time!(),
"genesis_time" => StoreDb.fetch_genesis_time!() |> Integer.to_string(),
"genesis_validators_root" =>
ChainSpec.get_genesis_validators_root() |> Utils.hex_encode(),
"genesis_fork_version" => ChainSpec.get("GENESIS_FORK_VERSION") |> Utils.hex_encode()
Expand Down Expand Up @@ -182,4 +185,33 @@ defmodule BeaconApi.V1.BeaconController do
}
})
end

@spec get_headers_by_block(Plug.Conn.t(), any) :: Plug.Conn.t()
def get_headers_by_block(conn, %{block_id: "head"}) do
{:ok, store} = StoreDb.fetch_store()
head_root = store.head_root
%{signed_block: %{message: message, signature: signature}} = Blocks.get_block_info(head_root)

conn
|> json(%{
execution_optimistic: false, # This is a placeholder
finalized: false, # This is obviously false for the head, but should be derived
data: %{
root: head_root |> Utils.hex_encode(),
canonical: true, # This needs to be derived
header: %{
message: %{
slot: message.slot |> Integer.to_string(),
proposer_index: message.proposer_index |> Integer.to_string(),
parent_root: message.parent_root |> Utils.hex_encode(),
state_root: message.state_root |> Utils.hex_encode(),
body_root: SszEx.hash_tree_root!(message.body) |> Utils.hex_encode()
},
signature: signature |> Utils.hex_encode()
}
}
})
end

def get_headers_by_block(conn, _params), do: conn |> ErrorController.not_found(nil)
end
61 changes: 61 additions & 0 deletions lib/beacon_api/controllers/v1/config_controller.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
defmodule BeaconApi.V1.ConfigController do
use BeaconApi, :controller
require Logger

alias BeaconApi.ApiSpec
alias BeaconApi.Utils

plug(OpenApiSpex.Plug.CastAndValidate, json_render_error_v2: true)

@chain_spec_removed_keys [
"ATTESTATION_SUBNET_COUNT",
"KZG_COMMITMENT_INCLUSION_PROOF_DEPTH",
"UPDATE_TIMEOUT"
]
@chain_spec_renamed_keys [
{"MAXIMUM_GOSSIP_CLOCK_DISPARITY", "MAXIMUM_GOSSIP_CLOCK_DISPARITY_MILLIS"}
]
@chain_spec_hex_fields [
"TERMINAL_BLOCK_HASH",
"GENESIS_FORK_VERSION",
"ALTAIR_FORK_VERSION",
"BELLATRIX_FORK_VERSION",
"CAPELLA_FORK_VERSION",
"DENEB_FORK_VERSION",
"ELECTRA_FORK_VERSION",
"DEPOSIT_CONTRACT_ADDRESS",
"MESSAGE_DOMAIN_INVALID_SNAPPY",
"MESSAGE_DOMAIN_VALID_SNAPPY"
]

# NOTE: this function is required by OpenApiSpex, and should return the information
# of each specific endpoint. We just return the specific entry from the parsed spec.
def open_api_operation(:get_spec),
do: ApiSpec.spec().paths["/eth/v1/config/spec"].get

# TODO: This is still an incomplete implementation, it should return some constants
# along with the chain spec. It's enough for assertoor.
@spec get_spec(Plug.Conn.t(), any) :: Plug.Conn.t()
def get_spec(conn, _params), do: json(conn, %{"data" => chain_spec()})

defp chain_spec() do
ChainSpec.get_all()
|> Map.drop(@chain_spec_removed_keys)
|> rename_keys(@chain_spec_renamed_keys)
|> Map.new(fn
{k, v} when is_integer(v) -> {k, Integer.to_string(v)}
{k, v} when k in @chain_spec_hex_fields -> {k, Utils.hex_encode(v)}
{k, v} -> {k, v}
end)
end

defp rename_keys(config, renamed_keys) do
renamed_keys
|> Enum.reduce(config, fn {old_key, new_key}, config ->
case Map.get(config, old_key) do
nil -> config
value -> Map.put_new(config, new_key, value) |> Map.delete(old_key)
end
end)
end
end
36 changes: 33 additions & 3 deletions lib/beacon_api/controllers/v1/node_controller.ex
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,21 @@ defmodule BeaconApi.V1.NodeController do
def open_api_operation(:version),
do: ApiSpec.spec().paths["/eth/v1/node/version"].get

def open_api_operation(:syncing),
do: ApiSpec.spec().paths["/eth/v1/node/syncing"].get

def open_api_operation(:peers),
do: ApiSpec.spec().paths["/eth/v1/node/peers"].get

@spec health(Plug.Conn.t(), any) :: Plug.Conn.t()
def health(conn, params) do
# TODO: respond with syncing status if we're still syncing
_syncing_status = Map.get(params, :syncing_status, 206)
%{syncing?: syncing?} = Libp2pPort.sync_status()

syncing_status = if syncing?, do: Map.get(params, :syncing_status, 206), else: 200

send_resp(conn, 200, "")
send_resp(conn, syncing_status, "")
rescue
_ -> send_resp(conn, 503, "")
end

@spec identity(Plug.Conn.t(), any) :: Plug.Conn.t()
Expand Down Expand Up @@ -62,4 +71,25 @@ defmodule BeaconApi.V1.NodeController do
}
})
end

@spec syncing(Plug.Conn.t(), any) :: Plug.Conn.t()
def syncing(conn, _params) do
%{
syncing?: is_syncing,
optimistic?: is_optimistic,
el_offline?: el_offline,
head_slot: head_slot,
sync_distance: sync_distance
} = Libp2pPort.sync_status()

json(conn, %{
"data" => %{
"is_syncing" => is_syncing,
"is_optimistic" => is_optimistic,
"el_offline" => el_offline,
"head_slot" => head_slot |> Integer.to_string(),
"sync_distance" => sync_distance |> Integer.to_string()
}
})
end
end
10 changes: 7 additions & 3 deletions lib/beacon_api/helpers.ex
Original file line number Diff line number Diff line change
Expand Up @@ -154,10 +154,14 @@ defmodule BeaconApi.Helpers do
@spec finality_checkpoint_by_id(state_id()) ::
{:ok, finality_info()} | {:error, String.t()} | :not_found | :empty_slot | :invalid_id
def finality_checkpoint_by_id(id) do
empty_checkpoint = %Types.Checkpoint{epoch: 0, root: <<0::256>>}

with {:ok, {state, optimistic, finalized}} <- state_by_state_id(id) do
{:ok,
{state.previous_justified_checkpoint, state.current_justified_checkpoint,
state.finalized_checkpoint, optimistic, finalized}}
previous_justified_ck = Map.get(state, :previous_justified_checkpoint, empty_checkpoint)
current_justified_ck = Map.get(state, :current_justified_checkpoint, empty_checkpoint)
finalized_ck = Map.get(state, :finalized_checkpoint, empty_checkpoint)

{:ok, {previous_justified_ck, current_justified_ck, finalized_ck, optimistic, finalized}}
end
end

Expand Down
21 changes: 21 additions & 0 deletions lib/beacon_api/router.ex
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
defmodule BeaconApi.Router do
use BeaconApi, :router
require Logger

pipeline :api do
plug(:accepts, ["json"])
plug(OpenApiSpex.Plug.PutApiSpec, module: BeaconApi.ApiSpec)
plug :log_requests
end

# Ethereum API Version 1
Expand All @@ -15,12 +17,19 @@ defmodule BeaconApi.Router do
get("/states/:state_id/root", BeaconController, :get_state_root)
get("/blocks/:block_id/root", BeaconController, :get_block_root)
get("/states/:state_id/finality_checkpoints", BeaconController, :get_finality_checkpoints)
get("/headers/:block_id", BeaconController, :get_headers_by_block)
end

scope "/config" do
get("/spec", ConfigController, :get_spec)
end

scope "/node" do
get("/health", NodeController, :health)
get("/identity", NodeController, :identity)
get("/version", NodeController, :version)
get("/syncing", NodeController, :syncing)
get("/peers", NodeController, :peers)
end
end

Expand All @@ -40,4 +49,16 @@ defmodule BeaconApi.Router do

# Catch-all route outside of any scope
match(:*, "/*path", BeaconApi.ErrorController, :not_found)

defp log_requests(conn, _opts) do
base_message = "[BeaconAPI Router] Processing request: #{conn.method} - #{conn.request_path}"
query = if conn.query_params != %{}, do: "Query: #{inspect(conn.query_params)}", else: ""
body = if conn.body_params != %{}, do: "Body: #{inspect(conn.body_params)}", else: ""

[base_message, query, body]
|> Enum.join("\n\t")
|> Logger.info()

conn
end
end
2 changes: 2 additions & 0 deletions lib/chain_spec/chain_spec.ex
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ defmodule ChainSpec do
# NOTE: this only works correctly for Capella
def get(name), do: get_config().get(name)

def get_all(), do: get_config().get_all()

def get_genesis_validators_root() do
Application.fetch_env!(:lambda_ethereum_consensus, __MODULE__)
|> Keyword.fetch!(:genesis_validators_root)
Expand Down
7 changes: 3 additions & 4 deletions lib/lambda_ethereum_consensus/beacon/sync_blocks.ex
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,10 @@ defmodule LambdaEthereumConsensus.Beacon.SyncBlocks do
finish, each block of those responses will be sent to libp2p port module individually using
Libp2pPort.add_block/1.
"""
@spec run() :: non_neg_integer()
def run() do
%{head_slot: head_slot} = ForkChoice.get_current_status_message()
@spec run(Types.Store.t()) :: non_neg_integer()
def run(%{head_slot: head_slot} = store) do
initial_slot = head_slot + 1
last_slot = ForkChoice.get_current_chain_slot()
last_slot = ForkChoice.get_current_slot(store)

# If we're around genesis, we consider ourselves synced
if last_slot <= 0 do
Expand Down
3 changes: 1 addition & 2 deletions lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,7 @@ defmodule LambdaEthereumConsensus.ForkChoice do
@doc """
Get the current chain slot based on the system time.

There are just 2 uses of this function outside this module:
- At the begining of SyncBlocks.run/1 function, to get the head slot
There are just 1 use of this function outside this module:
- In the Helpers.block_root_by_block_id/1 function
"""
@spec get_current_chain_slot() :: Types.slot()
Expand Down
39 changes: 37 additions & 2 deletions lib/libp2p_port.ex
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,21 @@ defmodule LambdaEthereumConsensus.Libp2pPort do
GenServer.cast(pid, {:error_downloading_chunk, range, reason})
end

@doc """
Returns the current sync status.
"""
@spec sync_status(pid | atom()) :: %{
syncing?: boolean(),
optimistic?: boolean(),
el_offline?: boolean(),
head_slot: Types.slot(),
sync_distance: non_neg_integer(),
blocks_remaining: non_neg_integer()
}
def sync_status(pid \\ __MODULE__) do
GenServer.call(pid, :sync_status)
end

########################
### GenServer Callbacks
########################
Expand Down Expand Up @@ -513,8 +528,8 @@ defmodule LambdaEthereumConsensus.Libp2pPort do
end

@impl GenServer
def handle_info(:sync_blocks, state) do
blocks_to_download = SyncBlocks.run()
def handle_info(:sync_blocks, %{store: store} = state) do
blocks_to_download = SyncBlocks.run(store)

new_state =
state |> Map.put(:blocks_remaining, blocks_to_download) |> subscribe_if_no_blocks()
Expand Down Expand Up @@ -575,6 +590,26 @@ defmodule LambdaEthereumConsensus.Libp2pPort do
{:reply, :ok, %{state | validator_set: validator_set}}
end

@impl GenServer
def handle_call(:sync_status, _from, %{syncing: syncing?, store: %Types.Store{} = store} = state) do
# TODO: (#1325) This is not the final implementation, we are lacking the el check,
# this is just in place for start using assertoor.
head_slot = store.head_slot
current_slot = ForkChoice.get_current_slot(store)
distance = current_slot - head_slot

result = %{
syncing?: syncing?,
optimistic?: syncing?,
el_offline?: false,
head_slot: store.head_slot,
sync_distance: distance,
blocks_remaining: Map.get(state, :blocks_remaining)
}

{:reply, result, state}
end

######################
### PRIVATE FUNCTIONS
######################
Expand Down
17 changes: 15 additions & 2 deletions network_params.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,18 @@ participants:
validator_count: 32
cl_max_mem: 4096
keymanager_enabled: true
network_params:
preset: minimal
# network_params:
# preset: minimal
additional_services:
- assertoor
- tx_spammer
- blob_spammer
- el_forkmon
- dora
- beacon_metrics_gazer
- prometheus_grafana
assertoor_params:
run_stability_check: false
run_block_proposal_check: false
tests:
- file: "./consensus-test.yaml"
Loading
Loading