Merge branch 'main' into fix-eternal-process-blocks

config/runtime.exs

@@ -1,4 +1,8 @@
 import Config
+alias LambdaEthereumConsensus.Beacon.StoreSetup
+alias LambdaEthereumConsensus.ForkChoice
+alias LambdaEthereumConsensus.SszEx
+alias Types.BeaconStateDeneb
 
 switches = [
   network: :string,
@@ -41,8 +45,10 @@ validator_file = Keyword.get(args, :validator_file)
 enable_beacon_api = Keyword.get(args, :beacon_api, false)
 beacon_api_port = Keyword.get(args, :beacon_api_port, 4000)
 
-config :lambda_ethereum_consensus, LambdaEthereumConsensus.ForkChoice,
-  checkpoint_sync_url: checkpoint_sync_url
+if not is_nil(testnet_dir) and not is_nil(checkpoint_sync_url) do
+  IO.puts("Both checkpoint sync and testnet url specified (only one should be specified).")
+  System.halt(2)
+end
 
 valid_modes = ["full", "db"]
 raw_mode = Keyword.get(args, :mode, "full")
@@ -57,43 +63,49 @@ mode =
 
 config :lambda_ethereum_consensus, LambdaEthereumConsensus, mode: mode
 
-datadir = Keyword.get(args, :datadir, "level_db/#{network}")
+# DB setup
+default_datadir =
+  case testnet_dir do
+    nil -> "level_db/#{network}"
+    _ -> "level_db/local_testnet"
+  end
+
+datadir = Keyword.get(args, :datadir, default_datadir)
 File.mkdir_p!(datadir)
 config :lambda_ethereum_consensus, LambdaEthereumConsensus.Store.Db, dir: datadir
 
-chain_config =
+# Network setup
+{chain_config, bootnodes} =
   case testnet_dir do
     nil ->
       config = ConfigUtils.parse_config(network)
       bootnodes = YamlElixir.read_from_file!("config/networks/#{network}/boot_enr.yaml")
-
-      %{
-        config: config,
-        genesis_validators_root: config.genesis_validators_root(),
-        bootnodes: bootnodes
-      }
+      {config, bootnodes}
 
     testnet_dir ->
       Path.join(testnet_dir, "config.yaml") |> CustomConfig.load_from_file!()
+      bootnodes = Path.join(testnet_dir, "boot_enr.yaml") |> YamlElixir.read_from_file!()
+      {CustomConfig, bootnodes}
+  end
 
-      # TODO: compute this from the genesis block
-      genesis_validators_root = <<0::256>>
+# We use put_env here as we need this immediately after to read the state.
+Application.put_env(:lambda_ethereum_consensus, ChainSpec, config: chain_config)
 
-      bootnodes = Path.join(testnet_dir, "boot_enr.yaml") |> YamlElixir.read_from_file!()
+strategy = StoreSetup.make_strategy!(testnet_dir, checkpoint_sync_url)
 
-      %{
-        config: CustomConfig,
-        genesis_validators_root: genesis_validators_root,
-        bootnodes: bootnodes
-      }
+genesis_validators_root =
+  case strategy do
+    {:file, state} -> state.genesis_validators_root
+    _ -> chain_config.genesis_validators_root()
   end
 
 config :lambda_ethereum_consensus, ChainSpec,
-  config: Map.fetch!(chain_config, :config),
-  genesis_validators_root: Map.fetch!(chain_config, :genesis_validators_root)
+  config: chain_config,
+  genesis_validators_root: genesis_validators_root
+
+config :lambda_ethereum_consensus, StoreSetup, strategy: strategy
 
 # Configures peer discovery
-bootnodes = Map.fetch!(chain_config, :bootnodes)
 config :lambda_ethereum_consensus, :discovery, port: 9000, bootnodes: bootnodes
 
 # Engine API

lib/chain_spec/configs/custom.ex

@@ -8,13 +8,33 @@ defmodule CustomConfig do
   def load_from_file!(path) do
     config = ConfigUtils.load_config_from_file!(path)
     preset = Map.fetch!(config, "PRESET_BASE") |> ConfigUtils.parse_preset()
-    merged_config = Map.merge(preset.get_preset(), config)
+    base_config = Map.fetch!(config, "CONFIG_NAME") |> ConfigUtils.parse_config()
+
+    merged_config =
+      preset.get_preset()
+      |> Map.merge(base_config.get_all())
+      |> Map.merge(config)
+
     Application.put_env(:lambda_ethereum_consensus, __MODULE__, merged: merged_config)
   end
 
-  defp get_config,
-    do: Application.get_env(:lambda_ethereum_consensus, __MODULE__) |> Keyword.fetch!(:merged)
+  @impl GenConfig
+  def get_all do
+    Application.get_env(:lambda_ethereum_consensus, __MODULE__)
+    |> Keyword.fetch!(:merged)
+    |> Map.new(fn {k, v} -> {k, parse_int(v)} end)
+  end
 
   @impl GenConfig
-  def get(key), do: get_config() |> Map.fetch!(key)
+  def get(key), do: get_all() |> Map.fetch!(key)
+
+  # Parses as integer if parsable. If not, returns original value.
+  defp parse_int(v) when is_binary(v) do
+    case Integer.parse(v) do
+      {i, ""} -> i
+      _ -> v
+    end
+  end
+
+  defp parse_int(v), do: v
 end

lib/chain_spec/configs/gen_config.ex

@@ -21,11 +21,19 @@ defmodule ChainSpec.GenConfig do
 
       @impl unquote(__MODULE__)
       def get(key), do: Map.fetch!(@__unified, key)
+
+      @impl unquote(__MODULE__)
+      def get_all, do: @__unified
     end
   end
 
   @doc """
   Fetches a value from config.
   """
   @callback get(String.t()) :: term()
+
+  @doc """
+  Fetches the full config dictionary.
+  """
+  @callback get_all() :: map()
 end

lib/chain_spec/utils.ex

@@ -7,7 +7,7 @@ defmodule ConfigUtils do
   def load_config_from_file!(path) do
     path
     |> File.read!()
-    |> String.replace(~r/(0x[0-9a-fA-F]+)/, "'\\g{1}'")
+    |> String.replace(~r/ (0x[0-9a-fA-F]+)/, " '\\g{1}'")
     |> YamlElixir.read_from_string!()
     |> Stream.map(fn
       {k, "0x" <> hash} -> {k, Base.decode16!(hash, case: :mixed)}

lib/lambda_ethereum_consensus/application.ex

@@ -44,17 +44,12 @@ defmodule LambdaEthereumConsensus.Application do
   defp get_children(:full) do
     get_children(:db) ++
       [
-        {LambdaEthereumConsensus.Beacon.BeaconNode, [checkpoint_sync_url()]},
+        LambdaEthereumConsensus.Beacon.BeaconNode,
         LambdaEthereumConsensus.P2P.Metadata,
         BeaconApi.Endpoint
       ]
   end
 
-  def checkpoint_sync_url do
-    Application.fetch_env!(:lambda_ethereum_consensus, LambdaEthereumConsensus.ForkChoice)
-    |> Keyword.fetch!(:checkpoint_sync_url)
-  end
-
   defp get_operation_mode do
     Application.fetch_env!(:lambda_ethereum_consensus, LambdaEthereumConsensus)
     |> Keyword.fetch!(:mode)

lib/lambda_ethereum_consensus/beacon/beacon_node.ex

@@ -4,50 +4,23 @@ defmodule LambdaEthereumConsensus.Beacon.BeaconNode do
   use Supervisor
   require Logger
 
-  alias LambdaEthereumConsensus.Beacon.CheckpointSync
+  alias LambdaEthereumConsensus.Beacon.StoreSetup
   alias LambdaEthereumConsensus.ForkChoice.Helpers
   alias LambdaEthereumConsensus.StateTransition.Cache
-  alias LambdaEthereumConsensus.StateTransition.Misc
   alias LambdaEthereumConsensus.Store.Blocks
   alias LambdaEthereumConsensus.Store.BlockStates
-  alias LambdaEthereumConsensus.Store.StoreDb
   alias LambdaEthereumConsensus.Validator
   alias Types.BeaconState
-  alias Types.Store
-
-  @max_epochs_before_stale 8
 
   def start_link(opts) do
     Supervisor.start_link(__MODULE__, opts, name: __MODULE__)
   end
 
   @impl true
-  def init([nil]) do
-    case restore_state_from_db() do
-      nil ->
-        Logger.error(
-          "[Sync] No recent state found. Please specify the URL to fetch them from via the --checkpoint-sync-url flag"
-        )
-
-        System.halt(1)
-
-      {_, {store, root}} ->
-        init_children(store, root)
-    end
-  end
-
-  def init([checkpoint_url]) do
-    case restore_state_from_db() do
-      {:ok, {store, root}} ->
-        Logger.warning("[Checkpoint sync] Recent state found. Ignoring the checkpoint URL.")
-        init_children(store, root)
+  def init(_) do
+    {store, genesis_validators_root} = StoreSetup.setup!()
+    deposit_tree_snapshot = StoreSetup.get_deposit_snapshot!()
 
-      _ ->
-        fetch_state_from_url(checkpoint_url)
-    end
-  end
-
-  defp init_children(%Store{} = store, genesis_validators_root, deposit_tree_snapshot \\ nil) do
     Cache.initialize_cache()
 
     config = Application.fetch_env!(:lambda_ethereum_consensus, :discovery)
@@ -93,72 +66,6 @@ defmodule LambdaEthereumConsensus.Beacon.BeaconNode do
     Supervisor.init(children, strategy: :one_for_all)
   end
 
-  defp restore_state_from_db do
-    # Try to fetch the old store from the database
-    case StoreDb.fetch_store() do
-      {:ok, %Store{finalized_checkpoint: %{epoch: finalized_epoch}} = store} ->
-        res = {store, ChainSpec.get_genesis_validators_root()}
-
-        if get_current_epoch(store) - finalized_epoch > @max_epochs_before_stale do
-          Logger.info("[Sync] Found old state in DB.")
-          {:old_state, res}
-        else
-          Logger.info("[Sync] Found recent state in DB.")
-          {:ok, res}
-        end
-
-      :not_found ->
-        nil
-    end
-  end
-
-  defp fetch_state_from_url(url) do
-    Logger.info("[Checkpoint sync] Initiating checkpoint sync")
-
-    genesis_validators_root = ChainSpec.get_genesis_validators_root()
-
-    case CheckpointSync.get_finalized_block_and_state(url, genesis_validators_root) do
-      {:ok, {anchor_state, anchor_block}} ->
-        Logger.info(
-          "[Checkpoint sync] Received beacon state and block",
-          slot: anchor_state.slot
-        )
-
-        # We already checked block and state match
-        {:ok, store} = Store.get_forkchoice_store(anchor_state, anchor_block)
-
-        # TODO: integrate into CheckpointSync, and validate snapshot
-        snapshot = fetch_deposit_snapshot(url)
-
-        # Save store in DB
-        StoreDb.persist_store(store)
-
-        init_children(store, genesis_validators_root, snapshot)
-
-      _ ->
-        Logger.error("[Checkpoint sync] Failed to fetch the latest finalized state and block")
-
-        System.halt(1)
-    end
-  end
-
-  defp get_current_epoch(store) do
-    (:os.system_time(:second) - store.genesis_time)
-    |> div(ChainSpec.get("SECONDS_PER_SLOT"))
-    |> Misc.compute_epoch_at_slot()
-  end
-
-  defp fetch_deposit_snapshot(url) do
-    case CheckpointSync.get_deposit_snapshot(url) do
-      {:ok, snapshot} ->
-        snapshot
-
-      _ ->
-        Logger.error("[Checkpoint sync] Failed to fetch the deposit snapshot")
-        System.halt(1)
-    end
-  end
-
   defp get_validator_children(nil, _, _, _) do
     Logger.warning("[Checkpoint sync] To enable validator features, checkpoint-sync is required.")