Builder Specs v0.2.0 (#3134)

## Issue Addressed

https://github.com/sigp/lighthouse/issues/3091

Extends https://github.com/sigp/lighthouse/pull/3062, adding pre-bellatrix block support on blinded endpoints and allowing the normal proposal flow (local payload construction) on blinded endpoints. This resulted in better fallback logic because the VC will not have to switch endpoints on failure in the BN <> Builder API, the BN can just fallback immediately and without repeating block processing that it shouldn't need to. We can also keep VC fallback from the VC<>BN API's blinded endpoint to full endpoint.

## Proposed Changes

- Pre-bellatrix blocks on blinded endpoints
- Add a new `PayloadCache` to the execution layer
- Better fallback-from-builder logic

## Todos

- [x] Remove VC transition logic
- [x] Add logic to only enable builder flow after Merge transition finalization
- [x] Tests
- [x] Fix metrics
- [x] Rustdocs


Co-authored-by: Mac L <mjladson@pm.me>
Co-authored-by: realbigsean <sean@sigmaprime.io>

beacon_node/beacon_chain/Cargo.toml

@@ -62,6 +62,7 @@ sensitive_url = { path = "../../common/sensitive_url" }
 superstruct = "0.5.0"
 hex = "0.4.2"
 exit-future = "0.2.0"
+unused_port = {path = "../../common/unused_port"}
 
 [[test]]
 name = "beacon_chain_tests"

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -53,7 +53,9 @@ use crate::BeaconForkChoiceStore;
 use crate::BeaconSnapshot;
 use crate::{metrics, BeaconChainError};
 use eth2::types::{EventKind, SseBlock, SyncDuty};
-use execution_layer::{ExecutionLayer, PayloadAttributes, PayloadStatus};
+use execution_layer::{
+    BuilderParams, ChainHealth, ExecutionLayer, FailedCondition, PayloadAttributes, PayloadStatus,
+};
 use fork_choice::{
     AttestationFromBlock, ExecutionStatus, ForkChoice, ForkchoiceUpdateParameters,
     InvalidationOperation, PayloadVerificationStatus,
@@ -3315,10 +3317,21 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
         let proposer_index = state.get_beacon_proposer_index(state.slot(), &self.spec)? as u64;
 
-        let pubkey_opt = state
+        let pubkey = state
             .validators()
             .get(proposer_index as usize)
-            .map(|v| v.pubkey);
+            .map(|v| v.pubkey)
+            .ok_or(BlockProductionError::BeaconChain(
+                BeaconChainError::ValidatorIndexUnknown(proposer_index as usize),
+            ))?;
+
+        let builder_params = BuilderParams {
+            pubkey,
+            slot: state.slot(),
+            chain_health: self
+                .is_healthy()
+                .map_err(BlockProductionError::BeaconChain)?,
+        };
 
         // If required, start the process of loading an execution payload from the EL early. This
         // allows it to run concurrently with things like attestation packing.
@@ -3326,7 +3339,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             BeaconState::Base(_) | BeaconState::Altair(_) => None,
             BeaconState::Merge(_) => {
                 let prepare_payload_handle =
-                    get_execution_payload(self.clone(), &state, proposer_index, pubkey_opt)?;
+                    get_execution_payload(self.clone(), &state, proposer_index, builder_params)?;
                 Some(prepare_payload_handle)
             }
         };
@@ -4539,6 +4552,74 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             .map(|duration| (fork_name, duration))
     }
 
+    /// This method serves to get a sense of the current chain health. It is used in block proposal
+    /// to determine whether we should outsource payload production duties.
+    ///
+    /// Since we are likely calling this during the slot we are going to propose in, don't take into
+    /// account the current slot when accounting for skips.
+    pub fn is_healthy(&self) -> Result<ChainHealth, Error> {
+        // Check if the merge has been finalized.
+        if let Some(finalized_hash) = self
+            .canonical_head
+            .cached_head()
+            .forkchoice_update_parameters()
+            .finalized_hash
+        {
+            if ExecutionBlockHash::zero() == finalized_hash {
+                return Ok(ChainHealth::PreMerge);
+            }
+        } else {
+            return Ok(ChainHealth::PreMerge);
+        };
+
+        if self.config.builder_fallback_disable_checks {
+            return Ok(ChainHealth::Healthy);
+        }
+
+        let current_slot = self.slot()?;
+
+        // Check slots at the head of the chain.
+        let prev_slot = current_slot.saturating_sub(Slot::new(1));
+        let head_skips = prev_slot.saturating_sub(self.canonical_head.cached_head().head_slot());
+        let head_skips_check = head_skips.as_usize() <= self.config.builder_fallback_skips;
+
+        // Check if finalization is advancing.
+        let current_epoch = current_slot.epoch(T::EthSpec::slots_per_epoch());
+        let epochs_since_finalization = current_epoch.saturating_sub(
+            self.canonical_head
+                .cached_head()
+                .finalized_checkpoint()
+                .epoch,
+        );
+        let finalization_check = epochs_since_finalization.as_usize()
+            <= self.config.builder_fallback_epochs_since_finalization;
+
+        // Check skip slots in the last `SLOTS_PER_EPOCH`.
+        let start_slot = current_slot.saturating_sub(T::EthSpec::slots_per_epoch());
+        let mut epoch_skips = 0;
+        for slot in start_slot.as_u64()..current_slot.as_u64() {
+            if self
+                .block_root_at_slot_skips_none(Slot::new(slot))?
+                .is_none()
+            {
+                epoch_skips += 1;
+            }
+        }
+        let epoch_skips_check = epoch_skips <= self.config.builder_fallback_skips_per_epoch;
+
+        if !head_skips_check {
+            Ok(ChainHealth::Unhealthy(FailedCondition::Skips))
+        } else if !finalization_check {
+            Ok(ChainHealth::Unhealthy(
+                FailedCondition::EpochsSinceFinalization,
+            ))
+        } else if !epoch_skips_check {
+            Ok(ChainHealth::Unhealthy(FailedCondition::SkipsPerEpoch))
+        } else {
+            Ok(ChainHealth::Healthy)
+        }
+    }
+
     pub fn dump_as_dot<W: Write>(&self, output: &mut W) {
         let canonical_head_hash = self.canonical_head.cached_head().head_block_root();
         let mut visited: HashSet<Hash256> = HashSet::new();

beacon_node/beacon_chain/src/chain_config.rs

@@ -24,6 +24,16 @@ pub struct ChainConfig {
     ///
     /// If set to 0 then block proposal will not wait for fork choice at all.
     pub fork_choice_before_proposal_timeout_ms: u64,
+    /// Number of skip slots in a row before the BN refuses to use connected builders during payload construction.
+    pub builder_fallback_skips: usize,
+    /// Number of skip slots in the past `SLOTS_PER_EPOCH` before the BN refuses to use connected
+    /// builders during payload construction.
+    pub builder_fallback_skips_per_epoch: usize,
+    /// Number of epochs since finalization before the BN refuses to use connected builders during
+    /// payload construction.
+    pub builder_fallback_epochs_since_finalization: usize,
+    /// Whether any chain health checks should be considered when deciding whether to use the builder API.
+    pub builder_fallback_disable_checks: bool,
     pub count_unrealized: bool,
 }
 
@@ -36,6 +46,11 @@ impl Default for ChainConfig {
             enable_lock_timeouts: true,
             max_network_size: 10 * 1_048_576, // 10M
             fork_choice_before_proposal_timeout_ms: DEFAULT_FORK_CHOICE_BEFORE_PROPOSAL_TIMEOUT,
+            // Builder fallback configs that are set in `clap` will override these.
+            builder_fallback_skips: 3,
+            builder_fallback_skips_per_epoch: 8,
+            builder_fallback_epochs_since_finalization: 3,
+            builder_fallback_disable_checks: false,
             count_unrealized: false,
         }
     }

beacon_node/beacon_chain/src/errors.rs

@@ -138,6 +138,7 @@ pub enum BeaconChainError {
         new_slot: Slot,
     },
     AltairForkDisabled,
+    BuilderMissing,
     ExecutionLayerMissing,
     BlockVariantLacksExecutionPayload(Hash256),
     ExecutionLayerErrorPayloadReconstruction(ExecutionBlockHash, execution_layer::Error),

beacon_node/beacon_chain/src/execution_payload.rs

@@ -11,7 +11,7 @@ use crate::{
     BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, BlockProductionError,
     ExecutionPayloadError,
 };
-use execution_layer::PayloadStatus;
+use execution_layer::{BuilderParams, PayloadStatus};
 use fork_choice::{InvalidationOperation, PayloadVerificationStatus};
 use proto_array::{Block as ProtoBlock, ExecutionStatus};
 use slog::debug;
@@ -303,12 +303,11 @@ pub fn get_execution_payload<
     chain: Arc<BeaconChain<T>>,
     state: &BeaconState<T::EthSpec>,
     proposer_index: u64,
-    pubkey: Option<PublicKeyBytes>,
+    builder_params: BuilderParams,
 ) -> Result<PreparePayloadHandle<Payload>, BlockProductionError> {
     // Compute all required values from the `state` now to avoid needing to pass it into a spawned
     // task.
     let spec = &chain.spec;
-    let slot = state.slot();
     let current_epoch = state.current_epoch();
     let is_merge_transition_complete = is_merge_transition_complete(state);
     let timestamp = compute_timestamp_at_slot(state, spec).map_err(BeaconStateError::from)?;
@@ -325,13 +324,12 @@ pub fn get_execution_payload<
             async move {
                 prepare_execution_payload::<T, Payload>(
                     &chain,
-                    slot,
                     is_merge_transition_complete,
                     timestamp,
                     random,
                     proposer_index,
-                    pubkey,
                     latest_execution_payload_header_block_hash,
+                    builder_params,
                 )
                 .await
             },
@@ -359,19 +357,18 @@ pub fn get_execution_payload<
 #[allow(clippy::too_many_arguments)]
 pub async fn prepare_execution_payload<T, Payload>(
     chain: &Arc<BeaconChain<T>>,
-    slot: Slot,
     is_merge_transition_complete: bool,
     timestamp: u64,
     random: Hash256,
     proposer_index: u64,
-    pubkey: Option<PublicKeyBytes>,
     latest_execution_payload_header_block_hash: ExecutionBlockHash,
+    builder_params: BuilderParams,
 ) -> Result<Payload, BlockProductionError>
 where
     T: BeaconChainTypes,
     Payload: ExecPayload<T::EthSpec> + Default,
 {
-    let current_epoch = slot.epoch(T::EthSpec::slots_per_epoch());
+    let current_epoch = builder_params.slot.epoch(T::EthSpec::slots_per_epoch());
     let spec = &chain.spec;
     let execution_layer = chain
         .execution_layer
@@ -432,9 +429,9 @@ where
             timestamp,
             random,
             proposer_index,
-            pubkey,
-            slot,
             forkchoice_update_params,
+            builder_params,
+            &chain.spec,
         )
         .await
         .map_err(BlockProductionError::GetPayloadFailed)?;

beacon_node/beacon_chain/src/lib.rs

@@ -15,7 +15,7 @@ mod early_attester_cache;
 mod errors;
 pub mod eth1_chain;
 pub mod events;
-mod execution_payload;
+pub mod execution_payload;
 pub mod fork_choice_signal;
 pub mod fork_revert;
 mod head_tracker;

beacon_node/beacon_chain/src/test_utils.rs

@@ -14,7 +14,9 @@ use bls::get_withdrawal_credentials;
 use execution_layer::test_utils::DEFAULT_JWT_SECRET;
 use execution_layer::{
     auth::JwtKey,
-    test_utils::{ExecutionBlockGenerator, MockExecutionLayer, DEFAULT_TERMINAL_BLOCK},
+    test_utils::{
+        ExecutionBlockGenerator, MockExecutionLayer, TestingBuilder, DEFAULT_TERMINAL_BLOCK,
+    },
     ExecutionLayer,
 };
 use fork_choice::CountUnrealized;
@@ -154,6 +156,7 @@ pub struct Builder<T: BeaconChainTypes> {
     store_mutator: Option<BoxedMutator<T::EthSpec, T::HotStore, T::ColdStore>>,
     execution_layer: Option<ExecutionLayer<T::EthSpec>>,
     mock_execution_layer: Option<MockExecutionLayer<T::EthSpec>>,
+    mock_builder: Option<TestingBuilder<T::EthSpec>>,
     runtime: TestRuntime,
     log: Logger,
 }
@@ -285,6 +288,7 @@ where
             store_mutator: None,
             execution_layer: None,
             mock_execution_layer: None,
+            mock_builder: None,
             runtime,
             log,
         }
@@ -388,6 +392,38 @@ where
         self
     }
 
+    pub fn mock_execution_layer_with_builder(mut self, beacon_url: SensitiveUrl) -> Self {
+        // Get a random unused port
+        let port = unused_port::unused_tcp_port().unwrap();
+        let builder_url = SensitiveUrl::parse(format!("http://127.0.0.1:{port}").as_str()).unwrap();
+
+        let spec = self.spec.clone().expect("cannot build without spec");
+        let mock_el = MockExecutionLayer::new(
+            self.runtime.task_executor.clone(),
+            spec.terminal_total_difficulty,
+            DEFAULT_TERMINAL_BLOCK,
+            spec.terminal_block_hash,
+            spec.terminal_block_hash_activation_epoch,
+            Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()),
+            Some(builder_url.clone()),
+        )
+        .move_to_terminal_block();
+
+        let mock_el_url = SensitiveUrl::parse(mock_el.server.url().as_str()).unwrap();
+
+        self.mock_builder = Some(TestingBuilder::new(
+            mock_el_url,
+            builder_url,
+            beacon_url,
+            spec,
+            self.runtime.task_executor.clone(),
+        ));
+        self.execution_layer = Some(mock_el.el.clone());
+        self.mock_execution_layer = Some(mock_el);
+
+        self
+    }
+
     /// Instruct the mock execution engine to always return a "valid" response to any payload it is
     /// asked to execute.
     pub fn mock_execution_layer_all_payloads_valid(self) -> Self {
@@ -456,6 +492,7 @@ where
             shutdown_receiver: Arc::new(Mutex::new(shutdown_receiver)),
             runtime: self.runtime,
             mock_execution_layer: self.mock_execution_layer,
+            mock_builder: self.mock_builder.map(Arc::new),
             rng: make_rng(),
         }
     }
@@ -474,6 +511,7 @@ pub struct BeaconChainHarness<T: BeaconChainTypes> {
     pub runtime: TestRuntime,
 
     pub mock_execution_layer: Option<MockExecutionLayer<T::EthSpec>>,
+    pub mock_builder: Option<Arc<TestingBuilder<T::EthSpec>>>,
 
     pub rng: Mutex<StdRng>,
 }

beacon_node/builder_client/src/lib.rs

@@ -1,4 +1,3 @@
-use eth2::ok_or_error;
 use eth2::types::builder_bid::SignedBuilderBid;
 use eth2::types::{
     BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash, ExecutionPayload,
@@ -6,23 +5,33 @@ use eth2::types::{
     Slot,
 };
 pub use eth2::Error;
+use eth2::{ok_or_error, StatusCode};
 use reqwest::{IntoUrl, Response};
 use sensitive_url::SensitiveUrl;
 use serde::de::DeserializeOwned;
 use serde::Serialize;
 use std::time::Duration;
 
-pub const DEFAULT_GET_HEADER_TIMEOUT_MILLIS: u64 = 500;
+pub const DEFAULT_TIMEOUT_MILLIS: u64 = 15000;
+
+/// This timeout is in accordance with v0.2.0 of the [builder specs](https://github.com/flashbots/mev-boost/pull/20).
+pub const DEFAULT_GET_HEADER_TIMEOUT_MILLIS: u64 = 1000;
 
 #[derive(Clone)]
 pub struct Timeouts {
     get_header: Duration,
+    post_validators: Duration,
+    post_blinded_blocks: Duration,
+    get_builder_status: Duration,
 }
 
 impl Default for Timeouts {
     fn default() -> Self {
         Self {
             get_header: Duration::from_millis(DEFAULT_GET_HEADER_TIMEOUT_MILLIS),
+            post_validators: Duration::from_millis(DEFAULT_TIMEOUT_MILLIS),
+            post_blinded_blocks: Duration::from_millis(DEFAULT_TIMEOUT_MILLIS),
+            get_builder_status: Duration::from_millis(DEFAULT_TIMEOUT_MILLIS),
         }
     }
 }
@@ -51,14 +60,6 @@ impl BuilderHttpClient {
         })
     }
 
-    async fn get<T: DeserializeOwned, U: IntoUrl>(&self, url: U) -> Result<T, Error> {
-        self.get_response_with_timeout(url, None)
-            .await?
-            .json()
-            .await
-            .map_err(Error::Reqwest)
-    }
-
     async fn get_with_timeout<T: DeserializeOwned, U: IntoUrl>(
         &self,
         url: U,
@@ -104,14 +105,13 @@ impl BuilderHttpClient {
         &self,
         url: U,
         body: &T,
+        timeout: Option<Duration>,
     ) -> Result<Response, Error> {
-        let response = self
-            .client
-            .post(url)
-            .json(body)
-            .send()
-            .await
-            .map_err(Error::Reqwest)?;
+        let mut builder = self.client.post(url);
+        if let Some(timeout) = timeout {
+            builder = builder.timeout(timeout);
+        }
+        let response = builder.json(body).send().await.map_err(Error::Reqwest)?;
         ok_or_error(response).await
     }
 
@@ -129,7 +129,8 @@ impl BuilderHttpClient {
             .push("builder")
             .push("validators");
 
-        self.post_generic(path, &validator, None).await?;
+        self.post_generic(path, &validator, Some(self.timeouts.post_validators))
+            .await?;
         Ok(())
     }
 
@@ -148,7 +149,11 @@ impl BuilderHttpClient {
             .push("blinded_blocks");
 
         Ok(self
-            .post_with_raw_response(path, &blinded_block)
+            .post_with_raw_response(
+                path,
+                &blinded_block,
+                Some(self.timeouts.post_blinded_blocks),
+            )
             .await?
             .json()
             .await?)
@@ -160,7 +165,7 @@ impl BuilderHttpClient {
         slot: Slot,
         parent_hash: ExecutionBlockHash,
         pubkey: &PublicKeyBytes,
-    ) -> Result<ForkVersionedResponse<SignedBuilderBid<E, Payload>>, Error> {
+    ) -> Result<Option<ForkVersionedResponse<SignedBuilderBid<E, Payload>>>, Error> {
         let mut path = self.server.full.clone();
 
         path.path_segments_mut()
@@ -173,7 +178,13 @@ impl BuilderHttpClient {
             .push(format!("{parent_hash:?}").as_str())
             .push(pubkey.as_hex_string().as_str());
 
-        self.get_with_timeout(path, self.timeouts.get_header).await
+        let resp = self.get_with_timeout(path, self.timeouts.get_header).await;
+
+        if matches!(resp, Err(Error::StatusCode(StatusCode::NO_CONTENT))) {
+            Ok(None)
+        } else {
+            resp.map(Some)
+        }
     }
 
     /// `GET /eth/v1/builder/status`
@@ -187,6 +198,7 @@ impl BuilderHttpClient {
             .push("builder")
             .push("status");
 
-        self.get(path).await
+        self.get_with_timeout(path, self.timeouts.get_builder_status)
+            .await
     }
 }

beacon_node/execution_layer/Cargo.toml

@@ -40,3 +40,7 @@ lazy_static = "1.4.0"
 ethers-core = { git = "https://github.com/gakonst/ethers-rs", rev = "02ad93a1cfb7b62eb051c77c61dc4c0218428e4a" }
 builder_client = { path = "../builder_client" }
 fork_choice = { path = "../../consensus/fork_choice" }
+mev-build-rs = {git = "https://github.com/ralexstokes/mev-rs", tag = "v0.2.0"}
+ethereum-consensus = {git = "https://github.com/ralexstokes/ethereum-consensus"}
+ssz-rs = {git = "https://github.com/ralexstokes/ssz-rs"}
+

beacon_node/execution_layer/src/engine_api/json_structures.rs

@@ -1,9 +1,6 @@
 use super::*;
 use serde::{Deserialize, Serialize};
-use types::{
-    EthSpec, ExecutionBlockHash, ExecutionPayloadHeader, FixedVector, Transaction, Unsigned,
-    VariableList,
-};
+use types::{EthSpec, ExecutionBlockHash, FixedVector, Transaction, Unsigned, VariableList};
 
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -432,59 +429,6 @@ impl From<ForkchoiceUpdatedResponse> for JsonForkchoiceUpdatedV1Response {
     }
 }
 
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
-pub enum JsonProposeBlindedBlockResponseStatus {
-    Valid,
-    Invalid,
-    Syncing,
-}
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-#[serde(bound = "E: EthSpec")]
-pub struct JsonProposeBlindedBlockResponse<E: EthSpec> {
-    pub result: ExecutionPayload<E>,
-    pub error: Option<String>,
-}
-
-impl<E: EthSpec> From<JsonProposeBlindedBlockResponse<E>> for ExecutionPayload<E> {
-    fn from(j: JsonProposeBlindedBlockResponse<E>) -> Self {
-        let JsonProposeBlindedBlockResponse { result, error: _ } = j;
-        result
-    }
-}
-
-impl From<JsonProposeBlindedBlockResponseStatus> for ProposeBlindedBlockResponseStatus {
-    fn from(j: JsonProposeBlindedBlockResponseStatus) -> Self {
-        match j {
-            JsonProposeBlindedBlockResponseStatus::Valid => {
-                ProposeBlindedBlockResponseStatus::Valid
-            }
-            JsonProposeBlindedBlockResponseStatus::Invalid => {
-                ProposeBlindedBlockResponseStatus::Invalid
-            }
-            JsonProposeBlindedBlockResponseStatus::Syncing => {
-                ProposeBlindedBlockResponseStatus::Syncing
-            }
-        }
-    }
-}
-impl From<ProposeBlindedBlockResponseStatus> for JsonProposeBlindedBlockResponseStatus {
-    fn from(f: ProposeBlindedBlockResponseStatus) -> Self {
-        match f {
-            ProposeBlindedBlockResponseStatus::Valid => {
-                JsonProposeBlindedBlockResponseStatus::Valid
-            }
-            ProposeBlindedBlockResponseStatus::Invalid => {
-                JsonProposeBlindedBlockResponseStatus::Invalid
-            }
-            ProposeBlindedBlockResponseStatus::Syncing => {
-                JsonProposeBlindedBlockResponseStatus::Syncing
-            }
-        }
-    }
-}
-
 #[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct TransitionConfigurationV1 {

beacon_node/execution_layer/src/lib.rs

@@ -4,6 +4,7 @@
 //! This crate only provides useful functionality for "The Merge", it does not provide any of the
 //! deposit-contract functionality that the `beacon_node/eth1` crate already provides.
 
+use crate::payload_cache::PayloadCache;
 use auth::{strip_prefix, Auth, JwtKey};
 use builder_client::BuilderHttpClient;
 use engine_api::Error as ApiError;
@@ -31,13 +32,14 @@ use tokio::{
     time::sleep,
 };
 use types::{
-    BlindedPayload, BlockType, ChainSpec, Epoch, ExecPayload, ExecutionBlockHash,
+    BlindedPayload, BlockType, ChainSpec, Epoch, ExecPayload, ExecutionBlockHash, ForkName,
     ProposerPreparationData, PublicKeyBytes, SignedBeaconBlock, Slot,
 };
 
 mod engine_api;
 mod engines;
 mod metrics;
+pub mod payload_cache;
 mod payload_status;
 pub mod test_utils;
 
@@ -69,6 +71,7 @@ pub enum Error {
     NoPayloadBuilder,
     ApiError(ApiError),
     Builder(builder_client::Error),
+    NoHeaderFromBuilder,
     EngineError(Box<EngineError>),
     NotSynced,
     ShuttingDown,
@@ -101,6 +104,26 @@ pub struct Proposer {
     payload_attributes: PayloadAttributes,
 }
 
+/// Information from the beacon chain that is necessary for querying the builder API.
+pub struct BuilderParams {
+    pub pubkey: PublicKeyBytes,
+    pub slot: Slot,
+    pub chain_health: ChainHealth,
+}
+
+pub enum ChainHealth {
+    Healthy,
+    Unhealthy(FailedCondition),
+    PreMerge,
+}
+
+#[derive(Debug)]
+pub enum FailedCondition {
+    Skips,
+    SkipsPerEpoch,
+    EpochsSinceFinalization,
+}
+
 struct Inner<E: EthSpec> {
     engine: Arc<Engine>,
     builder: Option<BuilderHttpClient>,
@@ -110,7 +133,7 @@ struct Inner<E: EthSpec> {
     execution_blocks: Mutex<LruCache<ExecutionBlockHash, ExecutionBlock>>,
     proposers: RwLock<HashMap<ProposerKey, Proposer>>,
     executor: TaskExecutor,
-    phantom: std::marker::PhantomData<E>,
+    payload_cache: PayloadCache<E>,
     log: Logger,
 }
 
@@ -212,7 +235,7 @@ impl<T: EthSpec> ExecutionLayer<T> {
             proposers: RwLock::new(HashMap::new()),
             execution_blocks: Mutex::new(LruCache::new(EXECUTION_BLOCKS_LRU_CACHE_SIZE)),
             executor,
-            phantom: std::marker::PhantomData,
+            payload_cache: PayloadCache::default(),
             log,
         };
 
@@ -231,6 +254,16 @@ impl<T: EthSpec> ExecutionLayer<T> {
         &self.inner.builder
     }
 
+    /// Cache a full payload, keyed on the `tree_hash_root` of its `transactions` field.
+    fn cache_payload(&self, payload: &ExecutionPayload<T>) -> Option<ExecutionPayload<T>> {
+        self.inner.payload_cache.put(payload.clone())
+    }
+
+    /// Attempt to retrieve a full payload from the payload cache by the `transactions_root`.
+    pub fn get_payload_by_root(&self, root: &Hash256) -> Option<ExecutionPayload<T>> {
+        self.inner.payload_cache.pop(root)
+    }
+
     pub fn executor(&self) -> &TaskExecutor {
         &self.inner.executor
     }
@@ -487,9 +520,9 @@ impl<T: EthSpec> ExecutionLayer<T> {
         timestamp: u64,
         prev_randao: Hash256,
         proposer_index: u64,
-        pubkey: Option<PublicKeyBytes>,
-        slot: Slot,
         forkchoice_update_params: ForkchoiceUpdateParameters,
+        builder_params: BuilderParams,
+        spec: &ChainSpec,
     ) -> Result<Payload, Error> {
         let suggested_fee_recipient = self.get_suggested_fee_recipient(proposer_index).await;
 
@@ -504,9 +537,9 @@ impl<T: EthSpec> ExecutionLayer<T> {
                     timestamp,
                     prev_randao,
                     suggested_fee_recipient,
-                    pubkey,
-                    slot,
                     forkchoice_update_params,
+                    builder_params,
+                    spec,
                 )
                 .await
             }
@@ -534,36 +567,137 @@ impl<T: EthSpec> ExecutionLayer<T> {
         timestamp: u64,
         prev_randao: Hash256,
         suggested_fee_recipient: Address,
-        pubkey_opt: Option<PublicKeyBytes>,
-        slot: Slot,
         forkchoice_update_params: ForkchoiceUpdateParameters,
+        builder_params: BuilderParams,
+        spec: &ChainSpec,
     ) -> Result<Payload, Error> {
-        //FIXME(sean) fallback logic included in PR #3134
+        if let Some(builder) = self.builder() {
+            let slot = builder_params.slot;
+            let pubkey = builder_params.pubkey;
 
-        // Don't attempt to outsource payload construction until after the merge transition has been
-        // finalized. We want to be conservative with payload construction until then.
-        if let (Some(builder), Some(pubkey)) = (self.builder(), pubkey_opt) {
-            if forkchoice_update_params
-                .finalized_hash
-                .map_or(false, |finalized_block_hash| {
-                    finalized_block_hash != ExecutionBlockHash::zero()
-                })
-            {
-                info!(
-                    self.log(),
-                    "Requesting blinded header from connected builder";
-                    "slot" => ?slot,
-                    "pubkey" => ?pubkey,
-                    "parent_hash" => ?parent_hash,
-                );
-                return builder
-                    .get_builder_header::<T, Payload>(slot, parent_hash, &pubkey)
-                    .await
-                    .map(|d| d.data.message.header)
-                    .map_err(Error::Builder);
+            match builder_params.chain_health {
+                ChainHealth::Healthy => {
+                    info!(
+                        self.log(),
+                        "Requesting blinded header from connected builder";
+                        "slot" => ?slot,
+                        "pubkey" => ?pubkey,
+                        "parent_hash" => ?parent_hash,
+                    );
+                    let (relay_result, local_result) = tokio::join!(
+                        builder.get_builder_header::<T, Payload>(slot, parent_hash, &pubkey),
+                        self.get_full_payload_caching(
+                            parent_hash,
+                            timestamp,
+                            prev_randao,
+                            suggested_fee_recipient,
+                            forkchoice_update_params,
+                        )
+                    );
+
+                    return match (relay_result, local_result) {
+                        (Err(e), Ok(local)) => {
+                            warn!(
+                                self.log(),
+                                "Unable to retrieve a payload from a connected \
+                                builder, falling back to the local execution client: {e:?}"
+                            );
+                            Ok(local)
+                        }
+                        (Ok(None), Ok(local)) => {
+                            warn!(
+                                self.log(),
+                                "No payload provided by connected builder. \
+                                Attempting to propose through local execution engine"
+                            );
+                            Ok(local)
+                        }
+                        (Ok(Some(relay)), Ok(local)) => {
+                            let is_signature_valid = relay.data.verify_signature(spec);
+                            let header = relay.data.message.header;
+
+                            info!(
+                                self.log(),
+                                "Received a payload header from the connected builder";
+                                "block_hash" => ?header.block_hash(),
+                            );
+
+                            if header.parent_hash() != parent_hash {
+                                warn!(
+                                    self.log(),
+                                    "Invalid parent hash from connected builder, \
+                                    falling back to local execution engine."
+                                );
+                                Ok(local)
+                            } else if header.prev_randao() != prev_randao {
+                                warn!(
+                                    self.log(),
+                                    "Invalid prev randao from connected builder, \
+                                    falling back to local execution engine."
+                                );
+                                Ok(local)
+                            } else if header.timestamp() != local.timestamp() {
+                                warn!(
+                                    self.log(),
+                                    "Invalid timestamp from connected builder, \
+                                    falling back to local execution engine."
+                                );
+                                Ok(local)
+                            } else if header.block_number() != local.block_number() {
+                                warn!(
+                                    self.log(),
+                                    "Invalid block number from connected builder, \
+                                    falling back to local execution engine."
+                                );
+                                Ok(local)
+                            } else if !matches!(relay.version, Some(ForkName::Merge)) {
+                                // Once fork information is added to the payload, we will need to
+                                // check that the local and relay payloads match. At this point, if
+                                // we are requesting a payload at all, we have to assume this is
+                                // the Bellatrix fork.
+                                warn!(
+                                    self.log(),
+                                    "Invalid fork from connected builder, falling \
+                                    back to local execution engine."
+                                );
+                                Ok(local)
+                            } else if !is_signature_valid {
+                                let pubkey_bytes = relay.data.message.pubkey;
+                                warn!(self.log(), "Invalid signature for pubkey {pubkey_bytes} on \
+                                    bid from connected builder, falling back to local execution engine.");
+                                Ok(local)
+                            } else {
+                                if header.fee_recipient() != suggested_fee_recipient {
+                                    info!(
+                                        self.log(),
+                                        "Fee recipient from connected builder does \
+                                        not match, using it anyways."
+                                    );
+                                }
+                                Ok(header)
+                            }
+                        }
+                        (relay_result, Err(local_error)) => {
+                            warn!(self.log(), "Failure from local execution engine. Attempting to \
+                                propose through connected builder"; "error" => ?local_error);
+                            relay_result
+                                .map_err(Error::Builder)?
+                                .ok_or(Error::NoHeaderFromBuilder)
+                                .map(|d| d.data.message.header)
+                        }
+                    };
+                }
+                ChainHealth::Unhealthy(condition) => {
+                    info!(self.log(), "Due to poor chain health the local execution engine will be used \
+                                        for payload construction. To adjust chain health conditions \
+                                        Use `builder-fallback` prefixed flags";
+                        "failed_condition" => ?condition)
+                }
+                // Intentional no-op, so we never attempt builder API proposals pre-merge.
+                ChainHealth::PreMerge => (),
             }
         }
-        self.get_full_payload::<Payload>(
+        self.get_full_payload_caching(
             parent_hash,
             timestamp,
             prev_randao,
@@ -593,6 +727,26 @@ impl<T: EthSpec> ExecutionLayer<T> {
         .await
     }
 
+    /// Get a full payload and cache its result in the execution layer's payload cache.
+    async fn get_full_payload_caching<Payload: ExecPayload<T>>(
+        &self,
+        parent_hash: ExecutionBlockHash,
+        timestamp: u64,
+        prev_randao: Hash256,
+        suggested_fee_recipient: Address,
+        forkchoice_update_params: ForkchoiceUpdateParameters,
+    ) -> Result<Payload, Error> {
+        self.get_full_payload_with(
+            parent_hash,
+            timestamp,
+            prev_randao,
+            suggested_fee_recipient,
+            forkchoice_update_params,
+            Self::cache_payload,
+        )
+        .await
+    }
+
     async fn get_full_payload_with<Payload: ExecPayload<T>>(
         &self,
         parent_hash: ExecutionBlockHash,

beacon_node/execution_layer/src/payload_cache.rs

@@ -0,0 +1,33 @@
+use lru::LruCache;
+use parking_lot::Mutex;
+use tree_hash::TreeHash;
+use types::{EthSpec, ExecutionPayload, Hash256};
+
+pub const DEFAULT_PAYLOAD_CACHE_SIZE: usize = 10;
+
+/// A cache mapping execution payloads by tree hash roots.
+pub struct PayloadCache<T: EthSpec> {
+    payloads: Mutex<LruCache<PayloadCacheId, ExecutionPayload<T>>>,
+}
+
+#[derive(Hash, PartialEq, Eq)]
+struct PayloadCacheId(Hash256);
+
+impl<T: EthSpec> Default for PayloadCache<T> {
+    fn default() -> Self {
+        PayloadCache {
+            payloads: Mutex::new(LruCache::new(DEFAULT_PAYLOAD_CACHE_SIZE)),
+        }
+    }
+}
+
+impl<T: EthSpec> PayloadCache<T> {
+    pub fn put(&self, payload: ExecutionPayload<T>) -> Option<ExecutionPayload<T>> {
+        let root = payload.tree_hash_root();
+        self.payloads.lock().put(PayloadCacheId(root), payload)
+    }
+
+    pub fn pop(&self, root: &Hash256) -> Option<ExecutionPayload<T>> {
+        self.payloads.lock().pop(&PayloadCacheId(*root))
+    }
+}

beacon_node/execution_layer/src/test_utils/mock_builder.rs

@@ -0,0 +1,383 @@
+use crate::test_utils::DEFAULT_JWT_SECRET;
+use crate::{Config, ExecutionLayer, PayloadAttributes};
+use async_trait::async_trait;
+use eth2::types::{BlockId, StateId, ValidatorId};
+use eth2::{BeaconNodeHttpClient, Timeouts};
+use ethereum_consensus::crypto::{SecretKey, Signature};
+use ethereum_consensus::primitives::BlsPublicKey;
+pub use ethereum_consensus::state_transition::Context;
+use fork_choice::ForkchoiceUpdateParameters;
+use mev_build_rs::{
+    sign_builder_message, verify_signed_builder_message, BidRequest, BlindedBlockProviderError,
+    BlindedBlockProviderServer, BuilderBid, ExecutionPayload as ServerPayload,
+    ExecutionPayloadHeader as ServerPayloadHeader, SignedBlindedBeaconBlock, SignedBuilderBid,
+    SignedValidatorRegistration,
+};
+use parking_lot::RwLock;
+use sensitive_url::SensitiveUrl;
+use ssz::{Decode, Encode};
+use ssz_rs::{Merkleized, SimpleSerialize};
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::net::Ipv4Addr;
+use std::sync::Arc;
+use std::time::Duration;
+use task_executor::TaskExecutor;
+use tempfile::NamedTempFile;
+use tree_hash::TreeHash;
+use types::{
+    Address, BeaconState, BlindedPayload, ChainSpec, EthSpec, ExecPayload, Hash256, Slot, Uint256,
+};
+
+#[derive(Clone)]
+pub enum Operation {
+    FeeRecipient(Address),
+    GasLimit(usize),
+    Value(usize),
+    ParentHash(Hash256),
+    PrevRandao(Hash256),
+    BlockNumber(usize),
+    Timestamp(usize),
+}
+
+impl Operation {
+    fn apply(self, bid: &mut BuilderBid) -> Result<(), BlindedBlockProviderError> {
+        match self {
+            Operation::FeeRecipient(fee_recipient) => {
+                bid.header.fee_recipient = to_ssz_rs(&fee_recipient)?
+            }
+            Operation::GasLimit(gas_limit) => bid.header.gas_limit = gas_limit as u64,
+            Operation::Value(value) => bid.value = to_ssz_rs(&Uint256::from(value))?,
+            Operation::ParentHash(parent_hash) => bid.header.parent_hash = to_ssz_rs(&parent_hash)?,
+            Operation::PrevRandao(prev_randao) => bid.header.prev_randao = to_ssz_rs(&prev_randao)?,
+            Operation::BlockNumber(block_number) => bid.header.block_number = block_number as u64,
+            Operation::Timestamp(timestamp) => bid.header.timestamp = timestamp as u64,
+        }
+        Ok(())
+    }
+}
+
+pub struct TestingBuilder<E: EthSpec> {
+    server: BlindedBlockProviderServer<MockBuilder<E>>,
+    pub builder: MockBuilder<E>,
+}
+
+impl<E: EthSpec> TestingBuilder<E> {
+    pub fn new(
+        mock_el_url: SensitiveUrl,
+        builder_url: SensitiveUrl,
+        beacon_url: SensitiveUrl,
+        spec: ChainSpec,
+        executor: TaskExecutor,
+    ) -> Self {
+        let file = NamedTempFile::new().unwrap();
+        let path = file.path().into();
+        std::fs::write(&path, hex::encode(DEFAULT_JWT_SECRET)).unwrap();
+
+        // This EL should not talk to a builder
+        let config = Config {
+            execution_endpoints: vec![mock_el_url],
+            secret_files: vec![path],
+            suggested_fee_recipient: None,
+            ..Default::default()
+        };
+
+        let el =
+            ExecutionLayer::from_config(config, executor.clone(), executor.log().clone()).unwrap();
+
+        // This should probably be done for all fields, we only update ones we are testing with so far.
+        let mut context = Context::for_mainnet();
+        context.terminal_total_difficulty = to_ssz_rs(&spec.terminal_total_difficulty).unwrap();
+        context.terminal_block_hash = to_ssz_rs(&spec.terminal_block_hash).unwrap();
+        context.terminal_block_hash_activation_epoch =
+            to_ssz_rs(&spec.terminal_block_hash_activation_epoch).unwrap();
+
+        let builder = MockBuilder::new(
+            el,
+            BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(Duration::from_secs(1))),
+            spec,
+            context,
+        );
+        let port = builder_url.full.port().unwrap();
+        let host: Ipv4Addr = builder_url
+            .full
+            .host_str()
+            .unwrap()
+            .to_string()
+            .parse()
+            .unwrap();
+        let server = BlindedBlockProviderServer::new(host, port, builder.clone());
+        Self { server, builder }
+    }
+
+    pub async fn run(&self) {
+        self.server.run().await
+    }
+}
+
+#[derive(Clone)]
+pub struct MockBuilder<E: EthSpec> {
+    el: ExecutionLayer<E>,
+    beacon_client: BeaconNodeHttpClient,
+    spec: ChainSpec,
+    context: Arc<Context>,
+    val_registration_cache: Arc<RwLock<HashMap<BlsPublicKey, SignedValidatorRegistration>>>,
+    builder_sk: SecretKey,
+    operations: Arc<RwLock<Vec<Operation>>>,
+    invalidate_signatures: Arc<RwLock<bool>>,
+}
+
+impl<E: EthSpec> MockBuilder<E> {
+    pub fn new(
+        el: ExecutionLayer<E>,
+        beacon_client: BeaconNodeHttpClient,
+        spec: ChainSpec,
+        context: Context,
+    ) -> Self {
+        let sk = SecretKey::random(&mut rand::thread_rng()).unwrap();
+        Self {
+            el,
+            beacon_client,
+            // Should keep spec and context consistent somehow
+            spec,
+            context: Arc::new(context),
+            val_registration_cache: Arc::new(RwLock::new(HashMap::new())),
+            builder_sk: sk,
+            operations: Arc::new(RwLock::new(vec![])),
+            invalidate_signatures: Arc::new(RwLock::new(false)),
+        }
+    }
+
+    pub fn add_operation(&self, op: Operation) {
+        self.operations.write().push(op);
+    }
+
+    pub fn invalid_signatures(&self) {
+        *self.invalidate_signatures.write() = true;
+    }
+
+    pub fn valid_signatures(&mut self) {
+        *self.invalidate_signatures.write() = false;
+    }
+
+    fn apply_operations(&self, bid: &mut BuilderBid) -> Result<(), BlindedBlockProviderError> {
+        let mut guard = self.operations.write();
+        while let Some(op) = guard.pop() {
+            op.apply(bid)?;
+        }
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl<E: EthSpec> mev_build_rs::BlindedBlockProvider for MockBuilder<E> {
+    async fn register_validators(
+        &self,
+        registrations: &mut [SignedValidatorRegistration],
+    ) -> Result<(), BlindedBlockProviderError> {
+        for registration in registrations {
+            let pubkey = registration.message.public_key.clone();
+            let message = &mut registration.message;
+            verify_signed_builder_message(
+                message,
+                &registration.signature,
+                &pubkey,
+                &self.context,
+            )?;
+            self.val_registration_cache.write().insert(
+                registration.message.public_key.clone(),
+                registration.clone(),
+            );
+        }
+
+        Ok(())
+    }
+
+    async fn fetch_best_bid(
+        &self,
+        bid_request: &BidRequest,
+    ) -> Result<SignedBuilderBid, BlindedBlockProviderError> {
+        let slot = Slot::new(bid_request.slot);
+        let signed_cached_data = self
+            .val_registration_cache
+            .read()
+            .get(&bid_request.public_key)
+            .ok_or_else(|| convert_err("missing registration"))?
+            .clone();
+        let cached_data = signed_cached_data.message;
+
+        let head = self
+            .beacon_client
+            .get_beacon_blocks::<E>(BlockId::Head)
+            .await
+            .map_err(convert_err)?
+            .ok_or_else(|| convert_err("missing head block"))?;
+
+        let block = head.data.message_merge().map_err(convert_err)?;
+        let head_block_root = block.tree_hash_root();
+        let head_execution_hash = block.body.execution_payload.execution_payload.block_hash;
+        if head_execution_hash != from_ssz_rs(&bid_request.parent_hash)? {
+            return Err(BlindedBlockProviderError::Custom(format!(
+                "head mismatch: {} {}",
+                head_execution_hash, bid_request.parent_hash
+            )));
+        }
+
+        let finalized_execution_hash = self
+            .beacon_client
+            .get_beacon_blocks::<E>(BlockId::Finalized)
+            .await
+            .map_err(convert_err)?
+            .ok_or_else(|| convert_err("missing finalized block"))?
+            .data
+            .message_merge()
+            .map_err(convert_err)?
+            .body
+            .execution_payload
+            .execution_payload
+            .block_hash;
+
+        let justified_execution_hash = self
+            .beacon_client
+            .get_beacon_blocks::<E>(BlockId::Justified)
+            .await
+            .map_err(convert_err)?
+            .ok_or_else(|| convert_err("missing finalized block"))?
+            .data
+            .message_merge()
+            .map_err(convert_err)?
+            .body
+            .execution_payload
+            .execution_payload
+            .block_hash;
+
+        let val_index = self
+            .beacon_client
+            .get_beacon_states_validator_id(
+                StateId::Head,
+                &ValidatorId::PublicKey(from_ssz_rs(&cached_data.public_key)?),
+            )
+            .await
+            .map_err(convert_err)?
+            .ok_or_else(|| convert_err("missing validator from state"))?
+            .data
+            .index;
+        let fee_recipient = from_ssz_rs(&cached_data.fee_recipient)?;
+        let slots_since_genesis = slot.as_u64() - self.spec.genesis_slot.as_u64();
+
+        let genesis_time = self
+            .beacon_client
+            .get_beacon_genesis()
+            .await
+            .map_err(convert_err)?
+            .data
+            .genesis_time;
+        let timestamp = (slots_since_genesis * self.spec.seconds_per_slot) + genesis_time;
+
+        let head_state: BeaconState<E> = self
+            .beacon_client
+            .get_debug_beacon_states(StateId::Head)
+            .await
+            .map_err(convert_err)?
+            .ok_or_else(|| BlindedBlockProviderError::Custom("missing head state".to_string()))?
+            .data;
+        let prev_randao = head_state
+            .get_randao_mix(head_state.current_epoch())
+            .map_err(convert_err)?;
+
+        let payload_attributes = PayloadAttributes {
+            timestamp,
+            prev_randao: *prev_randao,
+            suggested_fee_recipient: fee_recipient,
+        };
+
+        self.el
+            .insert_proposer(slot, head_block_root, val_index, payload_attributes)
+            .await;
+
+        let forkchoice_update_params = ForkchoiceUpdateParameters {
+            head_root: Hash256::zero(),
+            head_hash: None,
+            justified_hash: Some(justified_execution_hash),
+            finalized_hash: Some(finalized_execution_hash),
+        };
+
+        let payload = self
+            .el
+            .get_full_payload_caching::<BlindedPayload<E>>(
+                head_execution_hash,
+                timestamp,
+                *prev_randao,
+                fee_recipient,
+                forkchoice_update_params,
+            )
+            .await
+            .map_err(convert_err)?
+            .to_execution_payload_header();
+
+        let json_payload = serde_json::to_string(&payload).map_err(convert_err)?;
+        let mut header: ServerPayloadHeader =
+            serde_json::from_str(json_payload.as_str()).map_err(convert_err)?;
+
+        header.gas_limit = cached_data.gas_limit;
+
+        let mut message = BuilderBid {
+            header,
+            value: ssz_rs::U256::default(),
+            public_key: self.builder_sk.public_key(),
+        };
+
+        self.apply_operations(&mut message)?;
+
+        let mut signature =
+            sign_builder_message(&mut message, &self.builder_sk, self.context.as_ref())?;
+
+        if *self.invalidate_signatures.read() {
+            signature = Signature::default();
+        }
+
+        let signed_bid = SignedBuilderBid { message, signature };
+        Ok(signed_bid)
+    }
+
+    async fn open_bid(
+        &self,
+        signed_block: &mut SignedBlindedBeaconBlock,
+    ) -> Result<ServerPayload, BlindedBlockProviderError> {
+        let payload = self
+            .el
+            .get_payload_by_root(&from_ssz_rs(
+                &signed_block
+                    .message
+                    .body
+                    .execution_payload_header
+                    .hash_tree_root()
+                    .map_err(convert_err)?,
+            )?)
+            .ok_or_else(|| convert_err("missing payload for tx root"))?;
+
+        let json_payload = serde_json::to_string(&payload).map_err(convert_err)?;
+        serde_json::from_str(json_payload.as_str()).map_err(convert_err)
+    }
+}
+
+pub fn from_ssz_rs<T: SimpleSerialize, U: Decode>(
+    ssz_rs_data: &T,
+) -> Result<U, BlindedBlockProviderError> {
+    U::from_ssz_bytes(
+        ssz_rs::serialize(ssz_rs_data)
+            .map_err(convert_err)?
+            .as_ref(),
+    )
+    .map_err(convert_err)
+}
+
+pub fn to_ssz_rs<T: Encode, U: SimpleSerialize>(
+    ssz_data: &T,
+) -> Result<U, BlindedBlockProviderError> {
+    ssz_rs::deserialize::<U>(&ssz_data.as_ssz_bytes()).map_err(convert_err)
+}
+
+fn convert_err<E: Debug>(e: E) -> BlindedBlockProviderError {
+    BlindedBlockProviderError::Custom(format!("{e:?}"))
+}

beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs

@@ -7,6 +7,7 @@ use crate::{
 use sensitive_url::SensitiveUrl;
 use task_executor::TaskExecutor;
 use tempfile::NamedTempFile;
+use tree_hash::TreeHash;
 use types::{Address, ChainSpec, Epoch, EthSpec, FullPayload, Hash256, Uint256};
 
 pub struct MockExecutionLayer<T: EthSpec> {
@@ -124,6 +125,11 @@ impl<T: EthSpec> MockExecutionLayer<T> {
             .unwrap();
 
         let validator_index = 0;
+        let builder_params = BuilderParams {
+            pubkey: PublicKeyBytes::empty(),
+            slot,
+            chain_health: ChainHealth::Healthy,
+        };
         let payload = self
             .el
             .get_payload::<FullPayload<T>>(
@@ -131,9 +137,9 @@ impl<T: EthSpec> MockExecutionLayer<T> {
                 timestamp,
                 prev_randao,
                 validator_index,
-                None,
-                slot,
                 forkchoice_update_params,
+                builder_params,
+                &self.spec,
             )
             .await
             .unwrap()
@@ -144,6 +150,43 @@ impl<T: EthSpec> MockExecutionLayer<T> {
         assert_eq!(payload.timestamp, timestamp);
         assert_eq!(payload.prev_randao, prev_randao);
 
+        // Ensure the payload cache is empty.
+        assert!(self
+            .el
+            .get_payload_by_root(&payload.tree_hash_root())
+            .is_none());
+        let builder_params = BuilderParams {
+            pubkey: PublicKeyBytes::empty(),
+            slot,
+            chain_health: ChainHealth::Healthy,
+        };
+        let payload_header = self
+            .el
+            .get_payload::<BlindedPayload<T>>(
+                parent_hash,
+                timestamp,
+                prev_randao,
+                validator_index,
+                forkchoice_update_params,
+                builder_params,
+                &self.spec,
+            )
+            .await
+            .unwrap()
+            .execution_payload_header;
+        assert_eq!(payload_header.block_hash, block_hash);
+        assert_eq!(payload_header.parent_hash, parent_hash);
+        assert_eq!(payload_header.block_number, block_number);
+        assert_eq!(payload_header.timestamp, timestamp);
+        assert_eq!(payload_header.prev_randao, prev_randao);
+
+        // Ensure the payload cache has the correct payload.
+        assert_eq!(
+            self.el
+                .get_payload_by_root(&payload_header.tree_hash_root()),
+            Some(payload.clone())
+        );
+
         let status = self.el.notify_new_payload(&payload).await.unwrap();
         assert_eq!(status, PayloadStatus::Valid);
 

beacon_node/execution_layer/src/test_utils/mod.rs

@@ -22,6 +22,7 @@ use types::{EthSpec, ExecutionBlockHash, Uint256};
 use warp::{http::StatusCode, Filter, Rejection};
 
 pub use execution_block_generator::{generate_pow_block, Block, ExecutionBlockGenerator};
+pub use mock_builder::{Context as MockBuilderContext, MockBuilder, Operation, TestingBuilder};
 pub use mock_execution_layer::MockExecutionLayer;
 
 pub const DEFAULT_TERMINAL_DIFFICULTY: u64 = 6400;
@@ -30,6 +31,7 @@ pub const DEFAULT_JWT_SECRET: [u8; 32] = [42; 32];
 
 mod execution_block_generator;
 mod handle_rpc;
+mod mock_builder;
 mod mock_execution_layer;
 
 /// Configuration for the MockExecutionLayer.

beacon_node/http_api/Cargo.toml

@@ -32,15 +32,16 @@ parking_lot = "0.12.0"
 safe_arith = {path = "../../consensus/safe_arith"}
 task_executor = { path = "../../common/task_executor" }
 lru = "0.7.7"
+tree_hash = "0.4.1"
 
 [dev-dependencies]
 store = { path = "../store" }
 environment = { path = "../../lighthouse/environment" }
-tree_hash = "0.4.1"
 sensitive_url = { path = "../../common/sensitive_url" }
 logging = { path = "../../common/logging" }
 serde_json = "1.0.58"
 proto_array = { path = "../../consensus/proto_array" }
+unused_port = {path = "../../common/unused_port"}
 
 [[test]]
 name = "bn_http_api_tests"

beacon_node/http_api/src/lib.rs

@@ -13,17 +13,16 @@ mod block_rewards;
 mod database;
 mod metrics;
 mod proposer_duties;
+mod publish_blocks;
 mod state_id;
 mod sync_committees;
 mod validator_inclusion;
 mod version;
 
 use beacon_chain::{
-    attestation_verification::VerifiedAttestation,
-    observed_operations::ObservationOutcome,
-    validator_monitor::{get_block_delay_ms, timestamp_now},
-    AttestationError as AttnError, BeaconChain, BeaconChainError, BeaconChainTypes,
-    CountUnrealized, ProduceBlockVerification, WhenSlotSkipped,
+    attestation_verification::VerifiedAttestation, observed_operations::ObservationOutcome,
+    validator_monitor::timestamp_now, AttestationError as AttnError, BeaconChain, BeaconChainError,
+    BeaconChainTypes, ProduceBlockVerification, WhenSlotSkipped,
 };
 pub use block_id::BlockId;
 use eth2::types::{self as api_types, EndpointVersion, ValidatorId};
@@ -45,12 +44,11 @@ use std::sync::Arc;
 use tokio::sync::mpsc::UnboundedSender;
 use tokio_stream::{wrappers::BroadcastStream, StreamExt};
 use types::{
-    Attestation, AttesterSlashing, BeaconBlockBodyMerge, BeaconBlockMerge, BeaconStateError,
-    BlindedPayload, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, FullPayload,
-    ProposerPreparationData, ProposerSlashing, RelativeEpoch, Signature, SignedAggregateAndProof,
-    SignedBeaconBlock, SignedBeaconBlockMerge, SignedBlindedBeaconBlock,
-    SignedContributionAndProof, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot,
-    SyncCommitteeMessage, SyncContributionData,
+    Attestation, AttesterSlashing, BeaconStateError, BlindedPayload, CommitteeCache,
+    ConfigAndPreset, Epoch, EthSpec, ForkName, FullPayload, ProposerPreparationData,
+    ProposerSlashing, RelativeEpoch, Signature, SignedAggregateAndProof, SignedBeaconBlock,
+    SignedBlindedBeaconBlock, SignedContributionAndProof, SignedValidatorRegistrationData,
+    SignedVoluntaryExit, Slot, SyncCommitteeMessage, SyncContributionData,
 };
 use version::{
     add_consensus_version_header, execution_optimistic_fork_versioned_response,
@@ -1025,81 +1023,9 @@ pub fn serve<T: BeaconChainTypes>(
              chain: Arc<BeaconChain<T>>,
              network_tx: UnboundedSender<NetworkMessage<T::EthSpec>>,
              log: Logger| async move {
-                let seen_timestamp = timestamp_now();
-
-                // Send the block, regardless of whether or not it is valid. The API
-                // specification is very clear that this is the desired behaviour.
-                publish_pubsub_message(&network_tx, PubsubMessage::BeaconBlock(block.clone()))?;
-
-                // Determine the delay after the start of the slot, register it with metrics.
-                let delay = get_block_delay_ms(seen_timestamp, block.message(), &chain.slot_clock);
-                metrics::observe_duration(&metrics::HTTP_API_BLOCK_BROADCAST_DELAY_TIMES, delay);
-
-                match chain
-                    .process_block(block.clone(), CountUnrealized::True)
+                publish_blocks::publish_block(block, chain, &network_tx, log)
                     .await
-                {
-                    Ok(root) => {
-                        info!(
-                            log,
-                            "Valid block from HTTP API";
-                            "block_delay" => ?delay,
-                            "root" => format!("{}", root),
-                            "proposer_index" => block.message().proposer_index(),
-                            "slot" => block.slot(),
-                        );
-
-                        // Notify the validator monitor.
-                        chain.validator_monitor.read().register_api_block(
-                            seen_timestamp,
-                            block.message(),
-                            root,
-                            &chain.slot_clock,
-                        );
-
-                        // Update the head since it's likely this block will become the new
-                        // head.
-                        chain.recompute_head_at_current_slot().await;
-
-                        // Perform some logging to inform users if their blocks are being produced
-                        // late.
-                        //
-                        // Check to see the thresholds are non-zero to avoid logging errors with small
-                        // slot times (e.g., during testing)
-                        let crit_threshold = chain.slot_clock.unagg_attestation_production_delay();
-                        let error_threshold = crit_threshold / 2;
-                        if delay >= crit_threshold {
-                            crit!(
-                                log,
-                                "Block was broadcast too late";
-                                "msg" => "system may be overloaded, block likely to be orphaned",
-                                "delay_ms" => delay.as_millis(),
-                                "slot" => block.slot(),
-                                "root" => ?root,
-                            )
-                        } else if delay >= error_threshold {
-                            error!(
-                                log,
-                                "Block broadcast was delayed";
-                                "msg" => "system may be overloaded, block may be orphaned",
-                                "delay_ms" => delay.as_millis(),
-                                "slot" => block.slot(),
-                                "root" => ?root,
-                            )
-                        }
-
-                        Ok(warp::reply::json(&()))
-                    }
-                    Err(e) => {
-                        let msg = format!("{:?}", e);
-                        error!(
-                            log,
-                            "Invalid block provided to HTTP API";
-                            "reason" => &msg
-                        );
-                        Err(warp_utils::reject::broadcast_without_import(msg))
-                    }
-                }
+                    .map(|()| warp::reply())
             },
         );
 
@@ -1117,87 +1043,13 @@ pub fn serve<T: BeaconChainTypes>(
         .and(network_tx_filter.clone())
         .and(log_filter.clone())
         .and_then(
-            |block: Arc<SignedBeaconBlock<T::EthSpec, BlindedPayload<_>>>,
+            |block: SignedBeaconBlock<T::EthSpec, BlindedPayload<_>>,
              chain: Arc<BeaconChain<T>>,
              network_tx: UnboundedSender<NetworkMessage<T::EthSpec>>,
-             _log: Logger| async move {
-                if let Some(el) = chain.execution_layer.as_ref() {
-                    //FIXME(sean): we may not always receive the payload in this response because it
-                    // should be the relay's job to propogate the block. However, since this block is
-                    // already signed and sent this might be ok (so long as the relay validates
-                    // the block before revealing the payload).
-
-                    //FIXME(sean) additionally, this endpoint should serve blocks prior to Bellatrix, and should
-                    // be able to support the normal block proposal flow, because at some point full block endpoints
-                    // will be deprecated from the beacon API. This will entail creating full blocks in
-                    // `validator/blinded_blocks`, caching their payloads, and transforming them into blinded
-                    // blocks. We will access the payload of those blocks here. This flow should happen if the
-                    // execution layer has no payload builders or if we have not yet finalized post-merge transition.
-                    let payload = el.propose_blinded_beacon_block(&block).await.map_err(|e| {
-                        warp_utils::reject::custom_server_error(format!("proposal failed: {:?}", e))
-                    })?;
-                    let new_block = SignedBeaconBlock::Merge(SignedBeaconBlockMerge {
-                        message: BeaconBlockMerge {
-                            slot: block.message().slot(),
-                            proposer_index: block.message().proposer_index(),
-                            parent_root: block.message().parent_root(),
-                            state_root: block.message().state_root(),
-                            body: BeaconBlockBodyMerge {
-                                randao_reveal: block.message().body().randao_reveal().clone(),
-                                eth1_data: block.message().body().eth1_data().clone(),
-                                graffiti: *block.message().body().graffiti(),
-                                proposer_slashings: block
-                                    .message()
-                                    .body()
-                                    .proposer_slashings()
-                                    .clone(),
-                                attester_slashings: block
-                                    .message()
-                                    .body()
-                                    .attester_slashings()
-                                    .clone(),
-                                attestations: block.message().body().attestations().clone(),
-                                deposits: block.message().body().deposits().clone(),
-                                voluntary_exits: block.message().body().voluntary_exits().clone(),
-                                sync_aggregate: block
-                                    .message()
-                                    .body()
-                                    .sync_aggregate()
-                                    .unwrap()
-                                    .clone(),
-                                execution_payload: payload.into(),
-                            },
-                        },
-                        signature: block.signature().clone(),
-                    });
-                    let new_block = Arc::new(new_block);
-
-                    // Send the block, regardless of whether or not it is valid. The API
-                    // specification is very clear that this is the desired behaviour.
-                    publish_pubsub_message(
-                        &network_tx,
-                        PubsubMessage::BeaconBlock(new_block.clone()),
-                    )?;
-
-                    match chain.process_block(new_block, CountUnrealized::True).await {
-                        Ok(_) => {
-                            // Update the head since it's likely this block will become the new
-                            // head.
-                            chain.recompute_head_at_current_slot().await;
-
-                            Ok(warp::reply::json(&()))
-                        }
-                        Err(e) => {
-                            let msg = format!("{:?}", e);
-
-                            Err(warp_utils::reject::broadcast_without_import(msg))
-                        }
-                    }
-                } else {
-                    Err(warp_utils::reject::custom_server_error(
-                        "no execution layer found".to_string(),
-                    ))
-                }
+             log: Logger| async move {
+                publish_blocks::publish_blinded_block(block, chain, &network_tx, log)
+                    .await
+                    .map(|()| warp::reply())
             },
         );
 
@@ -2593,19 +2445,13 @@ pub fn serve<T: BeaconChainTypes>(
                     })
                     .collect::<Vec<_>>();
 
-                debug!(
-                    log,
-                    "Resolved validator request pubkeys";
-                    "count" => preparation_data.len()
-                );
-
                 // Update the prepare beacon proposer cache based on this request.
                 execution_layer
                     .update_proposer_preparation(current_epoch, &preparation_data)
                     .await;
 
                 // Call prepare beacon proposer blocking with the latest update in order to make
-                // sure we have a local payload to fall back to in the event of the blined block
+                // sure we have a local payload to fall back to in the event of the blinded block
                 // flow failing.
                 chain
                     .prepare_beacon_proposer(current_slot)
@@ -2617,9 +2463,37 @@ pub fn serve<T: BeaconChainTypes>(
                         ))
                     })?;
 
-                //TODO(sean): In the MEV-boost PR, add a call here to send the update request to the builder
+                let builder = execution_layer
+                    .builder()
+                    .as_ref()
+                    .ok_or(BeaconChainError::BuilderMissing)
+                    .map_err(warp_utils::reject::beacon_chain_error)?;
 
-                Ok::<_, warp::Rejection>(warp::reply::json(&()))
+                info!(
+                    log,
+                    "Forwarding register validator request to connected builder";
+                    "count" => register_val_data.len(),
+                );
+
+                builder
+                    .post_builder_validators(&register_val_data)
+                    .await
+                    .map(|resp| warp::reply::json(&resp))
+                    .map_err(|e| {
+                        error!(log, "Error from connected relay"; "error" => ?e);
+                        // Forward the HTTP status code if we are able to, otherwise fall back
+                        // to a server error.
+                        if let eth2::Error::ServerMessage(message) = e {
+                            if message.code == StatusCode::BAD_REQUEST.as_u16() {
+                                return warp_utils::reject::custom_bad_request(message.message);
+                            } else {
+                                // According to the spec this response should only be a 400 or 500,
+                                // so we fall back to a 500 here.
+                                return warp_utils::reject::custom_server_error(message.message);
+                            }
+                        }
+                        warp_utils::reject::custom_server_error(format!("{e:?}"))
+                    })
             },
         );
     // POST validator/sync_committee_subscriptions

beacon_node/http_api/src/publish_blocks.rs

@@ -0,0 +1,155 @@
+use crate::metrics;
+use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now};
+use beacon_chain::{BeaconChain, BeaconChainTypes, CountUnrealized};
+use lighthouse_network::PubsubMessage;
+use network::NetworkMessage;
+use slog::{crit, error, info, Logger};
+use slot_clock::SlotClock;
+use std::sync::Arc;
+use tokio::sync::mpsc::UnboundedSender;
+use tree_hash::TreeHash;
+use types::{
+    BlindedPayload, ExecPayload, ExecutionBlockHash, ExecutionPayload, FullPayload,
+    SignedBeaconBlock,
+};
+use warp::Rejection;
+
+/// Handles a request from the HTTP API for full blocks.
+pub async fn publish_block<T: BeaconChainTypes>(
+    block: Arc<SignedBeaconBlock<T::EthSpec>>,
+    chain: Arc<BeaconChain<T>>,
+    network_tx: &UnboundedSender<NetworkMessage<T::EthSpec>>,
+    log: Logger,
+) -> Result<(), Rejection> {
+    let seen_timestamp = timestamp_now();
+
+    // Send the block, regardless of whether or not it is valid. The API
+    // specification is very clear that this is the desired behaviour.
+    crate::publish_pubsub_message(network_tx, PubsubMessage::BeaconBlock(block.clone()))?;
+
+    // Determine the delay after the start of the slot, register it with metrics.
+    let delay = get_block_delay_ms(seen_timestamp, block.message(), &chain.slot_clock);
+    metrics::observe_duration(&metrics::HTTP_API_BLOCK_BROADCAST_DELAY_TIMES, delay);
+
+    match chain
+        .process_block(block.clone(), CountUnrealized::True)
+        .await
+    {
+        Ok(root) => {
+            info!(
+                log,
+                "Valid block from HTTP API";
+                "block_delay" => ?delay,
+                "root" => format!("{}", root),
+                "proposer_index" => block.message().proposer_index(),
+                "slot" => block.slot(),
+            );
+
+            // Notify the validator monitor.
+            chain.validator_monitor.read().register_api_block(
+                seen_timestamp,
+                block.message(),
+                root,
+                &chain.slot_clock,
+            );
+
+            // Update the head since it's likely this block will become the new
+            // head.
+            chain.recompute_head_at_current_slot().await;
+
+            // Perform some logging to inform users if their blocks are being produced
+            // late.
+            //
+            // Check to see the thresholds are non-zero to avoid logging errors with small
+            // slot times (e.g., during testing)
+            let crit_threshold = chain.slot_clock.unagg_attestation_production_delay();
+            let error_threshold = crit_threshold / 2;
+            if delay >= crit_threshold {
+                crit!(
+                    log,
+                    "Block was broadcast too late";
+                    "msg" => "system may be overloaded, block likely to be orphaned",
+                    "delay_ms" => delay.as_millis(),
+                    "slot" => block.slot(),
+                    "root" => ?root,
+                )
+            } else if delay >= error_threshold {
+                error!(
+                    log,
+                    "Block broadcast was delayed";
+                    "msg" => "system may be overloaded, block may be orphaned",
+                    "delay_ms" => delay.as_millis(),
+                    "slot" => block.slot(),
+                    "root" => ?root,
+                )
+            }
+
+            Ok(())
+        }
+        Err(e) => {
+            let msg = format!("{:?}", e);
+            error!(
+                log,
+                "Invalid block provided to HTTP API";
+                "reason" => &msg
+            );
+            Err(warp_utils::reject::broadcast_without_import(msg))
+        }
+    }
+}
+
+/// Handles a request from the HTTP API for blinded blocks. This converts blinded blocks into full
+/// blocks before publishing.
+pub async fn publish_blinded_block<T: BeaconChainTypes>(
+    block: SignedBeaconBlock<T::EthSpec, BlindedPayload<T::EthSpec>>,
+    chain: Arc<BeaconChain<T>>,
+    network_tx: &UnboundedSender<NetworkMessage<T::EthSpec>>,
+    log: Logger,
+) -> Result<(), Rejection> {
+    let full_block = reconstruct_block(chain.clone(), block, log.clone()).await?;
+    publish_block::<T>(Arc::new(full_block), chain, network_tx, log).await
+}
+
+/// Deconstruct the given blinded block, and construct a full block. This attempts to use the
+/// execution layer's payload cache, and if that misses, attempts a blind block proposal to retrieve
+/// the full payload.
+async fn reconstruct_block<T: BeaconChainTypes>(
+    chain: Arc<BeaconChain<T>>,
+    block: SignedBeaconBlock<T::EthSpec, BlindedPayload<T::EthSpec>>,
+    log: Logger,
+) -> Result<SignedBeaconBlock<T::EthSpec, FullPayload<T::EthSpec>>, Rejection> {
+    let full_payload = if let Ok(payload_header) = block.message().body().execution_payload() {
+        let el = chain.execution_layer.as_ref().ok_or_else(|| {
+            warp_utils::reject::custom_server_error("Missing execution layer".to_string())
+        })?;
+
+        // If the execution block hash is zero, use an empty payload.
+        let full_payload = if payload_header.block_hash() == ExecutionBlockHash::zero() {
+            ExecutionPayload::default()
+            // If we already have an execution payload with this transactions root cached, use it.
+        } else if let Some(cached_payload) =
+            el.get_payload_by_root(&payload_header.tree_hash_root())
+        {
+            info!(log, "Reconstructing a full block using a local payload"; "block_hash" => ?cached_payload.block_hash);
+            cached_payload
+            // Otherwise, this means we are attempting a blind block proposal.
+        } else {
+            let full_payload = el.propose_blinded_beacon_block(&block).await.map_err(|e| {
+                warp_utils::reject::custom_server_error(format!(
+                    "Blind block proposal failed: {:?}",
+                    e
+                ))
+            })?;
+            info!(log, "Successfully published a block to the builder network"; "block_hash" => ?full_payload.block_hash);
+            full_payload
+        };
+
+        Some(full_payload)
+    } else {
+        None
+    };
+
+    block.try_into_full_block(full_payload).ok_or_else(|| {
+        warp_utils::reject::custom_server_error("Unable to add payload to block".to_string())
+    })
+}

beacon_node/http_api/tests/common.rs

@@ -86,6 +86,16 @@ impl<E: EthSpec> InteractiveTester<E> {
 pub async fn create_api_server<T: BeaconChainTypes>(
     chain: Arc<BeaconChain<T>>,
     log: Logger,
+) -> ApiServer<T::EthSpec, impl Future<Output = ()>> {
+    // Get a random unused port.
+    let port = unused_port::unused_tcp_port().unwrap();
+    create_api_server_on_port(chain, log, port).await
+}
+
+pub async fn create_api_server_on_port<T: BeaconChainTypes>(
+    chain: Arc<BeaconChain<T>>,
+    log: Logger,
+    port: u16,
 ) -> ApiServer<T::EthSpec, impl Future<Output = ()>> {
     let (network_tx, network_rx) = mpsc::unbounded_channel();
 
@@ -129,7 +139,7 @@ pub async fn create_api_server<T: BeaconChainTypes>(
         config: Config {
             enabled: true,
             listen_addr: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
-            listen_port: 0,
+            listen_port: port,
             allow_origin: None,
             serve_legacy_spec: true,
             tls_config: None,

beacon_node/src/cli.rs

@@ -708,6 +708,46 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .default_value("250")
                 .takes_value(true)
         )
+        .arg(
+            Arg::with_name("builder-fallback-skips")
+                .long("builder-fallback-skips")
+                .help("If this node is proposing a block and has seen this number of skip slots \
+                        on the canonical chain in a row, it will NOT query any connected builders, \
+                        and will use the local execution engine for payload construction.")
+                .default_value("3")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("builder-fallback-skips-per-epoch")
+                .long("builder-fallback-skips-per-epoch")
+                .help("If this node is proposing a block and has seen this number of skip slots \
+                        on the canonical chain in the past `SLOTS_PER_EPOCH`, it will NOT query \
+                        any connected builders, and will use the local execution engine for \
+                        payload construction.")
+                .default_value("8")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("builder-fallback-epochs-since-finalization")
+                .long("builder-fallback-epochs-since-finalization")
+                .help("If this node is proposing a block and the chain has not finalized within \
+                        this number of epochs, it will NOT query any connected builders, \
+                        and will use the local execution engine for payload construction. Setting \
+                        this value to anything less than 2 will cause the node to NEVER query \
+                        connected builders. Setting it to 2 will cause this condition to be hit \
+                        if there are skips slots at the start of an epoch, right before this node \
+                        is set to propose.")
+                .default_value("3")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("builder-fallback-disable-checks")
+                .long("builder-fallback-disable-checks")
+                .help("This flag disables all checks related to chain health. This means the builder \
+                        API will always be used for payload construction, regardless of recent chain \
+                        conditions.")
+                .takes_value(false)
+        )
         .arg(
             Arg::with_name("count-unrealized")
                 .long("count-unrealized")

beacon_node/src/config.rs

@@ -634,6 +634,20 @@ pub fn get_config<E: EthSpec>(
         client_config.chain.count_unrealized = true;
     }
 
+    /*
+     * Builder fallback configs.
+     */
+    client_config.chain.builder_fallback_skips =
+        clap_utils::parse_required(cli_args, "builder-fallback-skips")?;
+    client_config.chain.builder_fallback_skips_per_epoch =
+        clap_utils::parse_required(cli_args, "builder-fallback-skips-per-epoch")?;
+    client_config
+        .chain
+        .builder_fallback_epochs_since_finalization =
+        clap_utils::parse_required(cli_args, "builder-fallback-epochs-since-finalization")?;
+    client_config.chain.builder_fallback_disable_checks =
+        cli_args.is_present("builder-fallback-disable-checks");
+
     Ok(client_config)
 }