Builder Specs v0.2.0 (#3134)

## Issue Addressed

https://github.com/sigp/lighthouse/issues/3091

Extends https://github.com/sigp/lighthouse/pull/3062, adding pre-bellatrix block support on blinded endpoints and allowing the normal proposal flow (local payload construction) on blinded endpoints. This resulted in better fallback logic because the VC will not have to switch endpoints on failure in the BN <> Builder API, the BN can just fallback immediately and without repeating block processing that it shouldn't need to. We can also keep VC fallback from the VC<>BN API's blinded endpoint to full endpoint.

## Proposed Changes

- Pre-bellatrix blocks on blinded endpoints
- Add a new `PayloadCache` to the execution layer
- Better fallback-from-builder logic

## Todos

- [x] Remove VC transition logic
- [x] Add logic to only enable builder flow after Merge transition finalization
- [x] Tests
- [x] Fix metrics
- [x] Rustdocs


Co-authored-by: Mac L <mjladson@pm.me>
Co-authored-by: realbigsean <sean@sigmaprime.io>

beacon_node/execution_layer/Cargo.toml

@@ -40,3 +40,7 @@ lazy_static = "1.4.0"
 ethers-core = { git = "https://github.com/gakonst/ethers-rs", rev = "02ad93a1cfb7b62eb051c77c61dc4c0218428e4a" }
 builder_client = { path = "../builder_client" }
 fork_choice = { path = "../../consensus/fork_choice" }
+mev-build-rs = {git = "https://github.com/ralexstokes/mev-rs", tag = "v0.2.0"}
+ethereum-consensus = {git = "https://github.com/ralexstokes/ethereum-consensus"}
+ssz-rs = {git = "https://github.com/ralexstokes/ssz-rs"}
+

beacon_node/execution_layer/src/engine_api/json_structures.rs

@@ -1,9 +1,6 @@
 use super::*;
 use serde::{Deserialize, Serialize};
-use types::{
-    EthSpec, ExecutionBlockHash, ExecutionPayloadHeader, FixedVector, Transaction, Unsigned,
-    VariableList,
-};
+use types::{EthSpec, ExecutionBlockHash, FixedVector, Transaction, Unsigned, VariableList};
 
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -432,59 +429,6 @@ impl From<ForkchoiceUpdatedResponse> for JsonForkchoiceUpdatedV1Response {
     }
 }
 
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
-pub enum JsonProposeBlindedBlockResponseStatus {
-    Valid,
-    Invalid,
-    Syncing,
-}
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-#[serde(bound = "E: EthSpec")]
-pub struct JsonProposeBlindedBlockResponse<E: EthSpec> {
-    pub result: ExecutionPayload<E>,
-    pub error: Option<String>,
-}
-
-impl<E: EthSpec> From<JsonProposeBlindedBlockResponse<E>> for ExecutionPayload<E> {
-    fn from(j: JsonProposeBlindedBlockResponse<E>) -> Self {
-        let JsonProposeBlindedBlockResponse { result, error: _ } = j;
-        result
-    }
-}
-
-impl From<JsonProposeBlindedBlockResponseStatus> for ProposeBlindedBlockResponseStatus {
-    fn from(j: JsonProposeBlindedBlockResponseStatus) -> Self {
-        match j {
-            JsonProposeBlindedBlockResponseStatus::Valid => {
-                ProposeBlindedBlockResponseStatus::Valid
-            }
-            JsonProposeBlindedBlockResponseStatus::Invalid => {
-                ProposeBlindedBlockResponseStatus::Invalid
-            }
-            JsonProposeBlindedBlockResponseStatus::Syncing => {
-                ProposeBlindedBlockResponseStatus::Syncing
-            }
-        }
-    }
-}
-impl From<ProposeBlindedBlockResponseStatus> for JsonProposeBlindedBlockResponseStatus {
-    fn from(f: ProposeBlindedBlockResponseStatus) -> Self {
-        match f {
-            ProposeBlindedBlockResponseStatus::Valid => {
-                JsonProposeBlindedBlockResponseStatus::Valid
-            }
-            ProposeBlindedBlockResponseStatus::Invalid => {
-                JsonProposeBlindedBlockResponseStatus::Invalid
-            }
-            ProposeBlindedBlockResponseStatus::Syncing => {
-                JsonProposeBlindedBlockResponseStatus::Syncing
-            }
-        }
-    }
-}
-
 #[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct TransitionConfigurationV1 {

beacon_node/execution_layer/src/lib.rs

@@ -4,6 +4,7 @@
 //! This crate only provides useful functionality for "The Merge", it does not provide any of the
 //! deposit-contract functionality that the `beacon_node/eth1` crate already provides.
 
+use crate::payload_cache::PayloadCache;
 use auth::{strip_prefix, Auth, JwtKey};
 use builder_client::BuilderHttpClient;
 use engine_api::Error as ApiError;
@@ -31,13 +32,14 @@ use tokio::{
     time::sleep,
 };
 use types::{
-    BlindedPayload, BlockType, ChainSpec, Epoch, ExecPayload, ExecutionBlockHash,
+    BlindedPayload, BlockType, ChainSpec, Epoch, ExecPayload, ExecutionBlockHash, ForkName,
     ProposerPreparationData, PublicKeyBytes, SignedBeaconBlock, Slot,
 };
 
 mod engine_api;
 mod engines;
 mod metrics;
+pub mod payload_cache;
 mod payload_status;
 pub mod test_utils;
 
@@ -69,6 +71,7 @@ pub enum Error {
     NoPayloadBuilder,
     ApiError(ApiError),
     Builder(builder_client::Error),
+    NoHeaderFromBuilder,
     EngineError(Box<EngineError>),
     NotSynced,
     ShuttingDown,
@@ -101,6 +104,26 @@ pub struct Proposer {
     payload_attributes: PayloadAttributes,
 }
 
+/// Information from the beacon chain that is necessary for querying the builder API.
+pub struct BuilderParams {
+    pub pubkey: PublicKeyBytes,
+    pub slot: Slot,
+    pub chain_health: ChainHealth,
+}
+
+pub enum ChainHealth {
+    Healthy,
+    Unhealthy(FailedCondition),
+    PreMerge,
+}
+
+#[derive(Debug)]
+pub enum FailedCondition {
+    Skips,
+    SkipsPerEpoch,
+    EpochsSinceFinalization,
+}
+
 struct Inner<E: EthSpec> {
     engine: Arc<Engine>,
     builder: Option<BuilderHttpClient>,
@@ -110,7 +133,7 @@ struct Inner<E: EthSpec> {
     execution_blocks: Mutex<LruCache<ExecutionBlockHash, ExecutionBlock>>,
     proposers: RwLock<HashMap<ProposerKey, Proposer>>,
     executor: TaskExecutor,
-    phantom: std::marker::PhantomData<E>,
+    payload_cache: PayloadCache<E>,
     log: Logger,
 }
 
@@ -212,7 +235,7 @@ impl<T: EthSpec> ExecutionLayer<T> {
             proposers: RwLock::new(HashMap::new()),
             execution_blocks: Mutex::new(LruCache::new(EXECUTION_BLOCKS_LRU_CACHE_SIZE)),
             executor,
-            phantom: std::marker::PhantomData,
+            payload_cache: PayloadCache::default(),
             log,
         };
 
@@ -231,6 +254,16 @@ impl<T: EthSpec> ExecutionLayer<T> {
         &self.inner.builder
     }
 
+    /// Cache a full payload, keyed on the `tree_hash_root` of its `transactions` field.
+    fn cache_payload(&self, payload: &ExecutionPayload<T>) -> Option<ExecutionPayload<T>> {
+        self.inner.payload_cache.put(payload.clone())
+    }
+
+    /// Attempt to retrieve a full payload from the payload cache by the `transactions_root`.
+    pub fn get_payload_by_root(&self, root: &Hash256) -> Option<ExecutionPayload<T>> {
+        self.inner.payload_cache.pop(root)
+    }
+
     pub fn executor(&self) -> &TaskExecutor {
         &self.inner.executor
     }
@@ -487,9 +520,9 @@ impl<T: EthSpec> ExecutionLayer<T> {
         timestamp: u64,
         prev_randao: Hash256,
         proposer_index: u64,
-        pubkey: Option<PublicKeyBytes>,
-        slot: Slot,
         forkchoice_update_params: ForkchoiceUpdateParameters,
+        builder_params: BuilderParams,
+        spec: &ChainSpec,
     ) -> Result<Payload, Error> {
         let suggested_fee_recipient = self.get_suggested_fee_recipient(proposer_index).await;
 
@@ -504,9 +537,9 @@ impl<T: EthSpec> ExecutionLayer<T> {
                     timestamp,
                     prev_randao,
                     suggested_fee_recipient,
-                    pubkey,
-                    slot,
                     forkchoice_update_params,
+                    builder_params,
+                    spec,
                 )
                 .await
             }
@@ -534,36 +567,137 @@ impl<T: EthSpec> ExecutionLayer<T> {
         timestamp: u64,
         prev_randao: Hash256,
         suggested_fee_recipient: Address,
-        pubkey_opt: Option<PublicKeyBytes>,
-        slot: Slot,
         forkchoice_update_params: ForkchoiceUpdateParameters,
+        builder_params: BuilderParams,
+        spec: &ChainSpec,
     ) -> Result<Payload, Error> {
-        //FIXME(sean) fallback logic included in PR #3134
+        if let Some(builder) = self.builder() {
+            let slot = builder_params.slot;
+            let pubkey = builder_params.pubkey;
 
-        // Don't attempt to outsource payload construction until after the merge transition has been
-        // finalized. We want to be conservative with payload construction until then.
-        if let (Some(builder), Some(pubkey)) = (self.builder(), pubkey_opt) {
-            if forkchoice_update_params
-                .finalized_hash
-                .map_or(false, |finalized_block_hash| {
-                    finalized_block_hash != ExecutionBlockHash::zero()
-                })
-            {
-                info!(
-                    self.log(),
-                    "Requesting blinded header from connected builder";
-                    "slot" => ?slot,
-                    "pubkey" => ?pubkey,
-                    "parent_hash" => ?parent_hash,
-                );
-                return builder
-                    .get_builder_header::<T, Payload>(slot, parent_hash, &pubkey)
-                    .await
-                    .map(|d| d.data.message.header)
-                    .map_err(Error::Builder);
+            match builder_params.chain_health {
+                ChainHealth::Healthy => {
+                    info!(
+                        self.log(),
+                        "Requesting blinded header from connected builder";
+                        "slot" => ?slot,
+                        "pubkey" => ?pubkey,
+                        "parent_hash" => ?parent_hash,
+                    );
+                    let (relay_result, local_result) = tokio::join!(
+                        builder.get_builder_header::<T, Payload>(slot, parent_hash, &pubkey),
+                        self.get_full_payload_caching(
+                            parent_hash,
+                            timestamp,
+                            prev_randao,
+                            suggested_fee_recipient,
+                            forkchoice_update_params,
+                        )
+                    );
+
+                    return match (relay_result, local_result) {
+                        (Err(e), Ok(local)) => {
+                            warn!(
+                                self.log(),
+                                "Unable to retrieve a payload from a connected \
+                                builder, falling back to the local execution client: {e:?}"
+                            );
+                            Ok(local)
+                        }
+                        (Ok(None), Ok(local)) => {
+                            warn!(
+                                self.log(),
+                                "No payload provided by connected builder. \
+                                Attempting to propose through local execution engine"
+                            );
+                            Ok(local)
+                        }
+                        (Ok(Some(relay)), Ok(local)) => {
+                            let is_signature_valid = relay.data.verify_signature(spec);
+                            let header = relay.data.message.header;
+
+                            info!(
+                                self.log(),
+                                "Received a payload header from the connected builder";
+                                "block_hash" => ?header.block_hash(),
+                            );
+
+                            if header.parent_hash() != parent_hash {
+                                warn!(
+                                    self.log(),
+                                    "Invalid parent hash from connected builder, \
+                                    falling back to local execution engine."
+                                );
+                                Ok(local)
+                            } else if header.prev_randao() != prev_randao {
+                                warn!(
+                                    self.log(),
+                                    "Invalid prev randao from connected builder, \
+                                    falling back to local execution engine."
+                                );
+                                Ok(local)
+                            } else if header.timestamp() != local.timestamp() {
+                                warn!(
+                                    self.log(),
+                                    "Invalid timestamp from connected builder, \
+                                    falling back to local execution engine."
+                                );
+                                Ok(local)
+                            } else if header.block_number() != local.block_number() {
+                                warn!(
+                                    self.log(),
+                                    "Invalid block number from connected builder, \
+                                    falling back to local execution engine."
+                                );
+                                Ok(local)
+                            } else if !matches!(relay.version, Some(ForkName::Merge)) {
+                                // Once fork information is added to the payload, we will need to
+                                // check that the local and relay payloads match. At this point, if
+                                // we are requesting a payload at all, we have to assume this is
+                                // the Bellatrix fork.
+                                warn!(
+                                    self.log(),
+                                    "Invalid fork from connected builder, falling \
+                                    back to local execution engine."
+                                );
+                                Ok(local)
+                            } else if !is_signature_valid {
+                                let pubkey_bytes = relay.data.message.pubkey;
+                                warn!(self.log(), "Invalid signature for pubkey {pubkey_bytes} on \
+                                    bid from connected builder, falling back to local execution engine.");
+                                Ok(local)
+                            } else {
+                                if header.fee_recipient() != suggested_fee_recipient {
+                                    info!(
+                                        self.log(),
+                                        "Fee recipient from connected builder does \
+                                        not match, using it anyways."
+                                    );
+                                }
+                                Ok(header)
+                            }
+                        }
+                        (relay_result, Err(local_error)) => {
+                            warn!(self.log(), "Failure from local execution engine. Attempting to \
+                                propose through connected builder"; "error" => ?local_error);
+                            relay_result
+                                .map_err(Error::Builder)?
+                                .ok_or(Error::NoHeaderFromBuilder)
+                                .map(|d| d.data.message.header)
+                        }
+                    };
+                }
+                ChainHealth::Unhealthy(condition) => {
+                    info!(self.log(), "Due to poor chain health the local execution engine will be used \
+                                        for payload construction. To adjust chain health conditions \
+                                        Use `builder-fallback` prefixed flags";
+                        "failed_condition" => ?condition)
+                }
+                // Intentional no-op, so we never attempt builder API proposals pre-merge.
+                ChainHealth::PreMerge => (),
             }
         }
-        self.get_full_payload::<Payload>(
+        self.get_full_payload_caching(
             parent_hash,
             timestamp,
             prev_randao,
@@ -593,6 +727,26 @@ impl<T: EthSpec> ExecutionLayer<T> {
         .await
     }
 
+    /// Get a full payload and cache its result in the execution layer's payload cache.
+    async fn get_full_payload_caching<Payload: ExecPayload<T>>(
+        &self,
+        parent_hash: ExecutionBlockHash,
+        timestamp: u64,
+        prev_randao: Hash256,
+        suggested_fee_recipient: Address,
+        forkchoice_update_params: ForkchoiceUpdateParameters,
+    ) -> Result<Payload, Error> {
+        self.get_full_payload_with(
+            parent_hash,
+            timestamp,
+            prev_randao,
+            suggested_fee_recipient,
+            forkchoice_update_params,
+            Self::cache_payload,
+        )
+        .await
+    }
+
     async fn get_full_payload_with<Payload: ExecPayload<T>>(
         &self,
         parent_hash: ExecutionBlockHash,

beacon_node/execution_layer/src/payload_cache.rs

@@ -0,0 +1,33 @@
+use lru::LruCache;
+use parking_lot::Mutex;
+use tree_hash::TreeHash;
+use types::{EthSpec, ExecutionPayload, Hash256};
+
+pub const DEFAULT_PAYLOAD_CACHE_SIZE: usize = 10;
+
+/// A cache mapping execution payloads by tree hash roots.
+pub struct PayloadCache<T: EthSpec> {
+    payloads: Mutex<LruCache<PayloadCacheId, ExecutionPayload<T>>>,
+}
+
+#[derive(Hash, PartialEq, Eq)]
+struct PayloadCacheId(Hash256);
+
+impl<T: EthSpec> Default for PayloadCache<T> {
+    fn default() -> Self {
+        PayloadCache {
+            payloads: Mutex::new(LruCache::new(DEFAULT_PAYLOAD_CACHE_SIZE)),
+        }
+    }
+}
+
+impl<T: EthSpec> PayloadCache<T> {
+    pub fn put(&self, payload: ExecutionPayload<T>) -> Option<ExecutionPayload<T>> {
+        let root = payload.tree_hash_root();
+        self.payloads.lock().put(PayloadCacheId(root), payload)
+    }
+
+    pub fn pop(&self, root: &Hash256) -> Option<ExecutionPayload<T>> {
+        self.payloads.lock().pop(&PayloadCacheId(*root))
+    }
+}

beacon_node/execution_layer/src/test_utils/mock_builder.rs

@@ -0,0 +1,383 @@
+use crate::test_utils::DEFAULT_JWT_SECRET;
+use crate::{Config, ExecutionLayer, PayloadAttributes};
+use async_trait::async_trait;
+use eth2::types::{BlockId, StateId, ValidatorId};
+use eth2::{BeaconNodeHttpClient, Timeouts};
+use ethereum_consensus::crypto::{SecretKey, Signature};
+use ethereum_consensus::primitives::BlsPublicKey;
+pub use ethereum_consensus::state_transition::Context;
+use fork_choice::ForkchoiceUpdateParameters;
+use mev_build_rs::{
+    sign_builder_message, verify_signed_builder_message, BidRequest, BlindedBlockProviderError,
+    BlindedBlockProviderServer, BuilderBid, ExecutionPayload as ServerPayload,
+    ExecutionPayloadHeader as ServerPayloadHeader, SignedBlindedBeaconBlock, SignedBuilderBid,
+    SignedValidatorRegistration,
+};
+use parking_lot::RwLock;
+use sensitive_url::SensitiveUrl;
+use ssz::{Decode, Encode};
+use ssz_rs::{Merkleized, SimpleSerialize};
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::net::Ipv4Addr;
+use std::sync::Arc;
+use std::time::Duration;
+use task_executor::TaskExecutor;
+use tempfile::NamedTempFile;
+use tree_hash::TreeHash;
+use types::{
+    Address, BeaconState, BlindedPayload, ChainSpec, EthSpec, ExecPayload, Hash256, Slot, Uint256,
+};
+
+#[derive(Clone)]
+pub enum Operation {
+    FeeRecipient(Address),
+    GasLimit(usize),
+    Value(usize),
+    ParentHash(Hash256),
+    PrevRandao(Hash256),
+    BlockNumber(usize),
+    Timestamp(usize),
+}
+
+impl Operation {
+    fn apply(self, bid: &mut BuilderBid) -> Result<(), BlindedBlockProviderError> {
+        match self {
+            Operation::FeeRecipient(fee_recipient) => {
+                bid.header.fee_recipient = to_ssz_rs(&fee_recipient)?
+            }
+            Operation::GasLimit(gas_limit) => bid.header.gas_limit = gas_limit as u64,
+            Operation::Value(value) => bid.value = to_ssz_rs(&Uint256::from(value))?,
+            Operation::ParentHash(parent_hash) => bid.header.parent_hash = to_ssz_rs(&parent_hash)?,
+            Operation::PrevRandao(prev_randao) => bid.header.prev_randao = to_ssz_rs(&prev_randao)?,
+            Operation::BlockNumber(block_number) => bid.header.block_number = block_number as u64,
+            Operation::Timestamp(timestamp) => bid.header.timestamp = timestamp as u64,
+        }
+        Ok(())
+    }
+}
+
+pub struct TestingBuilder<E: EthSpec> {
+    server: BlindedBlockProviderServer<MockBuilder<E>>,
+    pub builder: MockBuilder<E>,
+}
+
+impl<E: EthSpec> TestingBuilder<E> {
+    pub fn new(
+        mock_el_url: SensitiveUrl,
+        builder_url: SensitiveUrl,
+        beacon_url: SensitiveUrl,
+        spec: ChainSpec,
+        executor: TaskExecutor,
+    ) -> Self {
+        let file = NamedTempFile::new().unwrap();
+        let path = file.path().into();
+        std::fs::write(&path, hex::encode(DEFAULT_JWT_SECRET)).unwrap();
+
+        // This EL should not talk to a builder
+        let config = Config {
+            execution_endpoints: vec![mock_el_url],
+            secret_files: vec![path],
+            suggested_fee_recipient: None,
+            ..Default::default()
+        };
+
+        let el =
+            ExecutionLayer::from_config(config, executor.clone(), executor.log().clone()).unwrap();
+
+        // This should probably be done for all fields, we only update ones we are testing with so far.
+        let mut context = Context::for_mainnet();
+        context.terminal_total_difficulty = to_ssz_rs(&spec.terminal_total_difficulty).unwrap();
+        context.terminal_block_hash = to_ssz_rs(&spec.terminal_block_hash).unwrap();
+        context.terminal_block_hash_activation_epoch =
+            to_ssz_rs(&spec.terminal_block_hash_activation_epoch).unwrap();
+
+        let builder = MockBuilder::new(
+            el,
+            BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(Duration::from_secs(1))),
+            spec,
+            context,
+        );
+        let port = builder_url.full.port().unwrap();
+        let host: Ipv4Addr = builder_url
+            .full
+            .host_str()
+            .unwrap()
+            .to_string()
+            .parse()
+            .unwrap();
+        let server = BlindedBlockProviderServer::new(host, port, builder.clone());
+        Self { server, builder }
+    }
+
+    pub async fn run(&self) {
+        self.server.run().await
+    }
+}
+
+#[derive(Clone)]
+pub struct MockBuilder<E: EthSpec> {
+    el: ExecutionLayer<E>,
+    beacon_client: BeaconNodeHttpClient,
+    spec: ChainSpec,
+    context: Arc<Context>,
+    val_registration_cache: Arc<RwLock<HashMap<BlsPublicKey, SignedValidatorRegistration>>>,
+    builder_sk: SecretKey,
+    operations: Arc<RwLock<Vec<Operation>>>,
+    invalidate_signatures: Arc<RwLock<bool>>,
+}
+
+impl<E: EthSpec> MockBuilder<E> {
+    pub fn new(
+        el: ExecutionLayer<E>,
+        beacon_client: BeaconNodeHttpClient,
+        spec: ChainSpec,
+        context: Context,
+    ) -> Self {
+        let sk = SecretKey::random(&mut rand::thread_rng()).unwrap();
+        Self {
+            el,
+            beacon_client,
+            // Should keep spec and context consistent somehow
+            spec,
+            context: Arc::new(context),
+            val_registration_cache: Arc::new(RwLock::new(HashMap::new())),
+            builder_sk: sk,
+            operations: Arc::new(RwLock::new(vec![])),
+            invalidate_signatures: Arc::new(RwLock::new(false)),
+        }
+    }
+
+    pub fn add_operation(&self, op: Operation) {
+        self.operations.write().push(op);
+    }
+
+    pub fn invalid_signatures(&self) {
+        *self.invalidate_signatures.write() = true;
+    }
+
+    pub fn valid_signatures(&mut self) {
+        *self.invalidate_signatures.write() = false;
+    }
+
+    fn apply_operations(&self, bid: &mut BuilderBid) -> Result<(), BlindedBlockProviderError> {
+        let mut guard = self.operations.write();
+        while let Some(op) = guard.pop() {
+            op.apply(bid)?;
+        }
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl<E: EthSpec> mev_build_rs::BlindedBlockProvider for MockBuilder<E> {
+    async fn register_validators(
+        &self,
+        registrations: &mut [SignedValidatorRegistration],
+    ) -> Result<(), BlindedBlockProviderError> {
+        for registration in registrations {
+            let pubkey = registration.message.public_key.clone();
+            let message = &mut registration.message;
+            verify_signed_builder_message(
+                message,
+                &registration.signature,
+                &pubkey,
+                &self.context,
+            )?;
+            self.val_registration_cache.write().insert(
+                registration.message.public_key.clone(),
+                registration.clone(),
+            );
+        }
+
+        Ok(())
+    }
+
+    async fn fetch_best_bid(
+        &self,
+        bid_request: &BidRequest,
+    ) -> Result<SignedBuilderBid, BlindedBlockProviderError> {
+        let slot = Slot::new(bid_request.slot);
+        let signed_cached_data = self
+            .val_registration_cache
+            .read()
+            .get(&bid_request.public_key)
+            .ok_or_else(|| convert_err("missing registration"))?
+            .clone();
+        let cached_data = signed_cached_data.message;
+
+        let head = self
+            .beacon_client
+            .get_beacon_blocks::<E>(BlockId::Head)
+            .await
+            .map_err(convert_err)?
+            .ok_or_else(|| convert_err("missing head block"))?;
+
+        let block = head.data.message_merge().map_err(convert_err)?;
+        let head_block_root = block.tree_hash_root();
+        let head_execution_hash = block.body.execution_payload.execution_payload.block_hash;
+        if head_execution_hash != from_ssz_rs(&bid_request.parent_hash)? {
+            return Err(BlindedBlockProviderError::Custom(format!(
+                "head mismatch: {} {}",
+                head_execution_hash, bid_request.parent_hash
+            )));
+        }
+
+        let finalized_execution_hash = self
+            .beacon_client
+            .get_beacon_blocks::<E>(BlockId::Finalized)
+            .await
+            .map_err(convert_err)?
+            .ok_or_else(|| convert_err("missing finalized block"))?
+            .data
+            .message_merge()
+            .map_err(convert_err)?
+            .body
+            .execution_payload
+            .execution_payload
+            .block_hash;
+
+        let justified_execution_hash = self
+            .beacon_client
+            .get_beacon_blocks::<E>(BlockId::Justified)
+            .await
+            .map_err(convert_err)?
+            .ok_or_else(|| convert_err("missing finalized block"))?
+            .data
+            .message_merge()
+            .map_err(convert_err)?
+            .body
+            .execution_payload
+            .execution_payload
+            .block_hash;
+
+        let val_index = self
+            .beacon_client
+            .get_beacon_states_validator_id(
+                StateId::Head,
+                &ValidatorId::PublicKey(from_ssz_rs(&cached_data.public_key)?),
+            )
+            .await
+            .map_err(convert_err)?
+            .ok_or_else(|| convert_err("missing validator from state"))?
+            .data
+            .index;
+        let fee_recipient = from_ssz_rs(&cached_data.fee_recipient)?;
+        let slots_since_genesis = slot.as_u64() - self.spec.genesis_slot.as_u64();
+
+        let genesis_time = self
+            .beacon_client
+            .get_beacon_genesis()
+            .await
+            .map_err(convert_err)?
+            .data
+            .genesis_time;
+        let timestamp = (slots_since_genesis * self.spec.seconds_per_slot) + genesis_time;
+
+        let head_state: BeaconState<E> = self
+            .beacon_client
+            .get_debug_beacon_states(StateId::Head)
+            .await
+            .map_err(convert_err)?
+            .ok_or_else(|| BlindedBlockProviderError::Custom("missing head state".to_string()))?
+            .data;
+        let prev_randao = head_state
+            .get_randao_mix(head_state.current_epoch())
+            .map_err(convert_err)?;
+
+        let payload_attributes = PayloadAttributes {
+            timestamp,
+            prev_randao: *prev_randao,
+            suggested_fee_recipient: fee_recipient,
+        };
+
+        self.el
+            .insert_proposer(slot, head_block_root, val_index, payload_attributes)
+            .await;
+
+        let forkchoice_update_params = ForkchoiceUpdateParameters {
+            head_root: Hash256::zero(),
+            head_hash: None,
+            justified_hash: Some(justified_execution_hash),
+            finalized_hash: Some(finalized_execution_hash),
+        };
+
+        let payload = self
+            .el
+            .get_full_payload_caching::<BlindedPayload<E>>(
+                head_execution_hash,
+                timestamp,
+                *prev_randao,
+                fee_recipient,
+                forkchoice_update_params,
+            )
+            .await
+            .map_err(convert_err)?
+            .to_execution_payload_header();
+
+        let json_payload = serde_json::to_string(&payload).map_err(convert_err)?;
+        let mut header: ServerPayloadHeader =
+            serde_json::from_str(json_payload.as_str()).map_err(convert_err)?;
+
+        header.gas_limit = cached_data.gas_limit;
+
+        let mut message = BuilderBid {
+            header,
+            value: ssz_rs::U256::default(),
+            public_key: self.builder_sk.public_key(),
+        };
+
+        self.apply_operations(&mut message)?;
+
+        let mut signature =
+            sign_builder_message(&mut message, &self.builder_sk, self.context.as_ref())?;
+
+        if *self.invalidate_signatures.read() {
+            signature = Signature::default();
+        }
+
+        let signed_bid = SignedBuilderBid { message, signature };
+        Ok(signed_bid)
+    }
+
+    async fn open_bid(
+        &self,
+        signed_block: &mut SignedBlindedBeaconBlock,
+    ) -> Result<ServerPayload, BlindedBlockProviderError> {
+        let payload = self
+            .el
+            .get_payload_by_root(&from_ssz_rs(
+                &signed_block
+                    .message
+                    .body
+                    .execution_payload_header
+                    .hash_tree_root()
+                    .map_err(convert_err)?,
+            )?)
+            .ok_or_else(|| convert_err("missing payload for tx root"))?;
+
+        let json_payload = serde_json::to_string(&payload).map_err(convert_err)?;
+        serde_json::from_str(json_payload.as_str()).map_err(convert_err)
+    }
+}
+
+pub fn from_ssz_rs<T: SimpleSerialize, U: Decode>(
+    ssz_rs_data: &T,
+) -> Result<U, BlindedBlockProviderError> {
+    U::from_ssz_bytes(
+        ssz_rs::serialize(ssz_rs_data)
+            .map_err(convert_err)?
+            .as_ref(),
+    )
+    .map_err(convert_err)
+}
+
+pub fn to_ssz_rs<T: Encode, U: SimpleSerialize>(
+    ssz_data: &T,
+) -> Result<U, BlindedBlockProviderError> {
+    ssz_rs::deserialize::<U>(&ssz_data.as_ssz_bytes()).map_err(convert_err)
+}
+
+fn convert_err<E: Debug>(e: E) -> BlindedBlockProviderError {
+    BlindedBlockProviderError::Custom(format!("{e:?}"))
+}

beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs

@@ -7,6 +7,7 @@ use crate::{
 use sensitive_url::SensitiveUrl;
 use task_executor::TaskExecutor;
 use tempfile::NamedTempFile;
+use tree_hash::TreeHash;
 use types::{Address, ChainSpec, Epoch, EthSpec, FullPayload, Hash256, Uint256};
 
 pub struct MockExecutionLayer<T: EthSpec> {
@@ -124,6 +125,11 @@ impl<T: EthSpec> MockExecutionLayer<T> {
             .unwrap();
 
         let validator_index = 0;
+        let builder_params = BuilderParams {
+            pubkey: PublicKeyBytes::empty(),
+            slot,
+            chain_health: ChainHealth::Healthy,
+        };
         let payload = self
             .el
             .get_payload::<FullPayload<T>>(
@@ -131,9 +137,9 @@ impl<T: EthSpec> MockExecutionLayer<T> {
                 timestamp,
                 prev_randao,
                 validator_index,
-                None,
-                slot,
                 forkchoice_update_params,
+                builder_params,
+                &self.spec,
             )
             .await
             .unwrap()
@@ -144,6 +150,43 @@ impl<T: EthSpec> MockExecutionLayer<T> {
         assert_eq!(payload.timestamp, timestamp);
         assert_eq!(payload.prev_randao, prev_randao);
 
+        // Ensure the payload cache is empty.
+        assert!(self
+            .el
+            .get_payload_by_root(&payload.tree_hash_root())
+            .is_none());
+        let builder_params = BuilderParams {
+            pubkey: PublicKeyBytes::empty(),
+            slot,
+            chain_health: ChainHealth::Healthy,
+        };
+        let payload_header = self
+            .el
+            .get_payload::<BlindedPayload<T>>(
+                parent_hash,
+                timestamp,
+                prev_randao,
+                validator_index,
+                forkchoice_update_params,
+                builder_params,
+                &self.spec,
+            )
+            .await
+            .unwrap()
+            .execution_payload_header;
+        assert_eq!(payload_header.block_hash, block_hash);
+        assert_eq!(payload_header.parent_hash, parent_hash);
+        assert_eq!(payload_header.block_number, block_number);
+        assert_eq!(payload_header.timestamp, timestamp);
+        assert_eq!(payload_header.prev_randao, prev_randao);
+
+        // Ensure the payload cache has the correct payload.
+        assert_eq!(
+            self.el
+                .get_payload_by_root(&payload_header.tree_hash_root()),
+            Some(payload.clone())
+        );
+
         let status = self.el.notify_new_payload(&payload).await.unwrap();
         assert_eq!(status, PayloadStatus::Valid);
 

beacon_node/execution_layer/src/test_utils/mod.rs

@@ -22,6 +22,7 @@ use types::{EthSpec, ExecutionBlockHash, Uint256};
 use warp::{http::StatusCode, Filter, Rejection};
 
 pub use execution_block_generator::{generate_pow_block, Block, ExecutionBlockGenerator};
+pub use mock_builder::{Context as MockBuilderContext, MockBuilder, Operation, TestingBuilder};
 pub use mock_execution_layer::MockExecutionLayer;
 
 pub const DEFAULT_TERMINAL_DIFFICULTY: u64 = 6400;
@@ -30,6 +31,7 @@ pub const DEFAULT_JWT_SECRET: [u8; 32] = [42; 32];
 
 mod execution_block_generator;
 mod handle_rpc;
+mod mock_builder;
 mod mock_execution_layer;
 
 /// Configuration for the MockExecutionLayer.