Builder Specs v0.2.0 (#3134)

## Issue Addressed

https://github.com/sigp/lighthouse/issues/3091

Extends https://github.com/sigp/lighthouse/pull/3062, adding pre-bellatrix block support on blinded endpoints and allowing the normal proposal flow (local payload construction) on blinded endpoints. This resulted in better fallback logic because the VC will not have to switch endpoints on failure in the BN <> Builder API, the BN can just fallback immediately and without repeating block processing that it shouldn't need to. We can also keep VC fallback from the VC<>BN API's blinded endpoint to full endpoint.

## Proposed Changes

- Pre-bellatrix blocks on blinded endpoints
- Add a new `PayloadCache` to the execution layer
- Better fallback-from-builder logic

## Todos

- [x] Remove VC transition logic
- [x] Add logic to only enable builder flow after Merge transition finalization
- [x] Tests
- [x] Fix metrics
- [x] Rustdocs


Co-authored-by: Mac L <mjladson@pm.me>
Co-authored-by: realbigsean <sean@sigmaprime.io>

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -53,7 +53,9 @@ use crate::BeaconForkChoiceStore;
 use crate::BeaconSnapshot;
 use crate::{metrics, BeaconChainError};
 use eth2::types::{EventKind, SseBlock, SyncDuty};
-use execution_layer::{ExecutionLayer, PayloadAttributes, PayloadStatus};
+use execution_layer::{
+    BuilderParams, ChainHealth, ExecutionLayer, FailedCondition, PayloadAttributes, PayloadStatus,
+};
 use fork_choice::{
     AttestationFromBlock, ExecutionStatus, ForkChoice, ForkchoiceUpdateParameters,
     InvalidationOperation, PayloadVerificationStatus,
@@ -3315,10 +3317,21 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
         let proposer_index = state.get_beacon_proposer_index(state.slot(), &self.spec)? as u64;
 
-        let pubkey_opt = state
+        let pubkey = state
             .validators()
             .get(proposer_index as usize)
-            .map(|v| v.pubkey);
+            .map(|v| v.pubkey)
+            .ok_or(BlockProductionError::BeaconChain(
+                BeaconChainError::ValidatorIndexUnknown(proposer_index as usize),
+            ))?;
+
+        let builder_params = BuilderParams {
+            pubkey,
+            slot: state.slot(),
+            chain_health: self
+                .is_healthy()
+                .map_err(BlockProductionError::BeaconChain)?,
+        };
 
         // If required, start the process of loading an execution payload from the EL early. This
         // allows it to run concurrently with things like attestation packing.
@@ -3326,7 +3339,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             BeaconState::Base(_) | BeaconState::Altair(_) => None,
             BeaconState::Merge(_) => {
                 let prepare_payload_handle =
-                    get_execution_payload(self.clone(), &state, proposer_index, pubkey_opt)?;
+                    get_execution_payload(self.clone(), &state, proposer_index, builder_params)?;
                 Some(prepare_payload_handle)
             }
         };
@@ -4539,6 +4552,74 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             .map(|duration| (fork_name, duration))
     }
 
+    /// This method serves to get a sense of the current chain health. It is used in block proposal
+    /// to determine whether we should outsource payload production duties.
+    ///
+    /// Since we are likely calling this during the slot we are going to propose in, don't take into
+    /// account the current slot when accounting for skips.
+    pub fn is_healthy(&self) -> Result<ChainHealth, Error> {
+        // Check if the merge has been finalized.
+        if let Some(finalized_hash) = self
+            .canonical_head
+            .cached_head()
+            .forkchoice_update_parameters()
+            .finalized_hash
+        {
+            if ExecutionBlockHash::zero() == finalized_hash {
+                return Ok(ChainHealth::PreMerge);
+            }
+        } else {
+            return Ok(ChainHealth::PreMerge);
+        };
+
+        if self.config.builder_fallback_disable_checks {
+            return Ok(ChainHealth::Healthy);
+        }
+
+        let current_slot = self.slot()?;
+
+        // Check slots at the head of the chain.
+        let prev_slot = current_slot.saturating_sub(Slot::new(1));
+        let head_skips = prev_slot.saturating_sub(self.canonical_head.cached_head().head_slot());
+        let head_skips_check = head_skips.as_usize() <= self.config.builder_fallback_skips;
+
+        // Check if finalization is advancing.
+        let current_epoch = current_slot.epoch(T::EthSpec::slots_per_epoch());
+        let epochs_since_finalization = current_epoch.saturating_sub(
+            self.canonical_head
+                .cached_head()
+                .finalized_checkpoint()
+                .epoch,
+        );
+        let finalization_check = epochs_since_finalization.as_usize()
+            <= self.config.builder_fallback_epochs_since_finalization;
+
+        // Check skip slots in the last `SLOTS_PER_EPOCH`.
+        let start_slot = current_slot.saturating_sub(T::EthSpec::slots_per_epoch());
+        let mut epoch_skips = 0;
+        for slot in start_slot.as_u64()..current_slot.as_u64() {
+            if self
+                .block_root_at_slot_skips_none(Slot::new(slot))?
+                .is_none()
+            {
+                epoch_skips += 1;
+            }
+        }
+        let epoch_skips_check = epoch_skips <= self.config.builder_fallback_skips_per_epoch;
+
+        if !head_skips_check {
+            Ok(ChainHealth::Unhealthy(FailedCondition::Skips))
+        } else if !finalization_check {
+            Ok(ChainHealth::Unhealthy(
+                FailedCondition::EpochsSinceFinalization,
+            ))
+        } else if !epoch_skips_check {
+            Ok(ChainHealth::Unhealthy(FailedCondition::SkipsPerEpoch))
+        } else {
+            Ok(ChainHealth::Healthy)
+        }
+    }
+
     pub fn dump_as_dot<W: Write>(&self, output: &mut W) {
         let canonical_head_hash = self.canonical_head.cached_head().head_block_root();
         let mut visited: HashSet<Hash256> = HashSet::new();

beacon_node/beacon_chain/src/chain_config.rs

@@ -24,6 +24,16 @@ pub struct ChainConfig {
     ///
     /// If set to 0 then block proposal will not wait for fork choice at all.
     pub fork_choice_before_proposal_timeout_ms: u64,
+    /// Number of skip slots in a row before the BN refuses to use connected builders during payload construction.
+    pub builder_fallback_skips: usize,
+    /// Number of skip slots in the past `SLOTS_PER_EPOCH` before the BN refuses to use connected
+    /// builders during payload construction.
+    pub builder_fallback_skips_per_epoch: usize,
+    /// Number of epochs since finalization before the BN refuses to use connected builders during
+    /// payload construction.
+    pub builder_fallback_epochs_since_finalization: usize,
+    /// Whether any chain health checks should be considered when deciding whether to use the builder API.
+    pub builder_fallback_disable_checks: bool,
     pub count_unrealized: bool,
 }
 
@@ -36,6 +46,11 @@ impl Default for ChainConfig {
             enable_lock_timeouts: true,
             max_network_size: 10 * 1_048_576, // 10M
             fork_choice_before_proposal_timeout_ms: DEFAULT_FORK_CHOICE_BEFORE_PROPOSAL_TIMEOUT,
+            // Builder fallback configs that are set in `clap` will override these.
+            builder_fallback_skips: 3,
+            builder_fallback_skips_per_epoch: 8,
+            builder_fallback_epochs_since_finalization: 3,
+            builder_fallback_disable_checks: false,
             count_unrealized: false,
         }
     }

beacon_node/beacon_chain/src/errors.rs

@@ -138,6 +138,7 @@ pub enum BeaconChainError {
         new_slot: Slot,
     },
     AltairForkDisabled,
+    BuilderMissing,
     ExecutionLayerMissing,
     BlockVariantLacksExecutionPayload(Hash256),
     ExecutionLayerErrorPayloadReconstruction(ExecutionBlockHash, execution_layer::Error),

beacon_node/beacon_chain/src/execution_payload.rs

@@ -11,7 +11,7 @@ use crate::{
     BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, BlockProductionError,
     ExecutionPayloadError,
 };
-use execution_layer::PayloadStatus;
+use execution_layer::{BuilderParams, PayloadStatus};
 use fork_choice::{InvalidationOperation, PayloadVerificationStatus};
 use proto_array::{Block as ProtoBlock, ExecutionStatus};
 use slog::debug;
@@ -303,12 +303,11 @@ pub fn get_execution_payload<
     chain: Arc<BeaconChain<T>>,
     state: &BeaconState<T::EthSpec>,
     proposer_index: u64,
-    pubkey: Option<PublicKeyBytes>,
+    builder_params: BuilderParams,
 ) -> Result<PreparePayloadHandle<Payload>, BlockProductionError> {
     // Compute all required values from the `state` now to avoid needing to pass it into a spawned
     // task.
     let spec = &chain.spec;
-    let slot = state.slot();
     let current_epoch = state.current_epoch();
     let is_merge_transition_complete = is_merge_transition_complete(state);
     let timestamp = compute_timestamp_at_slot(state, spec).map_err(BeaconStateError::from)?;
@@ -325,13 +324,12 @@ pub fn get_execution_payload<
             async move {
                 prepare_execution_payload::<T, Payload>(
                     &chain,
-                    slot,
                     is_merge_transition_complete,
                     timestamp,
                     random,
                     proposer_index,
-                    pubkey,
                     latest_execution_payload_header_block_hash,
+                    builder_params,
                 )
                 .await
             },
@@ -359,19 +357,18 @@ pub fn get_execution_payload<
 #[allow(clippy::too_many_arguments)]
 pub async fn prepare_execution_payload<T, Payload>(
     chain: &Arc<BeaconChain<T>>,
-    slot: Slot,
     is_merge_transition_complete: bool,
     timestamp: u64,
     random: Hash256,
     proposer_index: u64,
-    pubkey: Option<PublicKeyBytes>,
     latest_execution_payload_header_block_hash: ExecutionBlockHash,
+    builder_params: BuilderParams,
 ) -> Result<Payload, BlockProductionError>
 where
     T: BeaconChainTypes,
     Payload: ExecPayload<T::EthSpec> + Default,
 {
-    let current_epoch = slot.epoch(T::EthSpec::slots_per_epoch());
+    let current_epoch = builder_params.slot.epoch(T::EthSpec::slots_per_epoch());
     let spec = &chain.spec;
     let execution_layer = chain
         .execution_layer
@@ -432,9 +429,9 @@ where
             timestamp,
             random,
             proposer_index,
-            pubkey,
-            slot,
             forkchoice_update_params,
+            builder_params,
+            &chain.spec,
         )
         .await
         .map_err(BlockProductionError::GetPayloadFailed)?;

beacon_node/beacon_chain/src/lib.rs

@@ -15,7 +15,7 @@ mod early_attester_cache;
 mod errors;
 pub mod eth1_chain;
 pub mod events;
-mod execution_payload;
+pub mod execution_payload;
 pub mod fork_choice_signal;
 pub mod fork_revert;
 mod head_tracker;

beacon_node/beacon_chain/src/test_utils.rs

@@ -14,7 +14,9 @@ use bls::get_withdrawal_credentials;
 use execution_layer::test_utils::DEFAULT_JWT_SECRET;
 use execution_layer::{
     auth::JwtKey,
-    test_utils::{ExecutionBlockGenerator, MockExecutionLayer, DEFAULT_TERMINAL_BLOCK},
+    test_utils::{
+        ExecutionBlockGenerator, MockExecutionLayer, TestingBuilder, DEFAULT_TERMINAL_BLOCK,
+    },
     ExecutionLayer,
 };
 use fork_choice::CountUnrealized;
@@ -154,6 +156,7 @@ pub struct Builder<T: BeaconChainTypes> {
     store_mutator: Option<BoxedMutator<T::EthSpec, T::HotStore, T::ColdStore>>,
     execution_layer: Option<ExecutionLayer<T::EthSpec>>,
     mock_execution_layer: Option<MockExecutionLayer<T::EthSpec>>,
+    mock_builder: Option<TestingBuilder<T::EthSpec>>,
     runtime: TestRuntime,
     log: Logger,
 }
@@ -285,6 +288,7 @@ where
             store_mutator: None,
             execution_layer: None,
             mock_execution_layer: None,
+            mock_builder: None,
             runtime,
             log,
         }
@@ -388,6 +392,38 @@ where
         self
     }
 
+    pub fn mock_execution_layer_with_builder(mut self, beacon_url: SensitiveUrl) -> Self {
+        // Get a random unused port
+        let port = unused_port::unused_tcp_port().unwrap();
+        let builder_url = SensitiveUrl::parse(format!("http://127.0.0.1:{port}").as_str()).unwrap();
+
+        let spec = self.spec.clone().expect("cannot build without spec");
+        let mock_el = MockExecutionLayer::new(
+            self.runtime.task_executor.clone(),
+            spec.terminal_total_difficulty,
+            DEFAULT_TERMINAL_BLOCK,
+            spec.terminal_block_hash,
+            spec.terminal_block_hash_activation_epoch,
+            Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()),
+            Some(builder_url.clone()),
+        )
+        .move_to_terminal_block();
+
+        let mock_el_url = SensitiveUrl::parse(mock_el.server.url().as_str()).unwrap();
+
+        self.mock_builder = Some(TestingBuilder::new(
+            mock_el_url,
+            builder_url,
+            beacon_url,
+            spec,
+            self.runtime.task_executor.clone(),
+        ));
+        self.execution_layer = Some(mock_el.el.clone());
+        self.mock_execution_layer = Some(mock_el);
+
+        self
+    }
+
     /// Instruct the mock execution engine to always return a "valid" response to any payload it is
     /// asked to execute.
     pub fn mock_execution_layer_all_payloads_valid(self) -> Self {
@@ -456,6 +492,7 @@ where
             shutdown_receiver: Arc::new(Mutex::new(shutdown_receiver)),
             runtime: self.runtime,
             mock_execution_layer: self.mock_execution_layer,
+            mock_builder: self.mock_builder.map(Arc::new),
             rng: make_rng(),
         }
     }
@@ -474,6 +511,7 @@ pub struct BeaconChainHarness<T: BeaconChainTypes> {
     pub runtime: TestRuntime,
 
     pub mock_execution_layer: Option<MockExecutionLayer<T::EthSpec>>,
+    pub mock_builder: Option<Arc<TestingBuilder<T::EthSpec>>>,
 
     pub rng: Mutex<StdRng>,
 }