v2.2.1 (#3149)

## Issue Addressed

Addresses sync stalls on v2.2.0 (i.e. https://github.com/sigp/lighthouse/issues/3147).

## Additional Info

I've avoided doing a full `cargo update` because I noticed there's a new patch version of libp2p and thought it could do with some more testing.



Co-authored-by: Paul Hauner <paul@paulhauner.com>

Cargo.lock

@@ -323,7 +323,7 @@ dependencies = [
 
 [[package]]
 name = "beacon_node"
-version = "2.2.0"
+version = "2.2.1"
 dependencies = [
  "beacon_chain",
  "clap",
@@ -467,7 +467,7 @@ dependencies = [
 
 [[package]]
 name = "boot_node"
-version = "2.2.0"
+version = "2.2.1"
 dependencies = [
  "beacon_node",
  "clap",
@@ -2685,7 +2685,7 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "lcli"
-version = "2.2.0"
+version = "2.2.1"
 dependencies = [
  "account_utils",
  "bls",
@@ -3178,7 +3178,7 @@ dependencies = [
 
 [[package]]
 name = "lighthouse"
-version = "2.2.0"
+version = "2.2.1"
 dependencies = [
  "account_manager",
  "account_utils",

beacon_node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "beacon_node"
-version = "2.2.0"
+version = "2.2.1"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com"]
 edition = "2021"
 

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -3481,9 +3481,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             .beacon_state
             .attester_shuffling_decision_root(self.genesis_block_root, RelativeEpoch::Current);
 
-        // Used later for the execution engine.
-        let is_merge_transition_complete = is_merge_transition_complete(&new_head.beacon_state);
-
         drop(lag_timer);
 
         // Clear the early attester cache in case it conflicts with `self.canonical_head`.
@@ -3690,45 +3687,50 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             }
         }
 
-        // If this is a post-merge block, update the execution layer.
-        if is_merge_transition_complete {
-            let current_slot = self.slot()?;
+        // Update the execution layer.
+        if let Err(e) = self.update_execution_engine_forkchoice_blocking(self.slot()?) {
+            crit!(
+                self.log,
+                "Failed to update execution head";
+                "error" => ?e
+            );
+        }
 
-            if let Err(e) = self.update_execution_engine_forkchoice_blocking(current_slot) {
-                crit!(
-                    self.log,
-                    "Failed to update execution head";
-                    "error" => ?e
-                );
-            }
-
-            // Performing this call immediately after
-            // `update_execution_engine_forkchoice_blocking` might result in two calls to fork
-            // choice updated, one *without* payload attributes and then a second *with*
-            // payload attributes.
-            //
-            // This seems OK. It's not a significant waste of EL<>CL bandwidth or resources, as
-            // far as I know.
-            if let Err(e) = self.prepare_beacon_proposer_blocking() {
-                crit!(
-                    self.log,
-                    "Failed to prepare proposers after fork choice";
-                    "error" => ?e
-                );
-            }
+        // Performing this call immediately after
+        // `update_execution_engine_forkchoice_blocking` might result in two calls to fork
+        // choice updated, one *without* payload attributes and then a second *with*
+        // payload attributes.
+        //
+        // This seems OK. It's not a significant waste of EL<>CL bandwidth or resources, as
+        // far as I know.
+        if let Err(e) = self.prepare_beacon_proposer_blocking() {
+            crit!(
+                self.log,
+                "Failed to prepare proposers after fork choice";
+                "error" => ?e
+            );
         }
 
         Ok(())
     }
 
     pub fn prepare_beacon_proposer_blocking(&self) -> Result<(), Error> {
+        let current_slot = self.slot()?;
+
+        // Avoids raising an error before Bellatrix.
+        //
+        // See `Self::prepare_beacon_proposer_async` for more detail.
+        if self.slot_is_prior_to_bellatrix(current_slot + 1) {
+            return Ok(());
+        }
+
         let execution_layer = self
             .execution_layer
             .as_ref()
             .ok_or(Error::ExecutionLayerMissing)?;
 
         execution_layer
-            .block_on_generic(|_| self.prepare_beacon_proposer_async())
+            .block_on_generic(|_| self.prepare_beacon_proposer_async(current_slot))
             .map_err(Error::PrepareProposerBlockingFailed)?
     }
 
@@ -3744,7 +3746,15 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// 1. We're in the tail-end of the slot (as defined by PAYLOAD_PREPARATION_LOOKAHEAD_FACTOR)
     /// 2. The head block is one slot (or less) behind the prepare slot (e.g., we're preparing for
     ///    the next slot and the block at the current slot is already known).
-    pub async fn prepare_beacon_proposer_async(&self) -> Result<(), Error> {
+    pub async fn prepare_beacon_proposer_async(&self, current_slot: Slot) -> Result<(), Error> {
+        let prepare_slot = current_slot + 1;
+        let prepare_epoch = prepare_slot.epoch(T::EthSpec::slots_per_epoch());
+
+        // There's no need to run the proposer preparation routine before the bellatrix fork.
+        if self.slot_is_prior_to_bellatrix(prepare_slot) {
+            return Ok(());
+        }
+
         let execution_layer = self
             .execution_layer
             .clone()
@@ -3757,7 +3767,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         }
 
         let head = self.head_info()?;
-        let current_slot = self.slot()?;
+        let head_epoch = head.slot.epoch(T::EthSpec::slots_per_epoch());
 
         // Don't bother with proposer prep if the head is more than
         // `PREPARE_PROPOSER_HISTORIC_EPOCHS` prior to the current slot.
@@ -3775,19 +3785,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             return Ok(());
         }
 
-        // We only start to push preparation data for some chain *after* the transition block
-        // has been imported.
-        //
-        // There is no payload preparation for the transition block (i.e., the first block with
-        // execution enabled in some chain).
-        if head.execution_payload_block_hash.is_none() {
-            return Ok(());
-        };
-
-        let head_epoch = head.slot.epoch(T::EthSpec::slots_per_epoch());
-        let prepare_slot = current_slot + 1;
-        let prepare_epoch = prepare_slot.epoch(T::EthSpec::slots_per_epoch());
-
         // Ensure that the shuffling decision root is correct relative to the epoch we wish to
         // query.
         let shuffling_decision_root = if head_epoch == prepare_epoch {
@@ -3954,6 +3951,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         &self,
         current_slot: Slot,
     ) -> Result<(), Error> {
+        // Avoids raising an error before Bellatrix.
+        //
+        // See `Self::update_execution_engine_forkchoice_async` for more detail.
+        if self.slot_is_prior_to_bellatrix(current_slot + 1) {
+            return Ok(());
+        }
+
         let execution_layer = self
             .execution_layer
             .as_ref()
@@ -3968,6 +3972,21 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         &self,
         current_slot: Slot,
     ) -> Result<(), Error> {
+        let next_slot = current_slot + 1;
+
+        // There is no need to issue a `forkchoiceUpdated` (fcU) message unless the Bellatrix fork
+        // has:
+        //
+        // 1. Already happened.
+        // 2. Will happen in the next slot.
+        //
+        // The reason for a fcU message in the slot prior to the Bellatrix fork is in case the
+        // terminal difficulty has already been reached and a payload preparation message needs to
+        // be issued.
+        if self.slot_is_prior_to_bellatrix(next_slot) {
+            return Ok(());
+        }
+
         let execution_layer = self
             .execution_layer
             .as_ref()
@@ -3994,34 +4013,71 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         // We are taking the `self.fork_choice` lock whilst holding the `forkchoice_lock`. This
         // is intentional, since it allows us to ensure a consistent ordering of messages to the
         // execution layer.
-        let (head_block_root, head_hash, finalized_hash) =
-            if let Some(params) = self.fork_choice.read().get_forkchoice_update_parameters() {
-                if let Some(head_hash) = params.head_hash {
-                    (
-                        params.head_root,
-                        head_hash,
-                        params
-                            .finalized_hash
-                            .unwrap_or_else(ExecutionBlockHash::zero),
-                    )
-                } else {
-                    // The head block does not have an execution block hash, there is no need to
-                    // send an update to the EL.
-                    return Ok(());
-                }
+        let forkchoice_update_parameters =
+            self.fork_choice.read().get_forkchoice_update_parameters();
+        let (head_block_root, head_hash, finalized_hash) = if let Some(params) =
+            forkchoice_update_parameters
+        {
+            if let Some(head_hash) = params.head_hash {
+                (
+                    params.head_root,
+                    head_hash,
+                    params
+                        .finalized_hash
+                        .unwrap_or_else(ExecutionBlockHash::zero),
+                )
             } else {
-                warn!(
-                    self.log,
-                    "Missing forkchoice params";
-                    "msg" => "please report this non-critical bug"
-                );
-                return Ok(());
-            };
+                // The head block does not have an execution block hash. We must check to see if we
+                // happen to be the proposer of the transition block, in which case we still need to
+                // send forkchoice_updated.
+                match self.spec.fork_name_at_slot::<T::EthSpec>(next_slot) {
+                    // We are pre-bellatrix; no need to update the EL.
+                    ForkName::Base | ForkName::Altair => return Ok(()),
+                    _ => {
+                        // We are post-bellatrix
+                        if execution_layer
+                            .payload_attributes(next_slot, params.head_root)
+                            .await
+                            .is_some()
+                        {
+                            // We are a proposer, check for terminal_pow_block_hash
+                            if let Some(terminal_pow_block_hash) = execution_layer
+                                .get_terminal_pow_block_hash(&self.spec)
+                                .await
+                                .map_err(Error::ForkchoiceUpdate)?
+                            {
+                                info!(
+                                    self.log,
+                                    "Prepared POS transition block proposer"; "slot" => next_slot
+                                );
+                                (
+                                    params.head_root,
+                                    terminal_pow_block_hash,
+                                    params
+                                        .finalized_hash
+                                        .unwrap_or_else(ExecutionBlockHash::zero),
+                                )
+                            } else {
+                                // TTD hasn't been reached yet, no need to update the EL.
+                                return Ok(());
+                            }
+                        } else {
+                            // We are not a proposer, no need to update the EL.
+                            return Ok(());
+                        }
+                    }
+                }
+            }
+        } else {
+            warn!(
+                self.log,
+                "Missing forkchoice params";
+                "msg" => "please report this non-critical bug"
+            );
+            return Ok(());
+        };
 
-        let forkchoice_updated_response = self
-            .execution_layer
-            .as_ref()
-            .ok_or(Error::ExecutionLayerMissing)?
+        let forkchoice_updated_response = execution_layer
             .notify_forkchoice_updated(head_hash, finalized_hash, current_slot, head_block_root)
             .await
             .map_err(Error::ExecutionForkChoiceUpdateFailed);
@@ -4032,7 +4088,25 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
         match forkchoice_updated_response {
             Ok(status) => match &status {
-                PayloadStatus::Valid | PayloadStatus::Syncing => Ok(()),
+                PayloadStatus::Valid => {
+                    // Ensure that fork choice knows that the block is no longer optimistic.
+                    if let Err(e) = self
+                        .fork_choice
+                        .write()
+                        .on_valid_execution_payload(head_block_root)
+                    {
+                        error!(
+                            self.log,
+                            "Failed to validate payload";
+                            "error" => ?e
+                        )
+                    };
+                    Ok(())
+                }
+                // There's nothing to be done for a syncing response. If the block is already
+                // `SYNCING` in fork choice, there's nothing to do. If already known to be `VALID`
+                // or `INVALID` then we don't want to change it to syncing.
+                PayloadStatus::Syncing => Ok(()),
                 // The specification doesn't list `ACCEPTED` as a valid response to a fork choice
                 // update. This response *seems* innocent enough, so we won't return early with an
                 // error. However, we create a log to bring attention to the issue.
@@ -4091,6 +4165,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         }
     }
 
+    /// Returns `true` if the given slot is prior to the `bellatrix_fork_epoch`.
+    fn slot_is_prior_to_bellatrix(&self, slot: Slot) -> bool {
+        self.spec.bellatrix_fork_epoch.map_or(true, |bellatrix| {
+            slot.epoch(T::EthSpec::slots_per_epoch()) < bellatrix
+        })
+    }
+
     /// Returns the status of the current head block, regarding the validity of the execution
     /// payload.
     pub fn head_safety_status(&self) -> Result<HeadSafetyStatus, BeaconChainError> {

beacon_node/beacon_chain/src/block_verification.rs

@@ -1006,21 +1006,25 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
         parent: PreProcessingSnapshot<T::EthSpec>,
         chain: &BeaconChain<T>,
     ) -> Result<Self, BlockError<T::EthSpec>> {
-        // Reject any block if its parent is not known to fork choice.
-        //
-        // A block that is not in fork choice is either:
-        //
-        //  - Not yet imported: we should reject this block because we should only import a child
-        //  after its parent has been fully imported.
-        //  - Pre-finalized: if the parent block is _prior_ to finalization, we should ignore it
-        //  because it will revert finalization. Note that the finalized block is stored in fork
-        //  choice, so we will not reject any child of the finalized block (this is relevant during
-        //  genesis).
-        if !chain
-            .fork_choice
-            .read()
-            .contains_block(&block.parent_root())
-        {
+        if let Some(parent) = chain.fork_choice.read().get_block(&block.parent_root()) {
+            // Reject any block where the parent has an invalid payload. It's impossible for a valid
+            // block to descend from an invalid parent.
+            if parent.execution_status.is_invalid() {
+                return Err(BlockError::ParentExecutionPayloadInvalid {
+                    parent_root: block.parent_root(),
+                });
+            }
+        } else {
+            // Reject any block if its parent is not known to fork choice.
+            //
+            // A block that is not in fork choice is either:
+            //
+            //  - Not yet imported: we should reject this block because we should only import a child
+            //  after its parent has been fully imported.
+            //  - Pre-finalized: if the parent block is _prior_ to finalization, we should ignore it
+            //  because it will revert finalization. Note that the finalized block is stored in fork
+            //  choice, so we will not reject any child of the finalized block (this is relevant during
+            //  genesis).
             return Err(BlockError::ParentUnknown(Box::new(block)));
         }
 

beacon_node/beacon_chain/src/proposer_prep_service.rs

@@ -50,12 +50,19 @@ async fn proposer_prep_service<T: BeaconChainTypes>(
                 let inner_chain = chain.clone();
                 executor.spawn(
                     async move {
-                        if let Err(e) = inner_chain.prepare_beacon_proposer_async().await {
-                            error!(
-                                inner_chain.log,
-                                "Proposer prepare routine failed";
-                                "error" => ?e
-                            );
+                        if let Ok(current_slot) = inner_chain.slot() {
+                            if let Err(e) = inner_chain
+                                .prepare_beacon_proposer_async(current_slot)
+                                .await
+                            {
+                                error!(
+                                    inner_chain.log,
+                                    "Proposer prepare routine failed";
+                                    "error" => ?e
+                                );
+                            }
+                        } else {
+                            debug!(inner_chain.log, "No slot for proposer prepare routine");
                         }
                     },
                     "proposer_prep_update",

beacon_node/beacon_chain/tests/payload_invalidation.rs

@@ -6,10 +6,13 @@ use beacon_chain::{
     WhenSlotSkipped, INVALID_JUSTIFIED_PAYLOAD_SHUTDOWN_REASON,
 };
 use execution_layer::{
-    json_structures::JsonPayloadAttributesV1, ExecutionLayer, PayloadAttributes,
+    json_structures::{JsonForkChoiceStateV1, JsonPayloadAttributesV1},
+    ExecutionLayer, ForkChoiceState, PayloadAttributes,
 };
-use proto_array::ExecutionStatus;
+use fork_choice::{Error as ForkChoiceError, InvalidationOperation, PayloadVerificationStatus};
+use proto_array::{Error as ProtoArrayError, ExecutionStatus};
 use slot_clock::SlotClock;
+use std::time::Duration;
 use task_executor::ShutdownReason;
 use types::*;
 
@@ -93,17 +96,28 @@ impl InvalidPayloadRig {
         self.harness.chain.head_info().unwrap()
     }
 
-    fn previous_payload_attributes(&self) -> PayloadAttributes {
+    fn previous_forkchoice_update_params(&self) -> (ForkChoiceState, PayloadAttributes) {
         let mock_execution_layer = self.harness.mock_execution_layer.as_ref().unwrap();
         let json = mock_execution_layer
             .server
             .take_previous_request()
             .expect("no previous request");
         let params = json.get("params").expect("no params");
+
+        let fork_choice_state_json = params.get(0).expect("no payload param");
+        let fork_choice_state: JsonForkChoiceStateV1 =
+            serde_json::from_value(fork_choice_state_json.clone()).unwrap();
+
         let payload_param_json = params.get(1).expect("no payload param");
         let attributes: JsonPayloadAttributesV1 =
             serde_json::from_value(payload_param_json.clone()).unwrap();
-        attributes.into()
+
+        (fork_choice_state.into(), attributes.into())
+    }
+
+    fn previous_payload_attributes(&self) -> PayloadAttributes {
+        let (_, payload_attributes) = self.previous_forkchoice_update_params();
+        payload_attributes
     }
 
     fn move_to_terminal_block(&self) {
@@ -115,6 +129,16 @@ impl InvalidPayloadRig {
             .unwrap();
     }
 
+    fn latest_execution_block_hash(&self) -> ExecutionBlockHash {
+        let mock_execution_layer = self.harness.mock_execution_layer.as_ref().unwrap();
+        mock_execution_layer
+            .server
+            .execution_block_generator()
+            .latest_execution_block()
+            .unwrap()
+            .block_hash
+    }
+
     fn build_blocks(&mut self, num_blocks: u64, is_valid: Payload) -> Vec<Hash256> {
         (0..num_blocks)
             .map(|_| self.import_block(is_valid.clone()))
@@ -147,6 +171,15 @@ impl InvalidPayloadRig {
             .unwrap()
     }
 
+    fn validate_manually(&self, block_root: Hash256) {
+        self.harness
+            .chain
+            .fork_choice
+            .write()
+            .on_valid_execution_payload(block_root)
+            .unwrap();
+    }
+
     fn import_block_parametric<F: Fn(&BlockError<E>) -> bool>(
         &mut self,
         is_valid: Payload,
@@ -233,6 +266,13 @@ impl InvalidPayloadRig {
 
         block_root
     }
+
+    fn invalidate_manually(&self, block_root: Hash256) {
+        self.harness
+            .chain
+            .process_invalid_execution_payload(&InvalidationOperation::InvalidateOne { block_root })
+            .unwrap();
+    }
 }
 
 /// Simple test of the different import types.
@@ -642,6 +682,42 @@ fn invalid_after_optimistic_sync() {
     assert_eq!(head.block_root, roots[1]);
 }
 
+#[test]
+fn manually_validate_child() {
+    let mut rig = InvalidPayloadRig::new().enable_attestations();
+    rig.move_to_terminal_block();
+    rig.import_block(Payload::Valid); // Import a valid transition block.
+
+    let parent = rig.import_block(Payload::Syncing);
+    let child = rig.import_block(Payload::Syncing);
+
+    assert!(rig.execution_status(parent).is_not_verified());
+    assert!(rig.execution_status(child).is_not_verified());
+
+    rig.validate_manually(child);
+
+    assert!(rig.execution_status(parent).is_valid());
+    assert!(rig.execution_status(child).is_valid());
+}
+
+#[test]
+fn manually_validate_parent() {
+    let mut rig = InvalidPayloadRig::new().enable_attestations();
+    rig.move_to_terminal_block();
+    rig.import_block(Payload::Valid); // Import a valid transition block.
+
+    let parent = rig.import_block(Payload::Syncing);
+    let child = rig.import_block(Payload::Syncing);
+
+    assert!(rig.execution_status(parent).is_not_verified());
+    assert!(rig.execution_status(child).is_not_verified());
+
+    rig.validate_manually(parent);
+
+    assert!(rig.execution_status(parent).is_valid());
+    assert!(rig.execution_status(child).is_not_verified());
+}
+
 #[test]
 fn payload_preparation() {
     let mut rig = InvalidPayloadRig::new();
@@ -693,3 +769,119 @@ fn payload_preparation() {
     };
     assert_eq!(rig.previous_payload_attributes(), payload_attributes);
 }
+
+#[test]
+fn invalid_parent() {
+    let mut rig = InvalidPayloadRig::new();
+    rig.move_to_terminal_block();
+    rig.import_block(Payload::Valid); // Import a valid transition block.
+
+    // Import a syncing block atop the transition block (we'll call this the "parent block" since we
+    // build another block on it later).
+    let parent_root = rig.import_block(Payload::Syncing);
+    let parent_block = rig.harness.get_block(parent_root.into()).unwrap();
+    let parent_state = rig
+        .harness
+        .get_hot_state(parent_block.state_root().into())
+        .unwrap();
+
+    // Produce another block atop the parent, but don't import yet.
+    let slot = parent_block.slot() + 1;
+    rig.harness.set_current_slot(slot);
+    let (block, state) = rig.harness.make_block(parent_state, slot);
+    let block_root = block.canonical_root();
+    assert_eq!(block.parent_root(), parent_root);
+
+    // Invalidate the parent block.
+    rig.invalidate_manually(parent_root);
+    assert!(rig.execution_status(parent_root).is_invalid());
+
+    // Ensure the block built atop an invalid payload is invalid for gossip.
+    assert!(matches!(
+        rig.harness.chain.verify_block_for_gossip(block.clone()),
+        Err(BlockError::ParentExecutionPayloadInvalid { parent_root: invalid_root })
+        if invalid_root == parent_root
+    ));
+
+    // Ensure the block built atop an invalid payload is invalid for import.
+    assert!(matches!(
+        rig.harness.chain.process_block(block.clone()),
+        Err(BlockError::ParentExecutionPayloadInvalid { parent_root: invalid_root })
+        if invalid_root == parent_root
+    ));
+
+    // Ensure the block built atop an invalid payload cannot be imported to fork choice.
+    let (block, _block_signature) = block.deconstruct();
+    assert!(matches!(
+        rig.harness.chain.fork_choice.write().on_block(
+            slot,
+            &block,
+            block_root,
+            Duration::from_secs(0),
+            &state,
+            PayloadVerificationStatus::NotVerified,
+            &rig.harness.chain.spec
+        ),
+        Err(ForkChoiceError::ProtoArrayError(message))
+        if message.contains(&format!(
+            "{:?}",
+            ProtoArrayError::ParentExecutionStatusIsInvalid {
+                block_root,
+                parent_root
+            }
+        ))
+    ));
+}
+
+/// Tests to ensure that we will still send a proposer preparation
+#[test]
+fn payload_preparation_before_transition_block() {
+    let rig = InvalidPayloadRig::new();
+    let el = rig.execution_layer();
+
+    let head = rig.harness.chain.head().unwrap();
+    let head_info = rig.head_info();
+    assert!(
+        !head_info.is_merge_transition_complete,
+        "the head block is pre-transition"
+    );
+    assert_eq!(
+        head_info.execution_payload_block_hash,
+        Some(ExecutionBlockHash::zero()),
+        "the head block is post-bellatrix"
+    );
+
+    let current_slot = rig.harness.chain.slot().unwrap();
+    let next_slot = current_slot + 1;
+    let proposer = head
+        .beacon_state
+        .get_beacon_proposer_index(next_slot, &rig.harness.chain.spec)
+        .unwrap();
+    let fee_recipient = Address::repeat_byte(99);
+
+    // Provide preparation data to the EL for `proposer`.
+    el.update_proposer_preparation_blocking(
+        Epoch::new(0),
+        &[ProposerPreparationData {
+            validator_index: proposer as u64,
+            fee_recipient,
+        }],
+    )
+    .unwrap();
+
+    rig.move_to_terminal_block();
+
+    rig.harness
+        .chain
+        .prepare_beacon_proposer_blocking()
+        .unwrap();
+    rig.harness
+        .chain
+        .update_execution_engine_forkchoice_blocking(current_slot)
+        .unwrap();
+
+    let (fork_choice_state, payload_attributes) = rig.previous_forkchoice_update_params();
+    let latest_block_hash = rig.latest_execution_block_hash();
+    assert_eq!(payload_attributes.suggested_fee_recipient, fee_recipient);
+    assert_eq!(fork_choice_state.head_block_hash, latest_block_hash);
+}

beacon_node/execution_layer/src/lib.rs

@@ -4,11 +4,17 @@
 //! This crate only provides useful functionality for "The Merge", it does not provide any of the
 //! deposit-contract functionality that the `beacon_node/eth1` crate already provides.
 
+use crate::engine_api::Builder;
+use crate::engines::Builders;
 use auth::{Auth, JwtKey};
-use engine_api::{Error as ApiError, *};
-use engines::{Engine, EngineError, Engines, ForkChoiceState, Logging};
+use engine_api::Error as ApiError;
+pub use engine_api::*;
+pub use engine_api::{http, http::HttpJsonRpc};
+pub use engines::ForkChoiceState;
+use engines::{Engine, EngineError, Engines, Logging};
 use lru::LruCache;
 use payload_status::process_multiple_payload_statuses;
+pub use payload_status::PayloadStatus;
 use sensitive_url::SensitiveUrl;
 use serde::{Deserialize, Serialize};
 use slog::{crit, debug, error, info, trace, Logger};
@@ -30,12 +36,6 @@ use types::{
     ProposerPreparationData, SignedBeaconBlock, Slot,
 };
 
-use crate::engine_api::Builder;
-use crate::engines::Builders;
-pub use engine_api::*;
-pub use engine_api::{http, http::HttpJsonRpc};
-pub use payload_status::PayloadStatus;
-
 mod engine_api;
 mod engines;
 mod metrics;

beacon_node/execution_layer/src/test_utils/handle_rpc.rs

@@ -100,7 +100,9 @@ pub async fn handle_rpc<T: EthSpec>(
             let forkchoice_state: JsonForkChoiceStateV1 = get_param(params, 0)?;
             let payload_attributes: Option<JsonPayloadAttributesV1> = get_param(params, 1)?;
 
-            let response = ctx
+            let head_block_hash = forkchoice_state.head_block_hash;
+
+            let mut response = ctx
                 .execution_block_generator
                 .write()
                 .forkchoice_updated_v1(
@@ -108,6 +110,14 @@ pub async fn handle_rpc<T: EthSpec>(
                     payload_attributes.map(|json| json.into()),
                 )?;
 
+            if let Some(mut status) = ctx.static_forkchoice_updated_response.lock().clone() {
+                if status.status == PayloadStatusV1Status::Valid {
+                    status.latest_valid_hash = Some(head_block_hash)
+                }
+
+                response.payload_status = status.into();
+            }
+
             Ok(serde_json::to_value(response).unwrap())
         }
         other => Err(format!(

beacon_node/execution_layer/src/test_utils/mod.rs

@@ -68,6 +68,7 @@ impl<T: EthSpec> MockServer<T> {
             previous_request: <_>::default(),
             preloaded_responses,
             static_new_payload_response: <_>::default(),
+            static_forkchoice_updated_response: <_>::default(),
             _phantom: PhantomData,
         });
 
@@ -134,6 +135,7 @@ impl<T: EthSpec> MockServer<T> {
             },
             should_import: true,
         };
+        *self.ctx.static_forkchoice_updated_response.lock() = Some(response.status.clone());
         *self.ctx.static_new_payload_response.lock() = Some(response)
     }
 
@@ -148,6 +150,7 @@ impl<T: EthSpec> MockServer<T> {
             },
             should_import,
         };
+        *self.ctx.static_forkchoice_updated_response.lock() = Some(response.status.clone());
         *self.ctx.static_new_payload_response.lock() = Some(response)
     }
 
@@ -160,6 +163,7 @@ impl<T: EthSpec> MockServer<T> {
             },
             should_import: true,
         };
+        *self.ctx.static_forkchoice_updated_response.lock() = Some(response.status.clone());
         *self.ctx.static_new_payload_response.lock() = Some(response)
     }
 
@@ -248,6 +252,7 @@ pub struct Context<T: EthSpec> {
     pub preloaded_responses: Arc<Mutex<Vec<serde_json::Value>>>,
     pub previous_request: Arc<Mutex<Option<serde_json::Value>>>,
     pub static_new_payload_response: Arc<Mutex<Option<StaticNewPayloadResponse>>>,
+    pub static_forkchoice_updated_response: Arc<Mutex<Option<PayloadStatusV1>>>,
     pub _phantom: PhantomData<T>,
 }
 

beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs

@@ -659,9 +659,11 @@ mod tests {
         ForkContext::new::<Spec>(current_slot, Hash256::zero(), &chain_spec)
     }
 
-    fn base_block() -> SignedBeaconBlock<Spec> {
-        let full_block = BeaconBlock::Base(BeaconBlockBase::<Spec>::full(&Spec::default_spec()));
-        SignedBeaconBlock::from_block(full_block, Signature::empty())
+    /// Smallest sized block across all current forks. Useful for testing
+    /// min length check conditions.
+    fn empty_base_block() -> SignedBeaconBlock<Spec> {
+        let empty_block = BeaconBlock::Base(BeaconBlockBase::<Spec>::empty(&Spec::default_spec()));
+        SignedBeaconBlock::from_block(empty_block, Signature::empty())
     }
 
     fn altair_block() -> SignedBeaconBlock<Spec> {
@@ -830,10 +832,12 @@ mod tests {
             encode_then_decode(
                 Protocol::BlocksByRange,
                 Version::V1,
-                RPCCodedResponse::Success(RPCResponse::BlocksByRange(Box::new(base_block()))),
+                RPCCodedResponse::Success(RPCResponse::BlocksByRange(Box::new(empty_base_block()))),
                 ForkName::Base,
             ),
-            Ok(Some(RPCResponse::BlocksByRange(Box::new(base_block()))))
+            Ok(Some(RPCResponse::BlocksByRange(Box::new(
+                empty_base_block()
+            ))))
         );
 
         assert!(
@@ -854,10 +858,12 @@ mod tests {
             encode_then_decode(
                 Protocol::BlocksByRoot,
                 Version::V1,
-                RPCCodedResponse::Success(RPCResponse::BlocksByRoot(Box::new(base_block()))),
+                RPCCodedResponse::Success(RPCResponse::BlocksByRoot(Box::new(empty_base_block()))),
                 ForkName::Base,
             ),
-            Ok(Some(RPCResponse::BlocksByRoot(Box::new(base_block()))))
+            Ok(Some(RPCResponse::BlocksByRoot(
+                Box::new(empty_base_block())
+            )))
         );
 
         assert!(
@@ -941,10 +947,27 @@ mod tests {
             encode_then_decode(
                 Protocol::BlocksByRange,
                 Version::V2,
-                RPCCodedResponse::Success(RPCResponse::BlocksByRange(Box::new(base_block()))),
+                RPCCodedResponse::Success(RPCResponse::BlocksByRange(Box::new(empty_base_block()))),
                 ForkName::Base,
             ),
-            Ok(Some(RPCResponse::BlocksByRange(Box::new(base_block()))))
+            Ok(Some(RPCResponse::BlocksByRange(Box::new(
+                empty_base_block()
+            ))))
+        );
+
+        // Decode the smallest possible base block when current fork is altair
+        // This is useful for checking that we allow for blocks smaller than
+        // the current_fork's rpc limit
+        assert_eq!(
+            encode_then_decode(
+                Protocol::BlocksByRange,
+                Version::V2,
+                RPCCodedResponse::Success(RPCResponse::BlocksByRange(Box::new(empty_base_block()))),
+                ForkName::Altair,
+            ),
+            Ok(Some(RPCResponse::BlocksByRange(Box::new(
+                empty_base_block()
+            ))))
         );
 
         assert_eq!(
@@ -996,10 +1019,27 @@ mod tests {
             encode_then_decode(
                 Protocol::BlocksByRoot,
                 Version::V2,
-                RPCCodedResponse::Success(RPCResponse::BlocksByRoot(Box::new(base_block()))),
+                RPCCodedResponse::Success(RPCResponse::BlocksByRoot(Box::new(empty_base_block()))),
                 ForkName::Base,
             ),
-            Ok(Some(RPCResponse::BlocksByRoot(Box::new(base_block())))),
+            Ok(Some(RPCResponse::BlocksByRoot(
+                Box::new(empty_base_block())
+            ))),
+        );
+
+        // Decode the smallest possible base block when current fork is altair
+        // This is useful for checking that we allow for blocks smaller than
+        // the current_fork's rpc limit
+        assert_eq!(
+            encode_then_decode(
+                Protocol::BlocksByRoot,
+                Version::V2,
+                RPCCodedResponse::Success(RPCResponse::BlocksByRoot(Box::new(empty_base_block()))),
+                ForkName::Altair,
+            ),
+            Ok(Some(RPCResponse::BlocksByRoot(
+                Box::new(empty_base_block())
+            )))
         );
 
         assert_eq!(
@@ -1073,7 +1113,7 @@ mod tests {
         let mut encoded_bytes = encode(
             Protocol::BlocksByRange,
             Version::V2,
-            RPCCodedResponse::Success(RPCResponse::BlocksByRange(Box::new(base_block()))),
+            RPCCodedResponse::Success(RPCResponse::BlocksByRange(Box::new(empty_base_block()))),
             ForkName::Base,
         )
         .unwrap();
@@ -1094,7 +1134,7 @@ mod tests {
         let mut encoded_bytes = encode(
             Protocol::BlocksByRoot,
             Version::V2,
-            RPCCodedResponse::Success(RPCResponse::BlocksByRoot(Box::new(base_block()))),
+            RPCCodedResponse::Success(RPCResponse::BlocksByRoot(Box::new(empty_base_block()))),
             ForkName::Base,
         )
         .unwrap();
@@ -1116,7 +1156,7 @@ mod tests {
         let mut encoded_bytes = encode(
             Protocol::BlocksByRange,
             Version::V2,
-            RPCCodedResponse::Success(RPCResponse::BlocksByRange(Box::new(base_block()))),
+            RPCCodedResponse::Success(RPCResponse::BlocksByRange(Box::new(empty_base_block()))),
             ForkName::Altair,
         )
         .unwrap();
@@ -1186,7 +1226,7 @@ mod tests {
         let mut encoded_bytes = encode(
             Protocol::BlocksByRoot,
             Version::V2,
-            RPCCodedResponse::Success(RPCResponse::BlocksByRoot(Box::new(base_block()))),
+            RPCCodedResponse::Success(RPCResponse::BlocksByRoot(Box::new(empty_base_block()))),
             ForkName::Altair,
         )
         .unwrap();
@@ -1210,7 +1250,7 @@ mod tests {
         let mut encoded_bytes = encode(
             Protocol::BlocksByRoot,
             Version::V2,
-            RPCCodedResponse::Success(RPCResponse::BlocksByRoot(Box::new(base_block()))),
+            RPCCodedResponse::Success(RPCResponse::BlocksByRoot(Box::new(empty_base_block()))),
             ForkName::Altair,
         )
         .unwrap();

beacon_node/lighthouse_network/src/rpc/protocol.rs

@@ -118,6 +118,26 @@ pub fn max_rpc_size(fork_context: &ForkContext) -> usize {
     }
 }
 
+/// Returns the rpc limits for beacon_block_by_range and beacon_block_by_root responses.
+///
+/// Note: This function should take care to return the min/max limits accounting for all
+/// previous valid forks when adding a new fork variant.
+pub fn rpc_block_limits_by_fork(current_fork: ForkName) -> RpcLimits {
+    match &current_fork {
+        ForkName::Base => {
+            RpcLimits::new(*SIGNED_BEACON_BLOCK_BASE_MIN, *SIGNED_BEACON_BLOCK_BASE_MAX)
+        }
+        ForkName::Altair => RpcLimits::new(
+            *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair blocks
+            *SIGNED_BEACON_BLOCK_ALTAIR_MAX, // Altair block is larger than base blocks
+        ),
+        ForkName::Merge => RpcLimits::new(
+            *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and merge blocks
+            *SIGNED_BEACON_BLOCK_MERGE_MAX, // Merge block is larger than base and altair blocks
+        ),
+    }
+}
+
 /// Protocol names to be used.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Protocol {
@@ -281,32 +301,8 @@ impl ProtocolId {
                 <StatusMessage as Encode>::ssz_fixed_len(),
             ),
             Protocol::Goodbye => RpcLimits::new(0, 0), // Goodbye request has no response
-            Protocol::BlocksByRange => match fork_context.current_fork() {
-                ForkName::Base => {
-                    RpcLimits::new(*SIGNED_BEACON_BLOCK_BASE_MIN, *SIGNED_BEACON_BLOCK_BASE_MAX)
-                }
-                ForkName::Altair => RpcLimits::new(
-                    *SIGNED_BEACON_BLOCK_ALTAIR_MIN,
-                    *SIGNED_BEACON_BLOCK_ALTAIR_MAX,
-                ),
-                ForkName::Merge => RpcLimits::new(
-                    *SIGNED_BEACON_BLOCK_MERGE_MIN,
-                    *SIGNED_BEACON_BLOCK_MERGE_MAX,
-                ),
-            },
-            Protocol::BlocksByRoot => match fork_context.current_fork() {
-                ForkName::Base => {
-                    RpcLimits::new(*SIGNED_BEACON_BLOCK_BASE_MIN, *SIGNED_BEACON_BLOCK_BASE_MAX)
-                }
-                ForkName::Altair => RpcLimits::new(
-                    *SIGNED_BEACON_BLOCK_ALTAIR_MIN,
-                    *SIGNED_BEACON_BLOCK_ALTAIR_MAX,
-                ),
-                ForkName::Merge => RpcLimits::new(
-                    *SIGNED_BEACON_BLOCK_MERGE_MIN,
-                    *SIGNED_BEACON_BLOCK_MERGE_MAX,
-                ),
-            },
+            Protocol::BlocksByRange => rpc_block_limits_by_fork(fork_context.current_fork()),
+            Protocol::BlocksByRoot => rpc_block_limits_by_fork(fork_context.current_fork()),
 
             Protocol::Ping => RpcLimits::new(
                 <Ping as Encode>::ssz_fixed_len(),

beacon_node/src/cli.rs

@@ -383,7 +383,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .value_name("SLOT_COUNT")
                 .help("Specifies how often a freezer DB restore point should be stored. \
                        Cannot be changed after initialization. \
-                       [default: 2048 (mainnet) or 64 (minimal)]")
+                       [default: 8192 (mainnet) or 64 (minimal)]")
                 .takes_value(true)
         )
         .arg(

boot_node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "boot_node"
-version = "2.2.0"
+version = "2.2.1"
 authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2021"
 

common/lighthouse_version/src/lib.rs

@@ -16,7 +16,7 @@ pub const VERSION: &str = git_version!(
         // NOTE: using --match instead of --exclude for compatibility with old Git
         "--match=thiswillnevermatchlol"
     ],
-    prefix = "Lighthouse/v2.2.0-",
+    prefix = "Lighthouse/v2.2.1-",
     fallback = "unknown"
 );
 

consensus/fork_choice/src/fork_choice.rs

@@ -18,6 +18,7 @@ pub enum Error<T> {
     InvalidProtoArrayBytes(String),
     InvalidLegacyProtoArrayBytes(String),
     FailedToProcessInvalidExecutionPayload(String),
+    FailedToProcessValidExecutionPayload(String),
     MissingProtoArrayBlock(Hash256),
     UnknownAncestor {
         ancestor_slot: Slot,
@@ -512,6 +513,16 @@ where
         Ok(true)
     }
 
+    /// See `ProtoArrayForkChoice::process_execution_payload_validation` for documentation.
+    pub fn on_valid_execution_payload(
+        &mut self,
+        block_root: Hash256,
+    ) -> Result<(), Error<T::Error>> {
+        self.proto_array
+            .process_execution_payload_validation(block_root)
+            .map_err(Error::FailedToProcessValidExecutionPayload)
+    }
+
     /// See `ProtoArrayForkChoice::process_execution_payload_invalidation` for documentation.
     pub fn on_invalid_execution_payload(
         &mut self,

consensus/proto_array/src/error.rs

@@ -44,6 +44,10 @@ pub enum Error {
     IrrelevantDescendant {
         block_root: Hash256,
     },
+    ParentExecutionStatusIsInvalid {
+        block_root: Hash256,
+        parent_root: Hash256,
+    },
 }
 
 #[derive(Clone, PartialEq, Debug)]

consensus/proto_array/src/proto_array.rs

@@ -315,6 +315,21 @@ impl ProtoArray {
             execution_status: block.execution_status,
         };
 
+        // If the parent has an invalid execution status, return an error before adding the block to
+        // `self`.
+        if let Some(parent_index) = node.parent {
+            let parent = self
+                .nodes
+                .get(parent_index)
+                .ok_or(Error::InvalidNodeIndex(parent_index))?;
+            if parent.execution_status.is_invalid() {
+                return Err(Error::ParentExecutionStatusIsInvalid {
+                    block_root: block.root,
+                    parent_root: parent.root,
+                });
+            }
+        }
+
         self.indices.insert(node.root, node_index);
         self.nodes.push(node.clone());
 
@@ -322,20 +337,37 @@ impl ProtoArray {
             self.maybe_update_best_child_and_descendant(parent_index, node_index)?;
 
             if matches!(block.execution_status, ExecutionStatus::Valid(_)) {
-                self.propagate_execution_payload_validation(parent_index)?;
+                self.propagate_execution_payload_validation_by_index(parent_index)?;
             }
         }
 
         Ok(())
     }
 
+    /// Updates the `block_root` and all ancestors to have validated execution payloads.
+    ///
+    /// Returns an error if:
+    ///
+    /// - The `block-root` is unknown.
+    /// - Any of the to-be-validated payloads are already invalid.
+    pub fn propagate_execution_payload_validation(
+        &mut self,
+        block_root: Hash256,
+    ) -> Result<(), Error> {
+        let index = *self
+            .indices
+            .get(&block_root)
+            .ok_or(Error::NodeUnknown(block_root))?;
+        self.propagate_execution_payload_validation_by_index(index)
+    }
+
     /// Updates the `verified_node_index` and all ancestors to have validated execution payloads.
     ///
     /// Returns an error if:
     ///
     /// - The `verified_node_index` is unknown.
     /// - Any of the to-be-validated payloads are already invalid.
-    pub fn propagate_execution_payload_validation(
+    fn propagate_execution_payload_validation_by_index(
         &mut self,
         verified_node_index: usize,
     ) -> Result<(), Error> {
@@ -460,7 +492,7 @@ impl ProtoArray {
 
                         // It might be new knowledge that this block is valid, ensure that it and all
                         // ancestors are marked as valid.
-                        self.propagate_execution_payload_validation(index)?;
+                        self.propagate_execution_payload_validation_by_index(index)?;
                         break;
                     }
                 }

consensus/proto_array/src/proto_array_fork_choice.rs

@@ -188,6 +188,16 @@ impl ProtoArrayForkChoice {
         })
     }
 
+    /// See `ProtoArray::propagate_execution_payload_validation` for documentation.
+    pub fn process_execution_payload_validation(
+        &mut self,
+        block_root: Hash256,
+    ) -> Result<(), String> {
+        self.proto_array
+            .propagate_execution_payload_validation(block_root)
+            .map_err(|e| format!("Failed to process valid payload: {:?}", e))
+    }
+
     /// See `ProtoArray::propagate_execution_payload_invalidation` for documentation.
     pub fn process_execution_payload_invalidation(
         &mut self,

lcli/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lcli"
 description = "Lighthouse CLI (modeled after zcli)"
-version = "2.2.0"
+version = "2.2.1"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2021"
 

lighthouse/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lighthouse"
-version = "2.2.0"
+version = "2.2.1"
 authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2021"
 autotests = false

testing/simulator/src/sync_sim.rs

@@ -62,6 +62,9 @@ fn syncing_sim(
     let end_after_checks = true;
     let eth1_block_time = Duration::from_millis(15_000 / speed_up_factor);
 
+    // Set fork epochs to test syncing across fork boundaries
+    spec.altair_fork_epoch = Some(Epoch::new(1));
+    spec.bellatrix_fork_epoch = Some(Epoch::new(2));
     spec.seconds_per_slot /= speed_up_factor;
     spec.seconds_per_slot = max(1, spec.seconds_per_slot);
     spec.eth1_follow_distance = 16;
@@ -86,6 +89,8 @@ fn syncing_sim(
     beacon_config.dummy_eth1_backend = true;
     beacon_config.sync_eth1_chain = true;
 
+    beacon_config.http_api.allow_sync_stalled = true;
+
     beacon_config.network.enr_address = Some(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)));
 
     // Generate the directories and keystores required for the validator clients.